Reland "Reland "Add support for GCS deps""
This is a reland of commit 46d5382f69895a756a26a7326f76ac41dde36920
Whats fixed: Removed the call_google_storage refactor so
that all `import download_from_google_storage` callsites
(in depot_tools and chromium/src) won't be affected.
Instead, gclient.py will import Gsutil from download_from_google_storage.
Original change's description:
> Reland "Add support for GCS deps"
>
> This is a reland of commit 3eedee7b55fe20103a3913f48844765217c837c9
>
> Fixed more imports in download_google_storage.py
>
> Original change's description:
> > Add support for GCS deps
> >
> > Also take out GCS calling logic from download_google_storage and
> > into call_google_storage.
> >
> > GCS deps look like:
> > 'src/third_party/node/linux': {
> > 'dep_type': 'gcs',
> > 'condition': 'checkout_linux',
> > 'bucket': 'chromium-nodejs/20.11.0',
> > 'object_name': '46795170ff5df9831955f163f6966abde581c8af',
> > 'sha256sum': '887504c37404898ca41b896f448ee6d7fc24179d8fb6a4b79d028ab7e1b7153d',
> > },
> >
> > 'src/third_party/llvm-build/Release+Asserts': {
> > 'dep_type': 'gcs',
> > 'condition': 'checkout_linux',
> > 'bucket': 'chromium-browser-clang',
> > 'object_name': 'Linux_x64/clang-llvmorg-18-init-17730-gf670112a-2.tar.xz',
> > 'sha256sum': '1e46df9b4e63c074064d75646310cb76be2f19815997a8486987189d80f991e8',
> > },
> >
> > Example directory for src/third_party/node/linux after gclient sync:
> > - tar_file.gz is the downloaded file from GCS.
> > - node_linux_x64/ is extracted in its path.
> > - `hash` contains the sha of GCS filename.
> > ```
> > chromium/src/ ->
> > third_party/node/linux/ ->
> > hash, tar_file.gz, node_linux_x64/
> > ```
> >
> > Bug: b/324418194
> > Change-Id: Ibcbbff27e211f194ddb8a08494af56570a84a12b
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5299722
> > Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> > Reviewed-by: Joanna Wang <jojwang@chromium.org>
>
> Bug: b/324418194
> Change-Id: Ie64265a86abcec0135408715a45c32a8bb7c7408
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5346338
> Reviewed-by: Joanna Wang <jojwang@chromium.org>
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Bug: b/324418194
Change-Id: I8b58dadbaa740fd9da1fbaf29b3b6ff5ef67fd12
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5352896
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Joanna Wang <jojwang@chromium.org>
1 year ago
|
|
|
#!/usr/bin/env vpython3
|
|
|
|
# Copyright (c) 2024 The Chromium Authors. All rights reserved.
|
|
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
|
|
# found in the LICENSE file.
|
|
|
|
"""Smoke tests for gclient.py.
|
|
|
|
|
|
|
|
Shell out 'gclient' and run gcs tests.
|
|
|
|
"""
|
|
|
|
|
|
|
|
import logging
|
|
|
|
import os
|
|
|
|
import sys
|
|
|
|
import unittest
|
|
|
|
|
|
|
|
from unittest import mock
|
|
|
|
import gclient_smoketest_base
|
|
|
|
import subprocess2
|
|
|
|
|
|
|
|
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
|
|
|
|
|
|
|
|
class GClientSmokeGcs(gclient_smoketest_base.GClientSmokeBase):
|
|
|
|
|
|
|
|
def setUp(self):
|
|
|
|
super(GClientSmokeGcs, self).setUp()
|
|
|
|
self.enabled = self.FAKE_REPOS.set_up_git()
|
|
|
|
if not self.enabled:
|
|
|
|
self.skipTest('git fake repos not available')
|
|
|
|
self.env['PATH'] = (os.path.join(ROOT_DIR, 'testing_support') +
|
|
|
|
os.pathsep + self.env['PATH'])
|
|
|
|
|
|
|
|
def testSyncGcs(self):
|
|
|
|
self.gclient(['config', self.git_base + 'repo_22', '--name', 'src'])
|
|
|
|
self.gclient(['sync'])
|
|
|
|
|
|
|
|
tree = self.mangle_git_tree(('repo_22@1', 'src'))
|
|
|
|
tree.update({
|
|
|
|
'src/another_gcs_dep/extracted_dir/extracted_file':
|
|
|
|
'extracted text',
|
|
|
|
'src/gcs_dep/extracted_dir/extracted_file':
|
|
|
|
'extracted text',
|
|
|
|
'src/gcs_dep_with_output_file/clang-format-no-extract':
|
|
|
|
'non-extractable file',
|
Reland "Reland "Add support for GCS deps""
This is a reland of commit 46d5382f69895a756a26a7326f76ac41dde36920
Whats fixed: Removed the call_google_storage refactor so
that all `import download_from_google_storage` callsites
(in depot_tools and chromium/src) won't be affected.
Instead, gclient.py will import Gsutil from download_from_google_storage.
Original change's description:
> Reland "Add support for GCS deps"
>
> This is a reland of commit 3eedee7b55fe20103a3913f48844765217c837c9
>
> Fixed more imports in download_google_storage.py
>
> Original change's description:
> > Add support for GCS deps
> >
> > Also take out GCS calling logic from download_google_storage and
> > into call_google_storage.
> >
> > GCS deps look like:
> > 'src/third_party/node/linux': {
> > 'dep_type': 'gcs',
> > 'condition': 'checkout_linux',
> > 'bucket': 'chromium-nodejs/20.11.0',
> > 'object_name': '46795170ff5df9831955f163f6966abde581c8af',
> > 'sha256sum': '887504c37404898ca41b896f448ee6d7fc24179d8fb6a4b79d028ab7e1b7153d',
> > },
> >
> > 'src/third_party/llvm-build/Release+Asserts': {
> > 'dep_type': 'gcs',
> > 'condition': 'checkout_linux',
> > 'bucket': 'chromium-browser-clang',
> > 'object_name': 'Linux_x64/clang-llvmorg-18-init-17730-gf670112a-2.tar.xz',
> > 'sha256sum': '1e46df9b4e63c074064d75646310cb76be2f19815997a8486987189d80f991e8',
> > },
> >
> > Example directory for src/third_party/node/linux after gclient sync:
> > - tar_file.gz is the downloaded file from GCS.
> > - node_linux_x64/ is extracted in its path.
> > - `hash` contains the sha of GCS filename.
> > ```
> > chromium/src/ ->
> > third_party/node/linux/ ->
> > hash, tar_file.gz, node_linux_x64/
> > ```
> >
> > Bug: b/324418194
> > Change-Id: Ibcbbff27e211f194ddb8a08494af56570a84a12b
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5299722
> > Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> > Reviewed-by: Joanna Wang <jojwang@chromium.org>
>
> Bug: b/324418194
> Change-Id: Ie64265a86abcec0135408715a45c32a8bb7c7408
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5346338
> Reviewed-by: Joanna Wang <jojwang@chromium.org>
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Bug: b/324418194
Change-Id: I8b58dadbaa740fd9da1fbaf29b3b6ff5ef67fd12
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5352896
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Joanna Wang <jojwang@chromium.org>
1 year ago
|
|
|
})
|
|
|
|
self.assertTree(tree)
|
|
|
|
|
|
|
|
def testRevInfo(self):
|
|
|
|
self.gclient(['config', self.git_base + 'repo_22', '--name', 'src'])
|
|
|
|
self.gclient(['sync'])
|
|
|
|
results = self.gclient(['revinfo'])
|
|
|
|
out = ('src: %(base)srepo_22\n'
|
Allow multiple objects per GCS dep
Allows multiple objects to use the same directory path.
This requires that each object has its own unique hash file and
unique migration file name. All object names must be unique.
Also update download_from_google_storage to check for the unique
migration file name.
Remove ConvertToGcs smoke tests since we're not converting
any git <-> gcs deps.
Example:
```
'src/third_party/llvm-build/Release+Asserts': {
'dep_type': 'gcs',
'condition': 'not llvm_force_head_revision',
'bucket': 'chromium-browser-clang',
'objects': [
{
'object_name': 'Linux_x64/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tar.xz',
'sha256sum': '7b33138d8592199f97d132242d7b3e10f460c5c9655d49a3ad3767218fba7a77',
'size_bytes': 50212876,
},
{
'object_name': 'Linux_x64/llvmobjdump-llvmorg-19-init-2941-ga0b3dbaf-22.tar.xz',
'sha256sum': '14d669650cd212eb0ccb8c34a9e655338dfdee66fe2ecdaa517f6bd607c09a97',
'size_bytes': 5302312,
},
]
},
```
TODO: update .gitignore to search for *_is_first_class and *_hash
Bug: b/324418194
Change-Id: I89d34b06ee24f4c1aa316cd51530ad078e823143
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5420793
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Joanna Wang <jojwang@chromium.org>
1 year ago
|
|
|
'src/another_gcs_dep:Linux/llvmfile.tar.gz: '
|
|
|
|
'gs://456bucket/Linux/llvmfile.tar.gz\n'
|
|
|
|
'src/gcs_dep:deadbeef: gs://123bucket/deadbeef\n'
|
|
|
|
'src/gcs_dep_with_output_file:clang-format-version123: '
|
|
|
|
'gs://789bucket/clang-format-version123\n' % {
|
Reland "Reland "Add support for GCS deps""
This is a reland of commit 46d5382f69895a756a26a7326f76ac41dde36920
Whats fixed: Removed the call_google_storage refactor so
that all `import download_from_google_storage` callsites
(in depot_tools and chromium/src) won't be affected.
Instead, gclient.py will import Gsutil from download_from_google_storage.
Original change's description:
> Reland "Add support for GCS deps"
>
> This is a reland of commit 3eedee7b55fe20103a3913f48844765217c837c9
>
> Fixed more imports in download_google_storage.py
>
> Original change's description:
> > Add support for GCS deps
> >
> > Also take out GCS calling logic from download_google_storage and
> > into call_google_storage.
> >
> > GCS deps look like:
> > 'src/third_party/node/linux': {
> > 'dep_type': 'gcs',
> > 'condition': 'checkout_linux',
> > 'bucket': 'chromium-nodejs/20.11.0',
> > 'object_name': '46795170ff5df9831955f163f6966abde581c8af',
> > 'sha256sum': '887504c37404898ca41b896f448ee6d7fc24179d8fb6a4b79d028ab7e1b7153d',
> > },
> >
> > 'src/third_party/llvm-build/Release+Asserts': {
> > 'dep_type': 'gcs',
> > 'condition': 'checkout_linux',
> > 'bucket': 'chromium-browser-clang',
> > 'object_name': 'Linux_x64/clang-llvmorg-18-init-17730-gf670112a-2.tar.xz',
> > 'sha256sum': '1e46df9b4e63c074064d75646310cb76be2f19815997a8486987189d80f991e8',
> > },
> >
> > Example directory for src/third_party/node/linux after gclient sync:
> > - tar_file.gz is the downloaded file from GCS.
> > - node_linux_x64/ is extracted in its path.
> > - `hash` contains the sha of GCS filename.
> > ```
> > chromium/src/ ->
> > third_party/node/linux/ ->
> > hash, tar_file.gz, node_linux_x64/
> > ```
> >
> > Bug: b/324418194
> > Change-Id: Ibcbbff27e211f194ddb8a08494af56570a84a12b
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5299722
> > Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> > Reviewed-by: Joanna Wang <jojwang@chromium.org>
>
> Bug: b/324418194
> Change-Id: Ie64265a86abcec0135408715a45c32a8bb7c7408
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5346338
> Reviewed-by: Joanna Wang <jojwang@chromium.org>
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Bug: b/324418194
Change-Id: I8b58dadbaa740fd9da1fbaf29b3b6ff5ef67fd12
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5352896
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Joanna Wang <jojwang@chromium.org>
1 year ago
|
|
|
'base': self.git_base,
|
|
|
|
})
|
|
|
|
self.check((out, '', 0), results)
|
|
|
|
|
|
|
|
def testRevInfoActual(self):
|
|
|
|
self.gclient(['config', self.git_base + 'repo_22', '--name', 'src'])
|
|
|
|
self.gclient(['sync'])
|
|
|
|
results = self.gclient(['revinfo', '--actual'])
|
Allow multiple objects per GCS dep
Allows multiple objects to use the same directory path.
This requires that each object has its own unique hash file and
unique migration file name. All object names must be unique.
Also update download_from_google_storage to check for the unique
migration file name.
Remove ConvertToGcs smoke tests since we're not converting
any git <-> gcs deps.
Example:
```
'src/third_party/llvm-build/Release+Asserts': {
'dep_type': 'gcs',
'condition': 'not llvm_force_head_revision',
'bucket': 'chromium-browser-clang',
'objects': [
{
'object_name': 'Linux_x64/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tar.xz',
'sha256sum': '7b33138d8592199f97d132242d7b3e10f460c5c9655d49a3ad3767218fba7a77',
'size_bytes': 50212876,
},
{
'object_name': 'Linux_x64/llvmobjdump-llvmorg-19-init-2941-ga0b3dbaf-22.tar.xz',
'sha256sum': '14d669650cd212eb0ccb8c34a9e655338dfdee66fe2ecdaa517f6bd607c09a97',
'size_bytes': 5302312,
},
]
},
```
TODO: update .gitignore to search for *_is_first_class and *_hash
Bug: b/324418194
Change-Id: I89d34b06ee24f4c1aa316cd51530ad078e823143
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5420793
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Joanna Wang <jojwang@chromium.org>
1 year ago
|
|
|
out = ('src: %(base)srepo_22@%(hash1)s\n'
|
|
|
|
'src/another_gcs_dep:Linux/llvmfile.tar.gz: '
|
|
|
|
'gs://456bucket/Linux/llvmfile.tar.gz\n'
|
|
|
|
'src/gcs_dep:deadbeef: gs://123bucket/deadbeef\n'
|
|
|
|
'src/gcs_dep_with_output_file:clang-format-version123: '
|
|
|
|
'gs://789bucket/clang-format-version123\n' % {
|
|
|
|
'base': self.git_base,
|
|
|
|
'hash1': self.githash('repo_22', 1),
|
|
|
|
})
|
Reland "Reland "Add support for GCS deps""
This is a reland of commit 46d5382f69895a756a26a7326f76ac41dde36920
Whats fixed: Removed the call_google_storage refactor so
that all `import download_from_google_storage` callsites
(in depot_tools and chromium/src) won't be affected.
Instead, gclient.py will import Gsutil from download_from_google_storage.
Original change's description:
> Reland "Add support for GCS deps"
>
> This is a reland of commit 3eedee7b55fe20103a3913f48844765217c837c9
>
> Fixed more imports in download_google_storage.py
>
> Original change's description:
> > Add support for GCS deps
> >
> > Also take out GCS calling logic from download_google_storage and
> > into call_google_storage.
> >
> > GCS deps look like:
> > 'src/third_party/node/linux': {
> > 'dep_type': 'gcs',
> > 'condition': 'checkout_linux',
> > 'bucket': 'chromium-nodejs/20.11.0',
> > 'object_name': '46795170ff5df9831955f163f6966abde581c8af',
> > 'sha256sum': '887504c37404898ca41b896f448ee6d7fc24179d8fb6a4b79d028ab7e1b7153d',
> > },
> >
> > 'src/third_party/llvm-build/Release+Asserts': {
> > 'dep_type': 'gcs',
> > 'condition': 'checkout_linux',
> > 'bucket': 'chromium-browser-clang',
> > 'object_name': 'Linux_x64/clang-llvmorg-18-init-17730-gf670112a-2.tar.xz',
> > 'sha256sum': '1e46df9b4e63c074064d75646310cb76be2f19815997a8486987189d80f991e8',
> > },
> >
> > Example directory for src/third_party/node/linux after gclient sync:
> > - tar_file.gz is the downloaded file from GCS.
> > - node_linux_x64/ is extracted in its path.
> > - `hash` contains the sha of GCS filename.
> > ```
> > chromium/src/ ->
> > third_party/node/linux/ ->
> > hash, tar_file.gz, node_linux_x64/
> > ```
> >
> > Bug: b/324418194
> > Change-Id: Ibcbbff27e211f194ddb8a08494af56570a84a12b
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5299722
> > Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> > Reviewed-by: Joanna Wang <jojwang@chromium.org>
>
> Bug: b/324418194
> Change-Id: Ie64265a86abcec0135408715a45c32a8bb7c7408
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5346338
> Reviewed-by: Joanna Wang <jojwang@chromium.org>
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Bug: b/324418194
Change-Id: I8b58dadbaa740fd9da1fbaf29b3b6ff5ef67fd12
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5352896
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Joanna Wang <jojwang@chromium.org>
1 year ago
|
|
|
self.check((out, '', 0), results)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
if '-v' in sys.argv:
|
|
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
|
|
unittest.main()
|