From c515a8bc469453091d7d55c906496d78761fb696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Hajdan=2C=20Jr?= Date: Wed, 31 May 2017 18:32:50 +0200 Subject: [PATCH] bot_update: expose --disable-syntax-validation to recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some recipes interacting with older revisions (e.g. bisect) will need this. Bug: 570091 Change-Id: I38e5ffa2db1a9bfae3667f015f00977c32ebe51e Reviewed-on: https://chromium-review.googlesource.com/519407 Commit-Queue: Paweł Hajdan Jr. Reviewed-by: Robbie Iannucci --- recipes/recipe_modules/bot_update/api.py | 8 +++++++- .../bot_update/example.expected/basic.json | 3 ++- .../basic_with_branch_heads.json | 3 ++- .../bot_update/example.expected/buildbot.json | 3 ++- .../bot_update/example.expected/clobber.json | 3 ++- .../deprecated_got_revision_mapping.json | 3 ++- .../gerrit_no_rebase_patch_ref.json | 3 ++- .../example.expected/gerrit_no_reset.json | 3 ++- .../bot_update/example.expected/no_shallow.json | 3 ++- .../reset_root_solution_revision.json | 3 ++- .../bot_update/example.expected/trychange.json | 3 ++- .../trychange_oauth2_buildbot.json | 3 ++- .../example.expected/trychange_oauth2_json.json | 3 ++- .../trychange_oauth2_json_win.json | 3 ++- .../bot_update/example.expected/tryjob.json | 3 ++- .../example.expected/tryjob_fail.json | 3 ++- .../example.expected/tryjob_fail_patch.json | 3 ++- .../tryjob_fail_patch_download.json | 3 ++- .../example.expected/tryjob_gerrit_angle.json | 3 ++- .../tryjob_gerrit_angle_deprecated.json | 3 ++- .../bot_update/example.expected/tryjob_v8.json | 3 ++- .../tryjob_v8_head_by_default.json | 3 ++- .../bot_update/example.expected/with_tags.json | 3 ++- recipes/recipe_modules/bot_update/example.py | 3 ++- .../bot_update/resources/bot_update.py | 17 ++++++++++++----- tests/bot_update_coverage_test.py | 1 + 26 files changed, 66 insertions(+), 29 deletions(-) diff --git a/recipes/recipe_modules/bot_update/api.py b/recipes/recipe_modules/bot_update/api.py index dbc59855f..c9e3ca06b 100644 --- a/recipes/recipe_modules/bot_update/api.py +++ b/recipes/recipe_modules/bot_update/api.py @@ -79,7 +79,8 @@ class BotUpdateApi(recipe_api.RecipeApi): use_site_config_creds=True, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, - gerrit_no_rebase_patch_ref=False, **kwargs): + gerrit_no_rebase_patch_ref=False, + disable_syntax_validation=False, **kwargs): """ Args: use_site_config_creds: If the oauth2 credentials are in the buildbot @@ -92,6 +93,9 @@ class BotUpdateApi(recipe_api.RecipeApi): the 'issue' property. patchset: The rietveld issue patchset to use. If omitted, will infer from the 'patchset' property. + disable_syntax_validation: (legacy) Disables syntax validation for DEPS. + Needed as migration paths for recipes dealing with older revisions, + such as bisect. """ refs = refs or [] # We can re-use the gclient spec from the gclient module, since all the @@ -239,6 +243,8 @@ class BotUpdateApi(recipe_api.RecipeApi): cmd.append('--gerrit_no_reset') if gerrit_no_rebase_patch_ref: cmd.append('--gerrit_no_rebase_patch_ref') + if disable_syntax_validation: + cmd.append('--disable-syntax-validation') # Inject Json output for testing. first_sln = cfg.solutions[0].name diff --git a/recipes/recipe_modules/bot_update/example.expected/basic.json b/recipes/recipe_modules/bot_update/example.expected/basic.json index fa98a47f8..ea978b771 100644 --- a/recipes/recipe_modules/bot_update/example.expected/basic.json +++ b/recipes/recipe_modules/bot_update/example.expected/basic.json @@ -15,7 +15,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@abc" + "src@abc", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json b/recipes/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json index c89990849..089f564f7 100644 --- a/recipes/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json +++ b/recipes/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json @@ -16,7 +16,8 @@ "/path/to/tmp/json", "--revision", "src@HEAD", - "--with_branch_heads" + "--with_branch_heads", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/buildbot.json b/recipes/recipe_modules/bot_update/example.expected/buildbot.json index fa98a47f8..ea978b771 100644 --- a/recipes/recipe_modules/bot_update/example.expected/buildbot.json +++ b/recipes/recipe_modules/bot_update/example.expected/buildbot.json @@ -15,7 +15,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@abc" + "src@abc", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/clobber.json b/recipes/recipe_modules/bot_update/example.expected/clobber.json index 265871047..7b3f85832 100644 --- a/recipes/recipe_modules/bot_update/example.expected/clobber.json +++ b/recipes/recipe_modules/bot_update/example.expected/clobber.json @@ -16,7 +16,8 @@ "/path/to/tmp/json", "--revision", "src@HEAD", - "--clobber" + "--clobber", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/deprecated_got_revision_mapping.json b/recipes/recipe_modules/bot_update/example.expected/deprecated_got_revision_mapping.json index e6fe19013..9b7173eed 100644 --- a/recipes/recipe_modules/bot_update/example.expected/deprecated_got_revision_mapping.json +++ b/recipes/recipe_modules/bot_update/example.expected/deprecated_got_revision_mapping.json @@ -21,7 +21,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD" + "src@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/gerrit_no_rebase_patch_ref.json b/recipes/recipe_modules/bot_update/example.expected/gerrit_no_rebase_patch_ref.json index 52c84aef6..bc22fdb6f 100644 --- a/recipes/recipe_modules/bot_update/example.expected/gerrit_no_rebase_patch_ref.json +++ b/recipes/recipe_modules/bot_update/example.expected/gerrit_no_rebase_patch_ref.json @@ -16,7 +16,8 @@ "/path/to/tmp/json", "--revision", "src@HEAD", - "--gerrit_no_rebase_patch_ref" + "--gerrit_no_rebase_patch_ref", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/gerrit_no_reset.json b/recipes/recipe_modules/bot_update/example.expected/gerrit_no_reset.json index 0d8bb4af5..e2e316672 100644 --- a/recipes/recipe_modules/bot_update/example.expected/gerrit_no_reset.json +++ b/recipes/recipe_modules/bot_update/example.expected/gerrit_no_reset.json @@ -16,7 +16,8 @@ "/path/to/tmp/json", "--revision", "src@HEAD", - "--gerrit_no_reset" + "--gerrit_no_reset", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/no_shallow.json b/recipes/recipe_modules/bot_update/example.expected/no_shallow.json index a50b0643b..b9e336ed9 100644 --- a/recipes/recipe_modules/bot_update/example.expected/no_shallow.json +++ b/recipes/recipe_modules/bot_update/example.expected/no_shallow.json @@ -16,7 +16,8 @@ "/path/to/tmp/json", "--revision", "src@HEAD", - "--no_shallow" + "--no_shallow", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json b/recipes/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json index 7d12d852e..029e16d50 100644 --- a/recipes/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json +++ b/recipes/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json @@ -15,7 +15,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@revision" + "src@revision", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/trychange.json b/recipes/recipe_modules/bot_update/example.expected/trychange.json index d7dced202..3d5bd5d96 100644 --- a/recipes/recipe_modules/bot_update/example.expected/trychange.json +++ b/recipes/recipe_modules/bot_update/example.expected/trychange.json @@ -17,7 +17,8 @@ "--revision", "src@HEAD", "--refs", - "+refs/change/1/2/333" + "+refs/change/1/2/333", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_buildbot.json b/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_buildbot.json index d508c0996..93d78a083 100644 --- a/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_buildbot.json +++ b/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_buildbot.json @@ -19,7 +19,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD" + "src@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json.json b/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json.json index 2b97028ab..006dff1c4 100644 --- a/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json.json +++ b/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json.json @@ -17,7 +17,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD" + "src@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json_win.json b/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json_win.json index 717ea6b10..cbb92821e 100644 --- a/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json_win.json +++ b/recipes/recipe_modules/bot_update/example.expected/trychange_oauth2_json_win.json @@ -17,7 +17,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD" + "src@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/tryjob.json b/recipes/recipe_modules/bot_update/example.expected/tryjob.json index e383a8e8c..3bdba86e2 100644 --- a/recipes/recipe_modules/bot_update/example.expected/tryjob.json +++ b/recipes/recipe_modules/bot_update/example.expected/tryjob.json @@ -21,7 +21,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD" + "src@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/tryjob_fail.json b/recipes/recipe_modules/bot_update/example.expected/tryjob_fail.json index ac1788df2..024eaef0d 100644 --- a/recipes/recipe_modules/bot_update/example.expected/tryjob_fail.json +++ b/recipes/recipe_modules/bot_update/example.expected/tryjob_fail.json @@ -21,7 +21,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD" + "src@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json b/recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json index 5f32e2bda..fc0542dcc 100644 --- a/recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json +++ b/recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json @@ -21,7 +21,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD" + "src@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json b/recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json index 3af494f51..4cc110b61 100644 --- a/recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json +++ b/recipes/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json @@ -21,7 +21,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD" + "src@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle.json b/recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle.json index 4fc9b1fa6..4c5b8b9f3 100644 --- a/recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle.json +++ b/recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle.json @@ -21,7 +21,8 @@ "--revision", "src@HEAD", "--revision", - "src/third_party/angle@HEAD" + "src/third_party/angle@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle_deprecated.json b/recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle_deprecated.json index 4fc9b1fa6..4c5b8b9f3 100644 --- a/recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle_deprecated.json +++ b/recipes/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle_deprecated.json @@ -21,7 +21,8 @@ "--revision", "src@HEAD", "--revision", - "src/third_party/angle@HEAD" + "src/third_party/angle@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/tryjob_v8.json b/recipes/recipe_modules/bot_update/example.expected/tryjob_v8.json index bf4c1e625..4efd25398 100644 --- a/recipes/recipe_modules/bot_update/example.expected/tryjob_v8.json +++ b/recipes/recipe_modules/bot_update/example.expected/tryjob_v8.json @@ -23,7 +23,8 @@ "--revision", "src@HEAD", "--revision", - "src/v8@abc" + "src/v8@abc", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/tryjob_v8_head_by_default.json b/recipes/recipe_modules/bot_update/example.expected/tryjob_v8_head_by_default.json index ef6015c34..6e9bf8109 100644 --- a/recipes/recipe_modules/bot_update/example.expected/tryjob_v8_head_by_default.json +++ b/recipes/recipe_modules/bot_update/example.expected/tryjob_v8_head_by_default.json @@ -23,7 +23,8 @@ "--revision", "src@HEAD", "--revision", - "src/v8@HEAD" + "src/v8@HEAD", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.expected/with_tags.json b/recipes/recipe_modules/bot_update/example.expected/with_tags.json index a71b7b6bc..49de00e18 100644 --- a/recipes/recipe_modules/bot_update/example.expected/with_tags.json +++ b/recipes/recipe_modules/bot_update/example.expected/with_tags.json @@ -16,7 +16,8 @@ "/path/to/tmp/json", "--revision", "src@HEAD", - "--with_tags" + "--with_tags", + "--disable-syntax-validation" ], "env": { "GIT_HTTP_LOW_SPEED_LIMIT": "1000", diff --git a/recipes/recipe_modules/bot_update/example.py b/recipes/recipe_modules/bot_update/example.py index c0cd3b2f9..3c4817049 100644 --- a/recipes/recipe_modules/bot_update/example.py +++ b/recipes/recipe_modules/bot_update/example.py @@ -59,7 +59,8 @@ def RunSteps(api): root_solution_revision=root_solution_revision, suffix=suffix, gerrit_no_reset=gerrit_no_reset, - gerrit_no_rebase_patch_ref=gerrit_no_rebase_patch_ref) + gerrit_no_rebase_patch_ref=gerrit_no_rebase_patch_ref, + disable_syntax_validation=True) if patch: api.bot_update.deapply_patch(bot_update_step) diff --git a/recipes/recipe_modules/bot_update/resources/bot_update.py b/recipes/recipe_modules/bot_update/resources/bot_update.py index d908ee8b7..e8e378877 100755 --- a/recipes/recipe_modules/bot_update/resources/bot_update.py +++ b/recipes/recipe_modules/bot_update/resources/bot_update.py @@ -328,7 +328,8 @@ def gclient_configure(solutions, target_os, target_os_only, git_cache_dir): def gclient_sync( - with_branch_heads, with_tags, shallow, revisions, break_repo_locks): + with_branch_heads, with_tags, shallow, revisions, break_repo_locks, + disable_syntax_validation): # We just need to allocate a filename. fd, gclient_output_file = tempfile.mkstemp(suffix='.json') os.close(fd) @@ -344,6 +345,8 @@ def gclient_sync( args += ['--shallow'] if break_repo_locks: args += ['--break_repo_locks'] + if disable_syntax_validation: + args += ['--disable-syntax-validation'] for name, revision in sorted(revisions.iteritems()): if revision.upper() == 'HEAD': revision = 'origin/master' @@ -747,7 +750,7 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, gerrit_ref, gerrit_rebase_patch_ref, revision_mapping, apply_issue_email_file, apply_issue_key_file, apply_issue_oauth2_file, shallow, refs, git_cache_dir, - gerrit_reset): + gerrit_reset, disable_syntax_validation): # Get a checkout of each solution, without DEPS or hooks. # Calling git directly because there is no way to run Gclient without # invoking DEPS. @@ -802,7 +805,8 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, TAGS_REFSPEC in refs, shallow, gc_revisions, - break_repo_locks) + break_repo_locks, + disable_syntax_validation) # Now that gclient_sync has finished, we should revert any .DEPS.git so that # presubmit doesn't complain about it being modified. @@ -928,7 +932,9 @@ def parse_args(): help='Always pass --with_tags to gclient. This ' 'does the same thing as --refs +refs/tags/*') parse.add_option('--git-cache-dir', help='Path to git cache directory.') - + parse.add_option( + '--disable-syntax-validation', action='store_true', + help='Disable validation of .gclient and DEPS syntax.') options, args = parse.parse_args() @@ -1037,7 +1043,8 @@ def checkout(options, git_slns, specs, revisions, step_text, shallow): shallow=shallow, refs=options.refs, git_cache_dir=options.git_cache_dir, - gerrit_reset=not options.gerrit_no_reset) + gerrit_reset=not options.gerrit_no_reset, + disable_syntax_validation=options.disable_syntax_validation) gclient_output = ensure_checkout(**checkout_parameters) except GclientSyncFailed: print 'We failed gclient sync, lets delete the checkout and retry.' diff --git a/tests/bot_update_coverage_test.py b/tests/bot_update_coverage_test.py index 8418e1c3b..d53a81598 100755 --- a/tests/bot_update_coverage_test.py +++ b/tests/bot_update_coverage_test.py @@ -41,6 +41,7 @@ DEFAULT_PARAMS = { 'refs': [], 'git_cache_dir': '', 'gerrit_reset': None, + 'disable_syntax_validation': False, }