diff --git a/recipe_modules/bot_update/api.py b/recipe_modules/bot_update/api.py index 610872e5f..4b870a4b7 100644 --- a/recipe_modules/bot_update/api.py +++ b/recipe_modules/bot_update/api.py @@ -8,6 +8,12 @@ from recipe_engine import recipe_api +# This is just for testing, to indicate if a master is using a Git scheduler +# or not. +SVN_MASTERS = ( + 'experimental.svn', +) + def jsonish_to_python(spec, is_top=False): """Turn a json spec into a python parsable object. @@ -61,10 +67,9 @@ class BotUpdateApi(recipe_api.RecipeApi): def properties(self): return self._properties - # Note: force is ignored. def ensure_checkout(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, - force=True, patch_root=None, no_shallow=False, + force=False, patch_root=None, no_shallow=False, with_branch_heads=False, refs=None, patch_project_roots=None, patch_oauth2=False, output_manifest=True, clobber=False, @@ -75,6 +80,11 @@ class BotUpdateApi(recipe_api.RecipeApi): cfg = gclient_config or self.m.gclient.c spec_string = jsonish_to_python(cfg.as_jsonish(), True) + # Used by bot_update to determine if we want to run or not. + master = self.m.properties['mastername'] + builder = self.m.properties['buildername'] + slave = self.m.properties['slavename'] + # Construct our bot_update command. This basically be inclusive of # everything required for bot_update to know: root = patch_root @@ -126,13 +136,18 @@ class BotUpdateApi(recipe_api.RecipeApi): rev_map = self.m.gclient.c.got_revision_mapping.as_jsonish() flags = [ - # 1. What do we want to check out (spec/root/rev/rev_map). + # 1. Do we want to run? (master/builder/slave). + ['--master', master], + ['--builder', builder], + ['--slave', slave], + + # 2. What do we want to check out (spec/root/rev/rev_map). ['--spec', spec_string], ['--root', root], ['--revision_mapping_file', self.m.json.input(rev_map)], ['--git-cache-dir', self.m.path['git_cache']], - # 2. How to find the patch, if any (issue/patchset/patch_url). + # 3. How to find the patch, if any (issue/patchset/patch_url). ['--issue', issue], ['--patchset', patchset], ['--patch_url', patch_url], @@ -142,7 +157,7 @@ class BotUpdateApi(recipe_api.RecipeApi): ['--apply_issue_email_file', email_file], ['--apply_issue_key_file', key_file], - # 3. Hookups to JSON output back into recipes. + # 4. Hookups to JSON output back into recipes. ['--output_json', self.m.json.output()],] @@ -182,6 +197,8 @@ class BotUpdateApi(recipe_api.RecipeApi): if clobber: cmd.append('--clobber') + if force: + cmd.append('--force') if no_shallow: cmd.append('--no_shallow') if output_manifest: @@ -190,9 +207,11 @@ class BotUpdateApi(recipe_api.RecipeApi): cmd.append('--with_branch_heads') # Inject Json output for testing. + git_mode = self.m.properties.get('mastername') not in SVN_MASTERS first_sln = cfg.solutions[0].name step_test_data = lambda: self.test_api.output_json( - root, first_sln, rev_map, self.m.properties.get('fail_patch', False), + master, builder, slave, root, first_sln, rev_map, git_mode, force, + self.m.properties.get('fail_patch', False), output_manifest=output_manifest, fixed_revisions=fixed_revisions) # Add suffixes to the step name, if specified. diff --git a/recipe_modules/bot_update/example.expected/basic.json b/recipe_modules/bot_update/example.expected/basic.json index 0ad0c943f..0cff6c0fe 100644 --- a/recipe_modules/bot_update/example.expected/basic.json +++ b/recipe_modules/bot_update/example.expected/basic.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "chromium.linux", + "--builder", + "Linux Builder", + "--slave", + "totallyaslave-m1", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/basic_output_manifest.json b/recipe_modules/bot_update/example.expected/basic_output_manifest.json index 6dbca9a27..170b4ab6d 100644 --- a/recipe_modules/bot_update/example.expected/basic_output_manifest.json +++ b/recipe_modules/bot_update/example.expected/basic_output_manifest.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "chromium.linux", + "--builder", + "Linux Builder", + "--slave", + "totallyaslave-m1", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json b/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json index 67d212db9..214604319 100644 --- a/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json +++ b/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "chromium.linux", + "--builder", + "Linux Builder", + "--slave", + "totallyaslave-m1", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/clobber.json b/recipe_modules/bot_update/example.expected/clobber.json index 2adc26fdd..ee6f935c9 100644 --- a/recipe_modules/bot_update/example.expected/clobber.json +++ b/recipe_modules/bot_update/example.expected/clobber.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "experimental", + "--builder", + "Experimental Builder", + "--slave", + "somehost", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", @@ -24,24 +30,11 @@ }, "name": "bot_update", "~followup_annotations": [ - "@@@STEP_TEXT@Some step text@@@", "@@@STEP_LOG_LINE@json.output@{@@@", - "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", - "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", - "@@@STEP_LOG_LINE@json.output@ \"src\": \"HEAD\"@@@", - "@@@STEP_LOG_LINE@json.output@ }, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", - "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\"@@@", - "@@@STEP_LOG_LINE@json.output@ }, @@@", - "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@", "@@@STEP_LOG_LINE@json.output@}@@@", - "@@@STEP_LOG_END@json.output@@@", - "@@@SET_BUILD_PROPERTY@got_cr_revision@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", - "@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@" + "@@@STEP_LOG_END@json.output@@@" ] }, { diff --git a/recipe_modules/bot_update/example.expected/forced.json b/recipe_modules/bot_update/example.expected/forced.json index a043d5a12..07b1af682 100644 --- a/recipe_modules/bot_update/example.expected/forced.json +++ b/recipe_modules/bot_update/example.expected/forced.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "experimental", + "--builder", + "Experimental Builder", + "--slave", + "somehost", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", @@ -15,7 +21,8 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD" + "src@HEAD", + "--force" ], "cwd": "[SLAVE_BUILD]", "env": { diff --git a/recipe_modules/bot_update/example.expected/no_shallow.json b/recipe_modules/bot_update/example.expected/no_shallow.json index 18abe3181..2008a4fa3 100644 --- a/recipe_modules/bot_update/example.expected/no_shallow.json +++ b/recipe_modules/bot_update/example.expected/no_shallow.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "experimental", + "--builder", + "Experimental Builder", + "--slave", + "somehost", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", @@ -24,24 +30,11 @@ }, "name": "bot_update", "~followup_annotations": [ - "@@@STEP_TEXT@Some step text@@@", "@@@STEP_LOG_LINE@json.output@{@@@", - "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", - "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", - "@@@STEP_LOG_LINE@json.output@ \"src\": \"HEAD\"@@@", - "@@@STEP_LOG_LINE@json.output@ }, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", - "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\"@@@", - "@@@STEP_LOG_LINE@json.output@ }, @@@", - "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@", "@@@STEP_LOG_LINE@json.output@}@@@", - "@@@STEP_LOG_END@json.output@@@", - "@@@SET_BUILD_PROPERTY@got_cr_revision@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", - "@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@" + "@@@STEP_LOG_END@json.output@@@" ] }, { diff --git a/recipe_modules/bot_update/example.expected/off.json b/recipe_modules/bot_update/example.expected/off.json new file mode 100644 index 000000000..48109b0ba --- /dev/null +++ b/recipe_modules/bot_update/example.expected/off.json @@ -0,0 +1,44 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "experimental", + "--builder", + "Experimental Builder", + "--slave", + "somehost", + "--spec", + "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", + "--root", + "src", + "--revision_mapping_file", + "{\"src\": \"got_cr_revision\"}", + "--git-cache-dir", + "[GIT_CACHE]", + "--output_json", + "/path/to/tmp/json", + "--revision", + "src@HEAD" + ], + "cwd": "[SLAVE_BUILD]", + "env": { + "PATH": "%(PATH)s:RECIPE_PACKAGE[depot_tools]" + }, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json b/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json index 69370788f..e50485c41 100644 --- a/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json +++ b/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "experimental", + "--builder", + "Experimental Builder", + "--slave", + "somehost", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", @@ -23,24 +29,11 @@ }, "name": "bot_update", "~followup_annotations": [ - "@@@STEP_TEXT@Some step text@@@", "@@@STEP_LOG_LINE@json.output@{@@@", - "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", - "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", - "@@@STEP_LOG_LINE@json.output@ \"src\": \"revision\"@@@", - "@@@STEP_LOG_LINE@json.output@ }, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", - "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\"@@@", - "@@@STEP_LOG_LINE@json.output@ }, @@@", - "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@", "@@@STEP_LOG_LINE@json.output@}@@@", - "@@@STEP_LOG_END@json.output@@@", - "@@@SET_BUILD_PROPERTY@got_cr_revision@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", - "@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@" + "@@@STEP_LOG_END@json.output@@@" ] }, { diff --git a/recipe_modules/bot_update/example.expected/svn_mode.json b/recipe_modules/bot_update/example.expected/svn_mode.json new file mode 100644 index 000000000..b12f9e40c --- /dev/null +++ b/recipe_modules/bot_update/example.expected/svn_mode.json @@ -0,0 +1,60 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "experimental.svn", + "--builder", + "Experimental SVN Builder", + "--slave", + "somehost", + "--spec", + "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", + "--root", + "src", + "--revision_mapping_file", + "{\"src\": \"got_cr_revision\"}", + "--git-cache-dir", + "[GIT_CACHE]", + "--output_json", + "/path/to/tmp/json", + "--revision", + "src@HEAD", + "--force" + ], + "cwd": "[SLAVE_BUILD]", + "env": { + "PATH": "%(PATH)s:RECIPE_PACKAGE[depot_tools]" + }, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src\": \"HEAD\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": 170242, @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_git\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_cr_revision@170242@@@", + "@@@SET_BUILD_PROPERTY@got_cr_revision_git@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", + "@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipe_modules/bot_update/example.expected/trychange.json b/recipe_modules/bot_update/example.expected/trychange.json index a4c0f646c..a7aa47328 100644 --- a/recipe_modules/bot_update/example.expected/trychange.json +++ b/recipe_modules/bot_update/example.expected/trychange.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "tryserver.chromium.linux", + "--builder", + "linux_rel", + "--slave", + "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/trychange_oauth2.json b/recipe_modules/bot_update/example.expected/trychange_oauth2.json index f364c3bc8..c81ede5ff 100644 --- a/recipe_modules/bot_update/example.expected/trychange_oauth2.json +++ b/recipe_modules/bot_update/example.expected/trychange_oauth2.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "tryserver.chromium.linux", + "--builder", + "linux_rel", + "--slave", + "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob.json b/recipe_modules/bot_update/example.expected/tryjob.json index 7d2f32a58..f34e0fe7e 100644 --- a/recipe_modules/bot_update/example.expected/tryjob.json +++ b/recipe_modules/bot_update/example.expected/tryjob.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "tryserver.chromium.linux", + "--builder", + "linux_rel", + "--slave", + "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob_fail.json b/recipe_modules/bot_update/example.expected/tryjob_fail.json index b8979be0b..66b377c29 100644 --- a/recipe_modules/bot_update/example.expected/tryjob_fail.json +++ b/recipe_modules/bot_update/example.expected/tryjob_fail.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "tryserver.chromium.linux", + "--builder", + "linux_rel", + "--slave", + "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json b/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json index b1163786b..9e2cd6f8d 100644 --- a/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json +++ b/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "tryserver.chromium.linux", + "--builder", + "linux_rel", + "--slave", + "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json b/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json index 6c38ce865..e7a93bdf1 100644 --- a/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json +++ b/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "tryserver.chromium.linux", + "--builder", + "linux_rel", + "--slave", + "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob_v8.json b/recipe_modules/bot_update/example.expected/tryjob_v8.json index 6d4d75afb..f426d8b95 100644 --- a/recipe_modules/bot_update/example.expected/tryjob_v8.json +++ b/recipe_modules/bot_update/example.expected/tryjob_v8.json @@ -4,6 +4,12 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--master", + "tryserver.chromium.linux", + "--builder", + "linux_rel", + "--slave", + "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.py b/recipe_modules/bot_update/example.py index 3e512e774..d866c0ceb 100644 --- a/recipe_modules/bot_update/example.py +++ b/recipe_modules/bot_update/example.py @@ -43,57 +43,110 @@ def RunSteps(api): def GenTests(api): yield api.test('basic') + api.properties( + mastername='chromium.linux', + buildername='Linux Builder', + slavename='totallyaslave-m1', patch=False, revision='abc' ) yield api.test('basic_with_branch_heads') + api.properties( + mastername='chromium.linux', + buildername='Linux Builder', + slavename='totallyaslave-m1', with_branch_heads=True, suffix='with branch heads' ) yield api.test('basic_output_manifest') + api.properties( + mastername='chromium.linux', + buildername='Linux Builder', + slavename='totallyaslave-m1', output_manifest=True, ) yield api.test('tryjob') + api.properties( + mastername='tryserver.chromium.linux', + buildername='linux_rel', + slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar' ) yield api.test('trychange') + api.properties( + mastername='tryserver.chromium.linux', + buildername='linux_rel', + slavename='totallyaslave-c4', refs=['+refs/change/1/2/333'], ) yield api.test('trychange_oauth2') + api.properties( + mastername='tryserver.chromium.linux', + buildername='linux_rel', + slavename='totallyaslave-c4', oauth2=True, ) yield api.test('tryjob_fail') + api.properties( + mastername='tryserver.chromium.linux', + buildername='linux_rel', + slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar', ) + api.step_data('bot_update', retcode=1) yield api.test('tryjob_fail_patch') + api.properties( + mastername='tryserver.chromium.linux', + buildername='linux_rel', + slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar', fail_patch='apply', ) + api.step_data('bot_update', retcode=88) yield api.test('tryjob_fail_patch_download') + api.properties( + mastername='tryserver.chromium.linux', + buildername='linux_rel', + slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar', fail_patch='download' ) + api.step_data('bot_update', retcode=87) yield api.test('forced') + api.properties( + mastername='experimental', + buildername='Experimental Builder', + slavename='somehost', force=1 ) yield api.test('no_shallow') + api.properties( + mastername='experimental', + buildername='Experimental Builder', + slavename='somehost', no_shallow=1 ) + yield api.test('off') + api.properties( + mastername='experimental', + buildername='Experimental Builder', + slavename='somehost', + ) + yield api.test('svn_mode') + api.properties( + mastername='experimental.svn', + buildername='Experimental SVN Builder', + slavename='somehost', + force=1 + ) yield api.test('clobber') + api.properties( + mastername='experimental', + buildername='Experimental Builder', + slavename='somehost', clobber=1 ) yield api.test('reset_root_solution_revision') + api.properties( + mastername='experimental', + buildername='Experimental Builder', + slavename='somehost', root_solution_revision='revision', ) yield api.test('tryjob_v8') + api.properties( + mastername='tryserver.chromium.linux', + buildername='linux_rel', + slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar', diff --git a/recipe_modules/bot_update/resources/bot_update.py b/recipe_modules/bot_update/resources/bot_update.py index 4ecbf6597..39c4bf61d 100755 --- a/recipe_modules/bot_update/resources/bot_update.py +++ b/recipe_modules/bot_update/resources/bot_update.py @@ -73,18 +73,19 @@ ROOT_DIR = path.dirname(BUILD_DIR) DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) +BUILD_INTERNAL_DIR = check_dir( + 'build_internal', [ + path.join(ROOT_DIR, 'build_internal'), + path.join(ROOT_DIR, # .recipe_deps + path.pardir, # slave + path.pardir, # scripts + path.pardir), # build_internal + ]) + + CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' -RECOGNIZED_PATHS = { - '/chrome/trunk/src': - CHROMIUM_SRC_URL, - '/chrome/trunk/src/tools/cros.DEPS': - CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git', - '/chrome-internal/trunk/src-internal': - 'https://chrome-internal.googlesource.com/chrome/src-internal.git', -} - # Official builds use buildspecs, so this is a special case. BUILDSPEC_TYPE = collections.namedtuple('buildspec', ('container', 'version')) @@ -167,8 +168,39 @@ GOT_REVISION_MAPPINGS = { BOT_UPDATE_MESSAGE = """ -Bot Update Debugging information: +What is the "Bot Update" step? +============================== + +This step ensures that the source checkout on the bot (e.g. Chromium's src/ and +its dependencies) is checked out in a consistent state. This means that all of +the necessary repositories are checked out, no extra repositories are checked +out, and no locally modified files are present. + +These actions used to be taken care of by the "gclient revert" and "update" +steps. However, those steps are known to be buggy and occasionally flaky. This +step has two main advantages over them: + * it only operates in Git, so the logic can be clearer and cleaner; and + * it is a slave-side script, so its behavior can be modified without + restarting the master. + +Why Git, you ask? Because that is the direction that the Chromium project is +heading. This step is an integral part of the transition from using the SVN repo +at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while +we fully convert everything to Git. This message will get out of your way +eventually, and the waterfall will be a happier place because of it. + +This step can be activated or deactivated independently on every builder on +every master. When it is active, the "gclient revert" and "update" steps become +no-ops. When it is inactive, it prints this message, cleans up after itself, and +lets everything else continue as though nothing has changed. Eventually, when +everything is stable enough, this step will replace them entirely. + +Debugging information: (master/builder/slave may be unspecified on recipes) +master: %(master)s +builder: %(builder)s +slave: %(slave)s +forced by recipes: %(recipe)s CURRENT_DIR: %(CURRENT_DIR)s BUILDER_DIR: %(BUILDER_DIR)s SLAVE_DIR: %(SLAVE_DIR)s @@ -176,7 +208,20 @@ THIS_DIR: %(THIS_DIR)s SCRIPTS_DIR: %(SCRIPTS_DIR)s BUILD_DIR: %(BUILD_DIR)s ROOT_DIR: %(ROOT_DIR)s -DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s""" +DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s +bot_update.py is:""" + +ACTIVATED_MESSAGE = """ACTIVE. +The bot will perform a Git checkout in this step. +The "gclient revert" and "update" steps are no-ops. + +""" + +NOT_ACTIVATED_MESSAGE = """INACTIVE. +This step does nothing. You actually want to look at the "update" step. + +""" + GCLIENT_TEMPLATE = """solutions = %(solutions)s @@ -186,14 +231,137 @@ cache_dir = r%(cache_dir)s """ +internal_data = {} +if BUILD_INTERNAL_DIR: + local_vars = {} + try: + execfile(os.path.join( + BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'), + local_vars) + except Exception: + # Same as if BUILD_INTERNAL_DIR didn't exist in the first place. + print 'Warning: unable to read internal configuration file.' + print 'If this is an internal bot, this step may be erroneously inactive.' + internal_data = local_vars + +RECOGNIZED_PATHS = { + # If SVN path matches key, the entire URL is rewritten to the Git url. + '/chrome/trunk/src': + CHROMIUM_SRC_URL, + '/chrome/trunk/src/tools/cros.DEPS': + CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git', +} +RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {})) + +ENABLED_MASTERS = [ + 'bot_update.always_on', + 'chromium.android', + 'chromium.angle', + 'chromium.chrome', + 'chromium.chromedriver', + 'chromium.chromiumos', + 'chromium', + 'chromium.fyi', + 'chromium.goma', + 'chromium.gpu', + 'chromium.gpu.fyi', + 'chromium.infra', + 'chromium.infra.cron', + 'chromium.linux', + 'chromium.lkgr', + 'chromium.mac', + 'chromium.memory', + 'chromium.memory.fyi', + 'chromium.perf', + 'chromium.perf.fyi', + 'chromium.swarm', + 'chromium.webkit', + 'chromium.webrtc', + 'chromium.webrtc.fyi', + 'chromium.win', + 'client.catapult', + 'client.drmemory', + 'client.mojo', + 'client.nacl', + 'client.nacl.ports', + 'client.nacl.sdk', + 'client.nacl.toolchain', + 'client.pdfium', + 'client.skia', + 'client.skia.fyi', + 'client.v8', + 'client.v8.branches', + 'client.v8.fyi', + 'client.webrtc', + 'client.webrtc.fyi', + 'tryserver.blink', + 'tryserver.client.catapult', + 'tryserver.client.mojo', + 'tryserver.chromium.android', + 'tryserver.chromium.angle', + 'tryserver.chromium.linux', + 'tryserver.chromium.mac', + 'tryserver.chromium.perf', + 'tryserver.chromium.win', + 'tryserver.infra', + 'tryserver.nacl', + 'tryserver.v8', + 'tryserver.webrtc', +] +ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', []) + +ENABLED_BUILDERS = { + 'client.dart.fyi': [ + 'v8-linux-release', + 'v8-mac-release', + 'v8-win-release', + ], + 'client.dynamorio': [ + 'linux-v8-dr', + ], +} +ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {})) + +ENABLED_SLAVES = {} +ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {})) + +# Disabled filters get run AFTER enabled filters, so for example if a builder +# config is enabled, but a bot on that builder is disabled, that bot will +# be disabled. +DISABLED_BUILDERS = {} +DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {})) + +DISABLED_SLAVES = {} +DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {})) + +# These masters work only in Git, meaning for got_revision, always output +# a git hash rather than a SVN rev. +GIT_MASTERS = [ + 'client.v8', + 'client.v8.branches', + 'tryserver.v8', +] +GIT_MASTERS += internal_data.get('GIT_MASTERS', []) + + # How many times to try before giving up. ATTEMPTS = 5 +# Find deps2git +DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git') +DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py') +S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal', + 'svn_to_git_internal.py') GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') # Find the patch tool. if sys.platform.startswith('win'): - PATCH_TOOL = path.join(THIS_DIR, 'patch.EXE') + if not BUILD_INTERNAL_DIR: + print 'Warning: could not find patch tool because there is no ' + print 'build_internal present.' + PATCH_TOOL = None + else: + PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE') else: PATCH_TOOL = '/usr/bin/patch' @@ -225,6 +393,11 @@ class InvalidDiff(Exception): pass +class Inactive(Exception): + """Not really an exception, just used to exit early cleanly.""" + pass + + RETRY = object() OK = object() FAIL = object() @@ -353,6 +526,34 @@ def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir): } +def check_enabled(master, builder, slave): + if master in ENABLED_MASTERS: + return True + builder_list = ENABLED_BUILDERS.get(master) + if builder_list and builder in builder_list: + return True + slave_list = ENABLED_SLAVES.get(master) + if slave_list and slave in slave_list: + return True + return False + + +def check_disabled(master, builder, slave): + """Returns True if disabled, False if not disabled.""" + builder_list = DISABLED_BUILDERS.get(master) + if builder_list and builder in builder_list: + return True + slave_list = DISABLED_SLAVES.get(master) + if slave_list and slave in slave_list: + return True + return False + + +def check_valid_host(master, builder, slave): + return (check_enabled(master, builder, slave) + and not check_disabled(master, builder, slave)) + + def maybe_ignore_revision(revision, buildspec): """Handle builders that don't care what buildbot tells them to build. @@ -580,6 +781,17 @@ def get_commit_message_footer(message, key): return get_commit_message_footer_map(message).get(key) +def get_svn_rev(git_hash, dir_name): + log = git('log', '-1', git_hash, cwd=dir_name) + git_svn_id = get_commit_message_footer(log, GIT_SVN_ID_FOOTER_KEY) + if not git_svn_id: + return None + m = GIT_SVN_ID_RE.match(git_svn_id) + if not m: + return None + return int(m.group(2)) + + def get_git_hash(revision, branch, sln_dir): """We want to search for the SVN revision on the git-svn branch. @@ -595,6 +807,59 @@ def get_git_hash(revision, branch, sln_dir): (revision, sln_dir)) +def _last_commit_for_file(filename, repo_base): + cmd = ['log', '--format=%H', '--max-count=1', '--', filename] + return git(*cmd, cwd=repo_base).strip() + + +def need_to_run_deps2git(repo_base, deps_file, deps_git_file): + """Checks to see if we need to run deps2git. + + Returns True if there was a DEPS change after the last .DEPS.git update + or if DEPS has local modifications. + """ + # See if DEPS is dirty + deps_file_status = git( + 'status', '--porcelain', deps_file, cwd=repo_base).strip() + if deps_file_status and deps_file_status.startswith('M '): + return True + + last_known_deps_ref = _last_commit_for_file(deps_file, repo_base) + last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base) + merge_base_ref = git('merge-base', last_known_deps_ref, + last_known_deps_git_ref, cwd=repo_base).strip() + + # If the merge base of the last DEPS and last .DEPS.git file is not + # equivilent to the hash of the last DEPS file, that means the DEPS file + # was committed after the last .DEPS.git file. + return last_known_deps_ref != merge_base_ref + + +def ensure_deps2git(solution, shallow, git_cache_dir): + repo_base = path.join(os.getcwd(), solution['name']) + deps_file = path.join(repo_base, 'DEPS') + deps_git_file = path.join(repo_base, '.DEPS.git') + if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or + not git('ls-files', '.DEPS.git', cwd=repo_base).strip()): + return + + print 'Checking if %s is newer than %s' % (deps_file, deps_git_file) + if not need_to_run_deps2git(repo_base, deps_file, deps_git_file): + return + + print '===DEPS file modified, need to run deps2git===' + cmd = [sys.executable, DEPS2GIT_PATH, + '--workspace', os.getcwd(), + '--cache_dir', git_cache_dir, + '--deps', deps_file, + '--out', deps_git_file] + if 'chrome-internal.googlesource' in solution['url']: + cmd.extend(['--extra-rules', S2G_INTERNAL_PATH]) + if shallow: + cmd.append('--shallow') + call(*cmd) + + def emit_log_lines(name, lines): for line in lines.splitlines(): print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line) @@ -660,7 +925,6 @@ def force_revision(folder_name, revision): ref = branch if branch.startswith('refs/') else 'origin/%s' % branch git('checkout', '--force', ref, cwd=folder_name) - def git_checkout(solutions, revisions, shallow, refs, git_cache_dir): build_dir = os.getcwd() # Before we do anything, break all git_cache locks. @@ -721,6 +985,16 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir): else: raise remove(sln_dir) + except SVNRevisionNotFound: + tries_left -= 1 + if tries_left > 0: + # If we don't have the correct revision, wait and try again. + print 'We can\'t find revision %s.' % revision + print 'The svn to git replicator is probably falling behind.' + print 'waiting 5 seconds and trying again...' + time.sleep(5) + else: + raise git('clean', '-dff', cwd=sln_dir) @@ -731,6 +1005,16 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir): return git_ref +def _download(url): + """Fetch url and return content, with retries for flake.""" + for attempt in xrange(ATTEMPTS): + try: + return urllib2.urlopen(url).read() + except Exception: + if attempt == ATTEMPTS - 1: + raise + + def parse_diff(diff): """Takes a unified diff and returns a list of diffed files and their diffs. @@ -940,8 +1224,12 @@ def get_commit_position(git_path, revision='HEAD'): return None -def parse_got_revision(gclient_output, got_revision_mapping): - """Translate git gclient revision mapping to build properties.""" +def parse_got_revision(gclient_output, got_revision_mapping, use_svn_revs): + """Translate git gclient revision mapping to build properties. + + If use_svn_revs is True, then translate git hashes in the revision mapping + to svn revision numbers. + """ properties = {} solutions_output = { # Make sure path always ends with a single slash. @@ -961,7 +1249,12 @@ def parse_got_revision(gclient_output, got_revision_mapping): # Since we are using .DEPS.git, everything had better be git. assert solution_output.get('scm') == 'git' git_revision = git('rev-parse', 'HEAD', cwd=dir_name).strip() - revision = git_revision + if use_svn_revs: + revision = get_svn_rev(git_revision, dir_name) + if not revision: + revision = git_revision + else: + revision = git_revision commit_position = get_commit_position(dir_name) properties[property_name] = revision @@ -993,6 +1286,7 @@ def ensure_deps_revisions(deps_url_mapping, solutions, revisions): revisions) if not revision: continue + # TODO(hinoka): Catch SVNRevisionNotFound error maybe? git('fetch', 'origin', cwd=deps_name) force_revision(deps_name, revision) @@ -1029,6 +1323,11 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, apply_issue_key_file, whitelist=[target]) already_patched.append(target) + if not buildspec: + # Run deps2git if there is a DEPS change after the last .DEPS.git commit. + for solution in solutions: + ensure_deps2git(solution, shallow, git_cache_dir) + # Ensure our build/ directory is set up with the correct .gclient file. gclient_configure(solutions, target_os, target_os_only, git_cache_dir) @@ -1132,6 +1431,8 @@ def parse_args(): help='--private-key-file option passthrough for ' 'apply_patch.py.') parse.add_option('--patch_url', help='Optional URL to SVN patch.') + parse.add_option('--root', dest='patch_root', + help='DEPRECATED: Use --patch_root.') parse.add_option('--patch_root', help='Directory to patch on top of.') parse.add_option('--rietveld_server', default='codereview.chromium.org', @@ -1140,6 +1441,10 @@ def parse_args(): help='Gerrit repository to pull the ref from.') parse.add_option('--gerrit_ref', help='Gerrit ref to apply.') parse.add_option('--specs', help='Gcilent spec.') + parse.add_option('--master', help='Master name.') + parse.add_option('-f', '--force', action='store_true', + help='Bypass check to see if we want to be run. ' + 'Should ONLY be used locally or by smart recipes.') parse.add_option('--revision_mapping', help='{"path/to/repo/": "property_name"}') parse.add_option('--revision_mapping_file', @@ -1155,6 +1460,11 @@ def parse_args(): 'set to :.') parse.add_option('--output_manifest', action='store_true', help=('Add manifest json to the json output.')) + parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0], + help='Hostname of the current machine, ' + 'used for determining whether or not to activate.') + parse.add_option('--builder_name', help='Name of the builder, ' + 'used for determining whether or not to activate.') parse.add_option('--build_dir', default=os.getcwd()) parse.add_option('--flag_file', default=path.join(os.getcwd(), 'update.flag')) @@ -1216,17 +1526,25 @@ def parse_args(): return options, args -def prepare(options, git_slns): +def prepare(options, git_slns, active): """Prepares the target folder before we checkout.""" dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] + # If we're active now, but the flag file doesn't exist (we weren't active + # last run) or vice versa, blow away all checkouts. + if bool(active) != bool(check_flag(options.flag_file)): + ensure_no_checkout(dir_names, '*') if options.output_json: # Make sure we tell recipes that we didn't run if the script exits here. - emit_json(options.output_json, did_run=True) - if options.clobber: - ensure_no_checkout(dir_names, '*') + emit_json(options.output_json, did_run=active) + if active: + if options.clobber: + ensure_no_checkout(dir_names, '*') + else: + ensure_no_checkout(dir_names, '.svn') + emit_flag(options.flag_file) else: - ensure_no_checkout(dir_names, '.svn') - emit_flag(options.flag_file) + delete_flag(options.flag_file) + raise Inactive # This is caught in main() and we exit cleanly. # Do a shallow checkout if the disk is less than 100GB. total_disk_space, free_disk_space = get_total_disk_space() @@ -1253,7 +1571,7 @@ def prepare(options, git_slns): return revisions, step_text -def checkout(options, git_slns, specs, buildspec, +def checkout(options, git_slns, specs, buildspec, master, svn_root, revisions, step_text): first_sln = git_slns[0]['name'] dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] @@ -1315,6 +1633,9 @@ def checkout(options, git_slns, specs, buildspec, print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text raise + # Revision is an svn revision, unless it's a git master. + use_svn_rev = master not in GIT_MASTERS + # Take care of got_revisions outputs. revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) if options.revision_mapping: @@ -1326,7 +1647,8 @@ def checkout(options, git_slns, specs, buildspec, if not revision_mapping: revision_mapping[first_sln] = 'got_revision' - got_revisions = parse_got_revision(gclient_output, revision_mapping) + got_revisions = parse_got_revision(gclient_output, revision_mapping, + use_svn_rev) if not got_revisions: # TODO(hinoka): We should probably bail out here, but in the interest @@ -1351,9 +1673,22 @@ def checkout(options, git_slns, specs, buildspec, emit_properties(got_revisions) -def print_help_text(master, builder, slave): +def print_help_text(force, output_json, active, master, builder, slave): """Print helpful messages to tell devs whats going on.""" + if force and output_json: + recipe_force = 'Forced on by recipes' + elif active and output_json: + recipe_force = 'Off by recipes, but forced on by bot update' + elif not active and output_json: + recipe_force = 'Forced off by recipes' + else: + recipe_force = 'N/A. Was not called by recipes' + print BOT_UPDATE_MESSAGE % { + 'master': master or 'Not specified', + 'builder': builder or 'Not specified', + 'slave': slave or 'Not specified', + 'recipe': recipe_force, 'CURRENT_DIR': CURRENT_DIR, 'BUILDER_DIR': BUILDER_DIR, 'SLAVE_DIR': SLAVE_DIR, @@ -1363,6 +1698,7 @@ def print_help_text(master, builder, slave): 'ROOT_DIR': ROOT_DIR, 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, }, + print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE def main(): @@ -1372,8 +1708,12 @@ def main(): slave = options.slave_name master = options.master - # Prints some debugging information. - print_help_text(master, builder, slave) + # Check if this script should activate or not. + active = check_valid_host(master, builder, slave) or options.force or False + + # Print a helpful message to tell developers whats going on with this step. + print_help_text( + options.force, options.output_json, active, master, builder, slave) # Parse, munipulate, and print the gclient solutions. specs = {} @@ -1386,10 +1726,13 @@ def main(): try: # Dun dun dun, the main part of bot_update. - revisions, step_text = prepare(options, git_slns) - checkout(options, git_slns, specs, buildspec, svn_root, revisions, + revisions, step_text = prepare(options, git_slns, active) + checkout(options, git_slns, specs, buildspec, master, svn_root, revisions, step_text) + except Inactive: + # Not active, should count as passing. + pass except PatchFailed as e: emit_flag(options.flag_file) # Return a specific non-zero exit code for patch failure (because it is diff --git a/recipe_modules/bot_update/resources/patch.exe b/recipe_modules/bot_update/resources/patch.exe deleted file mode 100755 index b8eca1167..000000000 Binary files a/recipe_modules/bot_update/resources/patch.exe and /dev/null differ diff --git a/recipe_modules/bot_update/test_api.py b/recipe_modules/bot_update/test_api.py index 97fac0852..64eab8ab1 100644 --- a/recipe_modules/bot_update/test_api.py +++ b/recipe_modules/bot_update/test_api.py @@ -14,61 +14,73 @@ import bot_update class BotUpdateTestApi(recipe_test_api.RecipeTestApi): - def output_json(self, root, first_sln, revision_mapping, fail_patch=False, + def output_json(self, master, builder, slave, root, first_sln, + revision_mapping, git_mode, force=False, fail_patch=False, output_manifest=False, fixed_revisions=None): """Deterministically synthesize json.output test data for gclient's --output-json option. """ + active = bot_update.check_valid_host(master, builder, slave) or force output = { - 'did_run': True, + 'did_run': active, 'patch_failure': False } - properties = { - property_name: self.gen_revision(project_name, True) - for project_name, property_name in revision_mapping.iteritems() - } - properties.update({ - '%s_cp' % property_name: ('refs/heads/master@{#%s}' % - self.gen_revision(project_name, False)) - for project_name, property_name in revision_mapping.iteritems() - }) + # Add in extra json output if active. + if active: + properties = { + property_name: self.gen_revision(project_name, git_mode) + for project_name, property_name in revision_mapping.iteritems() + } + properties.update({ + '%s_cp' % property_name: ('refs/heads/master@{#%s}' % + self.gen_revision(project_name, False)) + for project_name, property_name in revision_mapping.iteritems() + }) - output.update({ - 'patch_root': root or first_sln, - 'root': first_sln, - 'properties': properties, - 'step_text': 'Some step text' - }) + # We also want to simulate outputting "got_revision_git": ... + # when git mode is off to match what bot_update.py does. + if not git_mode: + properties.update({ + '%s_git' % property_name: self.gen_revision(project_name, True) + for project_name, property_name in revision_mapping.iteritems() + }) - if output_manifest: output.update({ - 'manifest': { - project_name: { - 'repository': 'https://fake.org/%s.git' % project_name, - 'revision': self.gen_revision(project_name, True), - } - for project_name in revision_mapping - } + 'patch_root': root or first_sln, + 'root': first_sln, + 'properties': properties, + 'step_text': 'Some step text' }) - if fixed_revisions: - output['fixed_revisions'] = fixed_revisions + if output_manifest: + output.update({ + 'manifest': { + project_name: { + 'repository': 'https://fake.org/%s.git' % project_name, + 'revision': self.gen_revision(project_name, git_mode), + } + for project_name in revision_mapping + } + }) + + if fixed_revisions: + output['fixed_revisions'] = fixed_revisions - if fail_patch: - output['log_lines'] = [('patch error', 'Patch failed to apply'),] - output['patch_failure'] = True - output['patch_apply_return_code'] = 1 - if fail_patch == 'download': - output['patch_apply_return_code'] = 3 + if fail_patch: + output['log_lines'] = [('patch error', 'Patch failed to apply'),] + output['patch_failure'] = True + output['patch_apply_return_code'] = 1 + if fail_patch == 'download': + output['patch_apply_return_code'] = 3 return self.m.json.output(output) @staticmethod - def gen_revision(project, git_mode): + def gen_revision(project, GIT_MODE): """Hash project to bogus deterministic revision values.""" h = hashlib.sha1(project) - if git_mode: + if GIT_MODE: return h.hexdigest() else: return struct.unpack('!I', h.digest()[:4])[0] % 300000 diff --git a/tests/bot_update_coverage_test.py b/tests/bot_update_coverage_test.py deleted file mode 100755 index 19adfd2f8..000000000 --- a/tests/bot_update_coverage_test.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2015 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import codecs -import copy -import json -import os -import sys -import unittest - -ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -RESOURCE_DIR = os.path.join( - ROOT_DIR, 'recipe_modules', 'bot_update', 'resources') - -sys.path.insert(0, RESOURCE_DIR) -sys.platform = 'linux2' # For consistency, ya know? -import bot_update - -DEFAULT_PARAMS = { - 'solutions': [{ - 'name': 'somename', - 'url': 'https://fake.com' - }], - 'revisions': [], - 'first_sln': 'somename', - 'target_os': None, - 'target_os_only': None, - 'patch_root': None, - 'issue': None, - 'patchset': None, - 'patch_url': None, - 'rietveld_server': None, - 'gerrit_ref': None, - 'gerrit_repo': None, - 'revision_mapping': {}, - 'apply_issue_email_file': None, - 'apply_issue_key_file': None, - 'buildspec': False, - 'gyp_env': None, - 'shallow': False, - 'runhooks': False, - 'refs': [], - 'git_cache_dir': 'fake_cache_dir' -} - - -class MockedPopen(object): - """A fake instance of a called subprocess. - - This is meant to be used in conjunction with MockedCall. - """ - def __init__(self, args=None, kwargs=None): - self.args = args or [] - self.kwargs = kwargs or {} - self.return_value = None - self.fails = False - - def returns(self, rv): - """Set the return value when this popen is called. - - rv can be a string, or a callable (eg function). - """ - self.return_value = rv - return self - - def check(self, args, kwargs): - """Check to see if the given args/kwargs call match this instance. - - This does a partial match, so that a call to "git clone foo" will match - this instance if this instance was recorded as "git clone" - """ - if any(input_arg != expected_arg - for (input_arg, expected_arg) in zip(args, self.args)): - return False - return self.return_value - - def __call__(self, args, kwargs): - """Actually call this popen instance.""" - if hasattr(self.return_value, '__call__'): - return self.return_value(*args, **kwargs) - return self.return_value - - -class MockedCall(object): - """A fake instance of bot_update.call(). - - This object is pre-seeded with "answers" in self.expectations. The type - is a MockedPopen object, or any object with a __call__() and check() method. - The check() method is used to check to see if the correct popen object is - chosen (can be a partial match, eg a "git clone" popen module would match - a "git clone foo" call). - By default, if no answers have been pre-seeded, the call() returns successful - with an empty string. - """ - def __init__(self, fake_filesystem): - self.expectations = [] - self.records = [] - - def expect(self, args=None, kwargs=None): - args = args or [] - kwargs = kwargs or {} - popen = MockedPopen(args, kwargs) - self.expectations.append(popen) - return popen - - def __call__(self, *args, **kwargs): - self.records.append((args, kwargs)) - for popen in self.expectations: - if popen.check(args, kwargs): - self.expectations.remove(popen) - return popen(args, kwargs) - return '' - - -class MockedGclientSync(): - """A class producing a callable instance of gclient sync. - - Because for bot_update, gclient sync also emits an output json file, we need - a callable object that can understand where the output json file is going, and - emit a (albite) fake file for bot_update to consume. - """ - def __init__(self, fake_filesystem): - self.output = {} - self.fake_filesystem = fake_filesystem - - def __call__(self, *args, **_): - output_json_index = args.index('--output-json') + 1 - with self.fake_filesystem.open(args[output_json_index], 'w') as f: - json.dump(self.output, f) - - -class FakeFile(): - def __init__(self): - self.contents = '' - - def write(self, buf): - self.contents += buf - - def read(self): - return self.contents - - def __enter__(self): - return self - - def __exit__(self, _, __, ___): - pass - - -class FakeFilesystem(): - def __init__(self): - self.files = {} - - def open(self, target, mode='r', encoding=None): - if 'w' in mode: - self.files[target] = FakeFile() - return self.files[target] - return self.files[target] - - -def fake_git(*args, **kwargs): - return bot_update.call('git', *args, **kwargs) - - -class EnsureCheckoutUnittests(unittest.TestCase): - def setUp(self): - self.filesystem = FakeFilesystem() - self.call = MockedCall(self.filesystem) - self.gclient = MockedGclientSync(self.filesystem) - self.call.expect(('gclient', 'sync')).returns(self.gclient) - self.old_call = getattr(bot_update, 'call') - self.params = copy.deepcopy(DEFAULT_PARAMS) - setattr(bot_update, 'call', self.call) - setattr(bot_update, 'git', fake_git) - - self.old_os_cwd = os.getcwd - setattr(os, 'getcwd', lambda: '/b/build/slave/foo/build') - - setattr(bot_update, 'open', self.filesystem.open) - self.old_codecs_open = codecs.open - setattr(codecs, 'open', self.filesystem.open) - - def tearDown(self): - setattr(bot_update, 'call', self.old_call) - setattr(os, 'getcwd', self.old_os_cwd) - delattr(bot_update, 'open') - setattr(codecs, 'open', self.old_codecs_open) - - def testBasic(self): - bot_update.ensure_checkout(**self.params) - return self.call.records - - def testBasicBuildspec(self): - self.params['buildspec'] = bot_update.BUILDSPEC_TYPE( - container='branches', - version='1.1.1.1' - ) - bot_update.ensure_checkout(**self.params) - return self.call.records - - def testBasicShallow(self): - self.params['shallow'] = True - bot_update.ensure_checkout(**self.params) - return self.call.records - - def testBasicSVNPatch(self): - self.params['patch_url'] = 'svn://server.com/patch.diff' - self.params['patch_root'] = 'somename' - bot_update.ensure_checkout(**self.params) - return self.call.records - - def testBasicRietveldPatch(self): - self.params['issue'] = '12345' - self.params['patchset'] = '1' - self.params['rietveld_server'] = 'https://rietveld.com' - self.params['patch_root'] = 'somename' - bot_update.ensure_checkout(**self.params) - return self.call.records - - -class SolutionsUnittests(unittest.TestCase): - def testBasicSVN(self): - sln = [{ - 'name': 'src', - 'url': 'svn://svn-mirror.golo.chromium.org/chrome/trunk/src' - }, { - 'name': 'git', - 'url': 'https://abc.googlesource.com/foo.git' - }] - git_slns, root, buildspec = bot_update.solutions_to_git(sln) - self.assertEquals(root, '/chrome/trunk/src') - self.assertEquals(git_slns, [{ - 'deps_file': '.DEPS.git', - 'managed': False, - 'name': 'src', - 'url': 'https://chromium.googlesource.com/chromium/src.git' - }, { - 'url': 'https://abc.googlesource.com/foo.git', - 'managed': False, - 'name': 'git' - }]) - self.assertFalse(buildspec) - bot_update.solutions_printer(git_slns) - - def testBasicBuildspec(self): - sln = [{ - 'name': 'buildspec', - 'url': ('svn://svn.chromium.org/chrome-internal/' - 'trunk/tools/buildspec/releases/1234'), - }] - git_slns, root, buildspec = bot_update.solutions_to_git(sln) - self.assertEquals( - buildspec, - bot_update.BUILDSPEC_TYPE(container='releases', version='1234')) - self.assertEquals( - root, '/chrome-internal/trunk/tools/buildspec/releases/1234') - self.assertEquals(git_slns[0]['deps_file'], 'releases/1234/DEPS') - -if __name__ == '__main__': - unittest.main()