diff --git a/internal/actions/mocks/client.go b/internal/actions/mocks/client.go index 737404a1..b482475e 100644 --- a/internal/actions/mocks/client.go +++ b/internal/actions/mocks/client.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + c "github.com/containrrr/watchtower/pkg/container" t "github.com/containrrr/watchtower/pkg/types" ) @@ -72,16 +73,16 @@ func (client MockClient) GetContainer(_ t.ContainerID) (t.Container, error) { } // ExecuteCommand is a mock method -func (client MockClient) ExecuteCommand(_ t.ContainerID, command string, _ int) (SkipUpdate bool, err error) { +func (client MockClient) ExecuteCommand(_ t.ContainerID, command string, _ time.Duration) error { switch command { case "/PreUpdateReturn0.sh": - return false, nil + return nil case "/PreUpdateReturn1.sh": - return false, fmt.Errorf("command exited with code 1") + return fmt.Errorf("command exited with code 1") case "/PreUpdateReturn75.sh": - return true, nil + return c.ErrorLifecycleSkip default: - return false, nil + return nil } } diff --git a/internal/util/time.go b/internal/util/time.go new file mode 100644 index 00000000..3ae7c012 --- /dev/null +++ b/internal/util/time.go @@ -0,0 +1,15 @@ +package util + +import ( + "strconv" + "time" +) + +// ParseDuration parses the input string as a duration, treating a plain number as implicitly using the specified unit +func ParseDuration(input string, unitlessUnit time.Duration) (time.Duration, error) { + if unitless, err := strconv.Atoi(input); err == nil { + return unitlessUnit * time.Duration(unitless), nil + } + + return time.ParseDuration(input) +} diff --git a/pkg/container/client.go b/pkg/container/client.go index c6c37deb..3af08db7 100644 --- a/pkg/container/client.go +++ b/pkg/container/client.go @@ -31,7 +31,7 @@ type Client interface { StartContainer(t.Container) (t.ContainerID, error) RenameContainer(t.Container, string) error IsContainerStale(t.Container, t.UpdateParams) (stale bool, latestImage t.ImageID, err error) - ExecuteCommand(containerID t.ContainerID, command string, timeout int) (SkipUpdate bool, err error) + ExecuteCommand(containerID t.ContainerID, command string, timeout time.Duration) error RemoveImageByID(t.ImageID) error WarnOnHeadPullFailed(container t.Container) bool } @@ -439,7 +439,7 @@ func (client dockerClient) RemoveImageByID(id t.ImageID) error { return err } -func (client dockerClient) ExecuteCommand(containerID t.ContainerID, command string, timeout int) (SkipUpdate bool, err error) { +func (client dockerClient) ExecuteCommand(containerID t.ContainerID, command string, timeout time.Duration) error { bg := context.Background() clog := log.WithField("containerID", containerID) @@ -452,7 +452,7 @@ func (client dockerClient) ExecuteCommand(containerID t.ContainerID, command str exec, err := client.api.ContainerExecCreate(bg, string(containerID), execConfig) if err != nil { - return false, err + return err } response, attachErr := client.api.ContainerExecAttach(bg, exec.ID, types.ExecStartCheck{ @@ -467,7 +467,7 @@ func (client dockerClient) ExecuteCommand(containerID t.ContainerID, command str execStartCheck := types.ExecStartCheck{Detach: false, Tty: true} err = client.api.ContainerExecStart(bg, exec.ID, execStartCheck) if err != nil { - return false, err + return err } var output string @@ -484,24 +484,16 @@ func (client dockerClient) ExecuteCommand(containerID t.ContainerID, command str // Inspect the exec to get the exit code and print a message if the // exit code is not success. - skipUpdate, err := client.waitForExecOrTimeout(bg, exec.ID, output, timeout) - if err != nil { - return true, err - } - - return skipUpdate, nil + return client.waitForExecOrTimeout(bg, exec.ID, output, timeout) } -func (client dockerClient) waitForExecOrTimeout(bg context.Context, ID string, execOutput string, timeout int) (SkipUpdate bool, err error) { +func (client dockerClient) waitForExecOrTimeout(ctx context.Context, ID string, execOutput string, timeout time.Duration) error { const ExTempFail = 75 - var ctx context.Context - var cancel context.CancelFunc if timeout > 0 { - ctx, cancel = context.WithTimeout(bg, time.Duration(timeout)*time.Minute) + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, timeout) defer cancel() - } else { - ctx = bg } for { @@ -516,7 +508,7 @@ func (client dockerClient) waitForExecOrTimeout(bg context.Context, ID string, e }).Debug("Awaiting timeout or completion") if err != nil { - return false, err + return err } if execInspect.Running { time.Sleep(1 * time.Second) @@ -527,15 +519,15 @@ func (client dockerClient) waitForExecOrTimeout(bg context.Context, ID string, e } if execInspect.ExitCode == ExTempFail { - return true, nil + return ErrorLifecycleSkip } if execInspect.ExitCode > 0 { - return false, fmt.Errorf("command exited with code %v %s", execInspect.ExitCode, execOutput) + return fmt.Errorf("command exited with code %v", execInspect.ExitCode) } break } - return false, nil + return nil } func (client dockerClient) waitForStopOrTimeout(c t.Container, waitTime time.Duration) error { diff --git a/pkg/container/client_test.go b/pkg/container/client_test.go index 4e754091..2fea1039 100644 --- a/pkg/container/client_test.go +++ b/pkg/container/client_test.go @@ -308,7 +308,7 @@ var _ = Describe("the client", func() { ), ) - _, err := client.ExecuteCommand(containerID, cmd, 1) + err := client.ExecuteCommand(containerID, cmd, 1) Expect(err).NotTo(HaveOccurred()) // Note: Since Execute requires opening up a raw TCP stream to the daemon for the output, this will fail // when using the mock API server. Regardless of the outcome, the log should include the container ID diff --git a/pkg/container/container.go b/pkg/container/container.go index 10ed677f..000a33d0 100644 --- a/pkg/container/container.go +++ b/pkg/container/container.go @@ -219,44 +219,6 @@ func (c Container) IsWatchtower() bool { return ContainsWatchtowerLabel(c.containerInfo.Config.Labels) } -// PreUpdateTimeout checks whether a container has a specific timeout set -// for how long the pre-update command is allowed to run. This value is expressed -// either as an integer, in minutes, or as 0 which will allow the command/script -// to run indefinitely. Users should be cautious with the 0 option, as that -// could result in watchtower waiting forever. -func (c Container) PreUpdateTimeout() int { - var minutes int - var err error - - val := c.getLabelValueOrEmpty(preUpdateTimeoutLabel) - - minutes, err = strconv.Atoi(val) - if err != nil || val == "" { - return 1 - } - - return minutes -} - -// PostUpdateTimeout checks whether a container has a specific timeout set -// for how long the post-update command is allowed to run. This value is expressed -// either as an integer, in minutes, or as 0 which will allow the command/script -// to run indefinitely. Users should be cautious with the 0 option, as that -// could result in watchtower waiting forever. -func (c Container) PostUpdateTimeout() int { - var minutes int - var err error - - val := c.getLabelValueOrEmpty(postUpdateTimeoutLabel) - - minutes, err = strconv.Atoi(val) - if err != nil || val == "" { - return 1 - } - - return minutes -} - // StopSignal returns the custom stop signal (if any) that is encoded in the // container's metadata. If the container has not specified a custom stop // signal, the empty string "" is returned. diff --git a/pkg/container/errors.go b/pkg/container/errors.go index 05dc7223..2caa2a05 100644 --- a/pkg/container/errors.go +++ b/pkg/container/errors.go @@ -6,3 +6,6 @@ var errorNoImageInfo = errors.New("no available image info") var errorNoContainerInfo = errors.New("no available container info") var errorInvalidConfig = errors.New("container configuration missing or invalid") var errorLabelNotFound = errors.New("label was not found in container") + +// ErrorLifecycleSkip is returned by a lifecycle hook when the exit code of the command indicated that it ought to be skipped +var ErrorLifecycleSkip = errors.New("skipping container as the pre-update command returned exit code 75 (EX_TEMPFAIL)") diff --git a/pkg/container/metadata.go b/pkg/container/metadata.go index 8ac5f345..f15b89ab 100644 --- a/pkg/container/metadata.go +++ b/pkg/container/metadata.go @@ -1,67 +1,92 @@ package container -import "strconv" +import ( + "errors" + "fmt" + "strconv" + "time" + + "github.com/containrrr/watchtower/internal/util" + wt "github.com/containrrr/watchtower/pkg/types" + + "github.com/sirupsen/logrus" +) const ( - watchtowerLabel = "com.centurylinklabs.watchtower" - signalLabel = "com.centurylinklabs.watchtower.stop-signal" - enableLabel = "com.centurylinklabs.watchtower.enable" - monitorOnlyLabel = "com.centurylinklabs.watchtower.monitor-only" - noPullLabel = "com.centurylinklabs.watchtower.no-pull" - dependsOnLabel = "com.centurylinklabs.watchtower.depends-on" - zodiacLabel = "com.centurylinklabs.zodiac.original-image" - scope = "com.centurylinklabs.watchtower.scope" - preCheckLabel = "com.centurylinklabs.watchtower.lifecycle.pre-check" - postCheckLabel = "com.centurylinklabs.watchtower.lifecycle.post-check" - preUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.pre-update" - postUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.post-update" - preUpdateTimeoutLabel = "com.centurylinklabs.watchtower.lifecycle.pre-update-timeout" - postUpdateTimeoutLabel = "com.centurylinklabs.watchtower.lifecycle.post-update-timeout" + namespace = "com.centurylinklabs.watchtower" + watchtowerLabel = namespace + signalLabel = namespace + ".stop-signal" + enableLabel = namespace + ".enable" + monitorOnlyLabel = namespace + ".monitor-only" + noPullLabel = namespace + ".no-pull" + dependsOnLabel = namespace + ".depends-on" + zodiacLabel = "com.centurylinklabs.zodiac.original-image" + scope = namespace + ".scope" ) -// GetLifecyclePreCheckCommand returns the pre-check command set in the container metadata or an empty string -func (c Container) GetLifecyclePreCheckCommand() string { - return c.getLabelValueOrEmpty(preCheckLabel) +// ContainsWatchtowerLabel takes a map of labels and values and tells +// the consumer whether it contains a valid watchtower instance label +func ContainsWatchtowerLabel(labels map[string]string) bool { + val, ok := labels[watchtowerLabel] + return ok && val == "true" } -// GetLifecyclePostCheckCommand returns the post-check command set in the container metadata or an empty string -func (c Container) GetLifecyclePostCheckCommand() string { - return c.getLabelValueOrEmpty(postCheckLabel) -} +// GetLifecycleCommand returns the lifecycle command set in the container metadata or an empty string +func (c *Container) GetLifecycleCommand(phase wt.LifecyclePhase) string { + label := fmt.Sprintf("%v.lifecycle.%v", namespace, phase) + value, found := c.getLabelValue(label) -// GetLifecyclePreUpdateCommand returns the pre-update command set in the container metadata or an empty string -func (c Container) GetLifecyclePreUpdateCommand() string { - return c.getLabelValueOrEmpty(preUpdateLabel) -} + if !found { + return "" + } -// GetLifecyclePostUpdateCommand returns the post-update command set in the container metadata or an empty string -func (c Container) GetLifecyclePostUpdateCommand() string { - return c.getLabelValueOrEmpty(postUpdateLabel) + return value } -// ContainsWatchtowerLabel takes a map of labels and values and tells -// the consumer whether it contains a valid watchtower instance label -func ContainsWatchtowerLabel(labels map[string]string) bool { - val, ok := labels[watchtowerLabel] - return ok && val == "true" +// GetLifecycleTimeout checks whether a container has a specific timeout set +// for how long the lifecycle command is allowed to run. This value is expressed +// either as a duration, an integer (minutes implied), or as 0 which will allow the command/script +// to run indefinitely. Users should be cautious with the 0 option, as that +// could result in watchtower waiting forever. +func (c *Container) GetLifecycleTimeout(phase wt.LifecyclePhase) time.Duration { + label := fmt.Sprintf("%v.lifecycle.%v-timeout", namespace, phase) + timeout, err := c.getDurationLabelValue(label, time.Minute) + + if err != nil { + timeout = time.Minute + if !errors.Is(err, errorLabelNotFound) { + logrus.WithError(err).Errorf("could not parse timeout label value for %v lifecycle", phase) + } + } + + return timeout } -func (c Container) getLabelValueOrEmpty(label string) string { +func (c *Container) getLabelValueOrEmpty(label string) string { if val, ok := c.containerInfo.Config.Labels[label]; ok { return val } return "" } -func (c Container) getLabelValue(label string) (string, bool) { +func (c *Container) getLabelValue(label string) (string, bool) { val, ok := c.containerInfo.Config.Labels[label] return val, ok } -func (c Container) getBoolLabelValue(label string) (bool, error) { +func (c *Container) getBoolLabelValue(label string) (bool, error) { if strVal, ok := c.containerInfo.Config.Labels[label]; ok { value, err := strconv.ParseBool(strVal) return value, err } return false, errorLabelNotFound } + +func (c *Container) getDurationLabelValue(label string, unitlessUnit time.Duration) (time.Duration, error) { + value, found := c.getLabelValue(label) + if !found || len(value) < 1 { + return 0, errorLabelNotFound + } + + return util.ParseDuration(value, unitlessUnit) +} diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index c0f962e4..db93fc04 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -6,101 +6,60 @@ import ( log "github.com/sirupsen/logrus" ) -// ExecutePreChecks tries to run the pre-check lifecycle hook for all containers included by the current filter. -func ExecutePreChecks(client container.Client, params types.UpdateParams) { - containers, err := client.ListContainers(params.Filter) - if err != nil { - return - } - for _, currentContainer := range containers { - ExecutePreCheckCommand(client, currentContainer) - } -} - -// ExecutePostChecks tries to run the post-check lifecycle hook for all containers included by the current filter. -func ExecutePostChecks(client container.Client, params types.UpdateParams) { - containers, err := client.ListContainers(params.Filter) - if err != nil { - return - } - for _, currentContainer := range containers { - ExecutePostCheckCommand(client, currentContainer) - } -} +type ExecCommandFunc func(client container.Client, container types.Container) // ExecutePreCheckCommand tries to run the pre-check lifecycle hook for a single container. func ExecutePreCheckCommand(client container.Client, container types.Container) { - clog := log.WithField("container", container.Name()) - command := container.GetLifecyclePreCheckCommand() - if len(command) == 0 { - clog.Debug("No pre-check command supplied. Skipping") - return - } - - clog.Debug("Executing pre-check command.") - _, err := client.ExecuteCommand(container.ID(), command, 1) + err := ExecuteLifeCyclePhaseCommand(types.PreCheck, client, container) if err != nil { - clog.Error(err) + log.WithField("container", container.Name()).Error(err) } } // ExecutePostCheckCommand tries to run the post-check lifecycle hook for a single container. func ExecutePostCheckCommand(client container.Client, container types.Container) { - clog := log.WithField("container", container.Name()) - command := container.GetLifecyclePostCheckCommand() - if len(command) == 0 { - clog.Debug("No post-check command supplied. Skipping") - return - } - - clog.Debug("Executing post-check command.") - _, err := client.ExecuteCommand(container.ID(), command, 1) + err := ExecuteLifeCyclePhaseCommand(types.PostCheck, client, container) if err != nil { - clog.Error(err) + log.WithField("container", container.Name()).Error(err) } } // ExecutePreUpdateCommand tries to run the pre-update lifecycle hook for a single container. -func ExecutePreUpdateCommand(client container.Client, container types.Container) (SkipUpdate bool, err error) { - timeout := container.PreUpdateTimeout() - command := container.GetLifecyclePreUpdateCommand() - clog := log.WithField("container", container.Name()) - - if len(command) == 0 { - clog.Debug("No pre-update command supplied. Skipping") - return false, nil - } - - if !container.IsRunning() || container.IsRestarting() { - clog.Debug("Container is not running. Skipping pre-update command.") - return false, nil - } - - clog.Debug("Executing pre-update command.") - return client.ExecuteCommand(container.ID(), command, timeout) +func ExecutePreUpdateCommand(client container.Client, container types.Container) error { + return ExecuteLifeCyclePhaseCommand(types.PreUpdate, client, container) } // ExecutePostUpdateCommand tries to run the post-update lifecycle hook for a single container. func ExecutePostUpdateCommand(client container.Client, newContainerID types.ContainerID) { newContainer, err := client.GetContainer(newContainerID) - timeout := newContainer.PostUpdateTimeout() - if err != nil { log.WithField("containerID", newContainerID.ShortID()).Error(err) return } - clog := log.WithField("container", newContainer.Name()) - command := newContainer.GetLifecyclePostUpdateCommand() - if len(command) == 0 { - clog.Debug("No post-update command supplied. Skipping") - return + err = ExecuteLifeCyclePhaseCommand(types.PostUpdate, client, newContainer) + if err != nil { + log.WithField("container", newContainer.Name()).Error(err) } +} - clog.Debug("Executing post-update command.") - _, err = client.ExecuteCommand(newContainerID, command, timeout) +// ExecuteLifeCyclePhaseCommand tries to run the corresponding lifecycle hook for a single container. +func ExecuteLifeCyclePhaseCommand(phase types.LifecyclePhase, client container.Client, container types.Container) error { - if err != nil { - clog.Error(err) + timeout := container.GetLifecycleTimeout(phase) + command := container.GetLifecycleCommand(phase) + clog := log.WithField("container", container.Name()) + + if len(command) == 0 { + clog.Debugf("No %v command supplied. Skipping", phase) + return nil } + + if !container.IsRunning() || container.IsRestarting() { + clog.Debugf("Container is not running. Skipping %v command.", phase) + return nil + } + + clog.Debugf("Executing %v command.", phase) + return client.ExecuteCommand(container.ID(), command, timeout) } diff --git a/pkg/types/container.go b/pkg/types/container.go index 8a22f440..cc8be9fd 100644 --- a/pkg/types/container.go +++ b/pkg/types/container.go @@ -2,6 +2,7 @@ package types import ( "strings" + "time" "github.com/docker/docker/api/types" dc "github.com/docker/docker/api/types/container" @@ -60,18 +61,14 @@ type Container interface { StopSignal() string HasImageInfo() bool ImageInfo() *types.ImageInspect - GetLifecyclePreCheckCommand() string - GetLifecyclePostCheckCommand() string - GetLifecyclePreUpdateCommand() string - GetLifecyclePostUpdateCommand() string + GetLifecycleCommand(LifecyclePhase) string + GetLifecycleTimeout(LifecyclePhase) time.Duration VerifyConfiguration() error SetStale(bool) IsStale() bool IsNoPull(UpdateParams) bool SetLinkedToRestarting(bool) IsLinkedToRestarting() bool - PreUpdateTimeout() int - PostUpdateTimeout() int IsRestarting() bool GetCreateConfig() *dc.Config GetCreateHostConfig() *dc.HostConfig diff --git a/pkg/types/lifecycle.go b/pkg/types/lifecycle.go new file mode 100644 index 00000000..2c7b46cb --- /dev/null +++ b/pkg/types/lifecycle.go @@ -0,0 +1,27 @@ +package types + +import "fmt" + +type LifecyclePhase int + +const ( + PreCheck LifecyclePhase = iota + PreUpdate + PostUpdate + PostCheck +) + +func (p LifecyclePhase) String() string { + switch p { + case PreCheck: + return "pre-check" + case PreUpdate: + return "pre-update" + case PostUpdate: + return "post-update" + case PostCheck: + return "post-check" + default: + return fmt.Sprintf("invalid(%d)", p) + } +}