Skip to content

Commit 303c8e4

Browse files
psviderskipsviderski
committed
feat(deploy): print last logs from new container when fails to become healthy
1 parent 8afe523 commit 303c8e4

3 files changed

Lines changed: 76 additions & 24 deletions

File tree

‎cmd/uncloud/deploy.go‎

Lines changed: 41 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"charm.land/lipgloss/v2"
1010
composecli "github.com/compose-spec/compose-go/v2/cli"
1111
"github.com/docker/compose/v2/pkg/progress"
12+
"github.com/docker/docker/pkg/stringid"
1213
"github.com/psviderski/uncloud/internal/cli"
1314
"github.com/psviderski/uncloud/internal/cli/logs"
1415
"github.com/psviderski/uncloud/internal/cli/tui"
@@ -20,10 +21,6 @@ import (
2021
"github.com/spf13/cobra"
2122
)
2223

23-
// failedContainerLogsTail is the number of recent log lines to print from a failed container to give the user immediate
24-
// context without requiring a follow-up 'uc logs' invocation.
25-
const failedContainerLogsTail = 10
26-
2724
type deployOptions struct {
2825
cli.BuildServicesOptions
2926

@@ -238,34 +235,46 @@ func runDeploy(ctx context.Context, uncli *cli.CLI, opts deployOptions) error {
238235
}, uncli.ProgressOut(), title)
239236
if err != nil {
240237
fmt.Println()
238+
239+
tail := failedContainerLogsTail()
241240
if hookErr, ok := errors.AsType[*operation.PreDeployHookError](err); ok {
242-
printPreDeployHookLogs(ctx, clusterClient, hookErr)
241+
printFailedContainerLogs(ctx, clusterClient,
242+
hookErr.ServiceName, hookErr.ContainerID, hookErr.MachineName, tail,
243+
fmt.Sprintf("Last %d log lines from failed pre-deploy hook:", tail))
244+
fmt.Println()
245+
} else if startErr, ok := errors.AsType[*operation.ContainerHealthError](err); ok {
246+
printFailedContainerLogs(ctx, clusterClient,
247+
startErr.ServiceName, startErr.ContainerID, startErr.MachineName, tail,
248+
fmt.Sprintf("Last %d log lines from failed container:", tail))
243249
fmt.Println()
244250
}
251+
245252
return err
246253
}
247254
return nil
248255
}
249256

250-
// printPreDeployHookLogs fetches the last log lines from the failed pre-deploy hook container and prints them using
251-
// the standard log formatter.
252-
func printPreDeployHookLogs(ctx context.Context, cli *client.Client, hookErr *operation.PreDeployHookError) {
253-
_, ch, err := cli.ServiceLogs(ctx, hookErr.ServiceName, api.ServiceLogsOptions{
254-
Containers: []string{hookErr.ContainerID},
255-
Machines: []string{hookErr.MachineName},
256-
Tail: failedContainerLogsTail,
257+
// printFailedContainerLogs fetches the last tail log lines from a container that failed during deployment and prints
258+
// them using the standard log formatter under the provided header.
259+
func printFailedContainerLogs(
260+
ctx context.Context, cli *client.Client, serviceName, containerID, machineName string, tail int, header string,
261+
) {
262+
_, ch, err := cli.ServiceLogs(ctx, serviceName, api.ServiceLogsOptions{
263+
Containers: []string{containerID},
264+
Machines: []string{machineName},
265+
Tail: tail,
257266
})
258267
if err != nil {
259-
fmt.Fprintf(os.Stderr, "Failed to fetch pre-deploy hook logs: %v\n", err)
260-
fmt.Fprintf(os.Stderr, "You can try manually with: uc logs %s\n", hookErr.ServiceName)
268+
shortCtrID := stringid.TruncateID(containerID)
269+
fmt.Fprintf(os.Stderr, "Failed to fetch container logs '%s/%s': %v\n", serviceName, shortCtrID, err)
270+
fmt.Fprintf(os.Stderr, "You can try manually with: uc logs %s/%s\n", serviceName, shortCtrID)
261271
return
262272
}
263273

264-
header := fmt.Sprintf("Last %d log lines from failed pre-deploy hook:", failedContainerLogsTail)
265274
fmt.Println(tui.BoldRed.Render(header))
266275

267276
logsEmpty := true
268-
formatter := logs.NewFormatter([]string{hookErr.MachineName}, []string{hookErr.ServiceName}, false)
277+
formatter := logs.NewFormatter([]string{machineName}, []string{serviceName}, false)
269278
for entry := range ch {
270279
logsEmpty = false
271280
formatter.PrintEntry(entry)
@@ -275,3 +284,19 @@ func printPreDeployHookLogs(ctx context.Context, cli *client.Client, hookErr *op
275284
fmt.Println("<no logs available>")
276285
}
277286
}
287+
288+
// defaultFailedContainerLogsTail is the default number of recent log lines to print from a failed container to give
289+
// the user immediate context without requiring a follow-up 'uc logs' invocation.
290+
// Overridable via UNCLOUD_FAILED_CONTAINER_LOGS_TAIL.
291+
const defaultFailedContainerLogsTail = 10
292+
293+
// failedContainerLogsTail returns the number of log lines to fetch from a failed container, honouring the
294+
// UNCLOUD_FAILED_CONTAINER_LOGS_TAIL environment variable override when set and valid.
295+
func failedContainerLogsTail() int {
296+
if v := os.Getenv("UNCLOUD_FAILED_CONTAINER_LOGS_TAIL"); v != "" {
297+
if tail, err := logs.Tail(v); err == nil && (tail == -1 || tail > 0) {
298+
return tail
299+
}
300+
}
301+
return defaultFailedContainerLogsTail
302+
}

‎pkg/client/deploy/operation/container.go‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,18 @@ import (
1313
"github.com/psviderski/uncloud/pkg/api"
1414
)
1515

16+
// ContainerHealthError indicates that a service container failed to become healthy during deployment.
17+
type ContainerHealthError struct {
18+
error
19+
ServiceName string
20+
ContainerID string
21+
MachineName string
22+
}
23+
24+
func (e *ContainerHealthError) Unwrap() error {
25+
return e.error
26+
}
27+
1628
// RunContainerOperation creates and starts a new container on a specific machine.
1729
type RunContainerOperation struct {
1830
ServiceID string
@@ -43,8 +55,13 @@ func (o *RunContainerOperation) Execute(ctx context.Context, cli Client) error {
4355

4456
opts := api.WaitContainerHealthyOptions{MonitorPeriod: o.Spec.UpdateConfig.MonitorPeriod}
4557
if err = cli.WaitContainerHealthy(ctx, o.ServiceID, resp.ID, opts); err != nil {
46-
return fmt.Errorf("container '%s/%s' failed to become healthy: %w",
47-
o.Spec.Name, stringid.TruncateID(resp.ID), err)
58+
return &ContainerHealthError{
59+
error: fmt.Errorf("container '%s/%s' failed to become healthy: %w",
60+
o.Spec.Name, stringid.TruncateID(resp.ID), err),
61+
ServiceName: o.Spec.Name,
62+
ContainerID: resp.ID,
63+
MachineName: o.MachineName,
64+
}
4865
}
4966

5067
return nil
@@ -187,7 +204,6 @@ func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) err
187204
if err = cli.WaitContainerHealthy(newCtx, o.ServiceID, resp.ID, opts); err != nil {
188205
// New container failed to become healthy. Stop it and roll back to the previous container.
189206
// Don't remove the new stopped container to allow users to inspect logs and state.
190-
// TODO: collect logs from the new container and include in the error message to speed up debugging.
191207

192208
// Use context without progress to not overwrite the container Unhealthy status with Stopped.
193209
ctxWithoutProgress := progress.WithContextWriter(ctx, nil)
@@ -196,21 +212,28 @@ func (o *ReplaceContainerOperation) Execute(ctx context.Context, cli Client) err
196212
newCtr := fmt.Sprintf("%s/%s", o.Spec.Name, stringid.TruncateID(resp.ID))
197213
healthErr := fmt.Errorf(
198214
"new container '%s' failed to become healthy: %w. "+
199-
"It's stopped and available for inspection. Fetch logs with 'uc logs %s'",
200-
newCtr, err, o.Spec.Name,
215+
"It's stopped and available for inspection. View logs with 'uc logs %s'",
216+
newCtr, err, newCtr,
201217
)
202218

219+
finalErr := healthErr
203220
if stopFirst && wasRunning {
204221
// Restart the old container only if it was running before we stopped it.
205222
oldCtr := fmt.Sprintf("%s/%s", o.OldContainer.ServiceSpec.Name, o.OldContainer.ShortID())
206223
if rollbackErr := cli.StartContainer(ctx, o.ServiceID, o.OldContainer.ID); rollbackErr != nil {
207-
return fmt.Errorf("%w. Rolled back to old container '%s' but failed to restart it: %w",
224+
finalErr = fmt.Errorf("%w. Rolled back to old container '%s' but failed to restart it: %w",
208225
healthErr, oldCtr, rollbackErr)
226+
} else {
227+
finalErr = fmt.Errorf("%w. Rolled back to old container '%s'", healthErr, oldCtr)
209228
}
210-
return fmt.Errorf("%w. Rolled back to old container '%s'", healthErr, oldCtr)
211229
}
212230

213-
return healthErr
231+
return &ContainerHealthError{
232+
error: finalErr,
233+
ServiceName: o.Spec.Name,
234+
ContainerID: resp.ID,
235+
MachineName: o.MachineName,
236+
}
214237
}
215238
}
216239

‎pkg/client/deploy/operation/predeploy.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ type PreDeployHookError struct {
2424
MachineName string
2525
}
2626

27+
func (e *PreDeployHookError) Unwrap() error {
28+
return e.error
29+
}
30+
2731
// DefaultPreDeployTimeout is the maximum duration to wait for a pre-deploy hook container to complete.
2832
const DefaultPreDeployTimeout = 5 * time.Minute
2933

0 commit comments

Comments
 (0)