Skip to content

Commit 4879e54

Browse files
Oleksandr Shestopaloshe-coupa
authored andcommitted
copy: Add text-based progress output for non-TTY environments
When copying images in non-TTY environments (CI/CD pipelines, redirected output, piped commands), the visual mpb progress bars are discarded, leaving users with no visibility into transfer progress. This makes it difficult to detect stalled transfers or monitor long-running copies. This change adds a nonTTYProgressWriter that consumes progress events from the existing Progress channel and prints periodic aggregate progress lines suitable for log output: Progress: 13.1 MiB / 52.3 MiB Progress: 26.2 MiB / 52.3 MiB Progress: 52.3 MiB / 52.3 MiB The feature is enabled when, output is not a TTY, we check if option.Progress is set, otherwise create a new buffered channel for progress events. Note: Unbuffered channels are replaced with buffered ones to prevent blocking during parallel blob downloads. Callers who need custom consumption should provide a properly buffered channel. Relates-to: containers/skopeo#658 Signed-off-by: Oleksandr Shestopal <ar.shestopal-oshegithub@gmail.com>
1 parent 026c353 commit 4879e54

3 files changed

Lines changed: 276 additions & 1 deletion

File tree

image/copy/copy.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,10 +257,13 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
257257

258258
// If reportWriter is not a TTY (e.g., when piping to a file), do not
259259
// print the progress bars to avoid long and hard to parse output.
260-
// Instead use printCopyInfo() to print single line "Copying ..." messages.
260+
// Instead use text-based aggregate progress via nonTTYProgressWriter.
261261
progressOutput := reportWriter
262262
if !isTTY(reportWriter) {
263263
progressOutput = io.Discard
264+
265+
cleanupProgress := setupNonTTYProgressWriter(reportWriter, options)
266+
defer cleanupProgress()
264267
}
265268

266269
c := &copier{

image/copy/progress_nontty.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package copy
2+
3+
import (
4+
"fmt"
5+
"io"
6+
"time"
7+
8+
"go.podman.io/image/v5/types"
9+
"github.com/vbauerster/mpb/v8/decor"
10+
)
11+
12+
const (
13+
// nonTTYProgressChannelSize is the buffer size for the progress channel
14+
// in non-TTY mode. Buffered to prevent blocking during parallel downloads.
15+
nonTTYProgressChannelSize = 10
16+
17+
// nonTTYProgressInterval is how often aggregate progress is printed
18+
// in non-TTY mode.
19+
nonTTYProgressInterval = 500 * time.Millisecond
20+
)
21+
22+
// nonTTYProgressWriter consumes ProgressProperties from a channel and writes
23+
// aggregate text-based progress output suitable for non-TTY environments.
24+
// No mutex needed - single goroutine processes events sequentially from channel.
25+
type nonTTYProgressWriter struct {
26+
output io.Writer
27+
28+
// Aggregate tracking (no per-blob state needed)
29+
totalSize int64 // Sum of all known blob sizes
30+
downloaded int64 // Total bytes downloaded (accumulated from OffsetUpdate)
31+
32+
// Output throttling
33+
lastOutput time.Time
34+
outputInterval time.Duration
35+
}
36+
37+
// newNonTTYProgressWriter creates a writer that outputs aggregate download
38+
// progress as simple text lines, suitable for non-TTY environments like
39+
// CI/CD pipelines or redirected output.
40+
func newNonTTYProgressWriter(output io.Writer, interval time.Duration) *nonTTYProgressWriter {
41+
return &nonTTYProgressWriter{
42+
output: output,
43+
outputInterval: interval,
44+
}
45+
}
46+
47+
// setupNonTTYProgressWriter configures text-based progress output for non-TTY
48+
// environments unless the caller already provided a buffered Progress channel.
49+
// Returns a cleanup function that must be deferred by the caller.
50+
func setupNonTTYProgressWriter(reportWriter io.Writer, options *Options) func() {
51+
if options.Progress != nil && cap(options.Progress) > 0 {
52+
return func() {}
53+
}
54+
55+
// Use user's interval if greater than our default, otherwise use default.
56+
// This allows users to slow down output while maintaining a sensible minimum.
57+
interval := max(options.ProgressInterval, nonTTYProgressInterval)
58+
if options.ProgressInterval <= 0 {
59+
options.ProgressInterval = nonTTYProgressInterval
60+
}
61+
62+
progressChan := make(chan types.ProgressProperties, nonTTYProgressChannelSize)
63+
options.Progress = progressChan
64+
65+
pw := newNonTTYProgressWriter(reportWriter, interval)
66+
go pw.Run(progressChan)
67+
68+
return func() { close(progressChan) }
69+
}
70+
71+
// Run consumes progress events from the channel and prints throttled
72+
// aggregate progress. Blocks until the channel is closed. Intended to
73+
// be called as a goroutine: go tw.Run(progressChan)
74+
func (w *nonTTYProgressWriter) Run(ch <-chan types.ProgressProperties) {
75+
for props := range ch {
76+
switch props.Event {
77+
case types.ProgressEventNewArtifact:
78+
// New blob starting - add its size to total
79+
w.totalSize += props.Artifact.Size
80+
81+
case types.ProgressEventRead:
82+
// Bytes downloaded - accumulate and maybe print
83+
w.downloaded += int64(props.OffsetUpdate)
84+
if time.Since(w.lastOutput) > w.outputInterval {
85+
fmt.Fprintf(w.output, "Progress: %.1f / %.1f\n",
86+
decor.SizeB1024(w.downloaded), decor.SizeB1024(w.totalSize))
87+
w.lastOutput = time.Now()
88+
}
89+
}
90+
}
91+
}

image/copy/progress_nontty_test.go

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
package copy
2+
3+
import (
4+
"bytes"
5+
"strings"
6+
"testing"
7+
"time"
8+
9+
"go.podman.io/image/v5/types"
10+
"github.com/stretchr/testify/assert"
11+
)
12+
13+
func TestNonTTYProgressWriterRun(t *testing.T) {
14+
tests := []struct {
15+
name string
16+
interval time.Duration
17+
events []types.ProgressProperties
18+
wantTotalSize int64
19+
wantDownloaded int64
20+
wantLines int
21+
wantContains string
22+
}{
23+
{
24+
name: "new artifacts only",
25+
interval: time.Nanosecond,
26+
events: []types.ProgressProperties{
27+
{Event: types.ProgressEventNewArtifact, Artifact: types.BlobInfo{Size: 1024}},
28+
{Event: types.ProgressEventNewArtifact, Artifact: types.BlobInfo{Size: 2048}},
29+
},
30+
wantTotalSize: 3072,
31+
wantDownloaded: 0,
32+
wantLines: 0,
33+
},
34+
{
35+
name: "read events produce output",
36+
interval: time.Nanosecond,
37+
events: []types.ProgressProperties{
38+
{Event: types.ProgressEventNewArtifact, Artifact: types.BlobInfo{Size: 10240}},
39+
{Event: types.ProgressEventRead, OffsetUpdate: 5120},
40+
},
41+
wantTotalSize: 10240,
42+
wantDownloaded: 5120,
43+
wantLines: 1,
44+
wantContains: "Progress:",
45+
},
46+
{
47+
name: "throttling limits output",
48+
interval: 5 * time.Second,
49+
events: []types.ProgressProperties{
50+
{Event: types.ProgressEventNewArtifact, Artifact: types.BlobInfo{Size: 10240}},
51+
{Event: types.ProgressEventRead, OffsetUpdate: 1024},
52+
{Event: types.ProgressEventRead, OffsetUpdate: 1024},
53+
{Event: types.ProgressEventRead, OffsetUpdate: 1024},
54+
},
55+
wantTotalSize: 10240,
56+
wantDownloaded: 3072,
57+
wantLines: 1,
58+
},
59+
{
60+
name: "unknown events ignored",
61+
interval: time.Nanosecond,
62+
events: []types.ProgressProperties{
63+
{Event: types.ProgressEventDone},
64+
},
65+
wantTotalSize: 0,
66+
wantDownloaded: 0,
67+
wantLines: 0,
68+
},
69+
}
70+
71+
for _, tt := range tests {
72+
t.Run(tt.name, func(t *testing.T) {
73+
var buf bytes.Buffer
74+
pw := newNonTTYProgressWriter(&buf, tt.interval)
75+
76+
ch := make(chan types.ProgressProperties, len(tt.events))
77+
for _, e := range tt.events {
78+
ch <- e
79+
}
80+
close(ch)
81+
82+
pw.Run(ch)
83+
84+
assert.Equal(t, tt.wantTotalSize, pw.totalSize)
85+
assert.Equal(t, tt.wantDownloaded, pw.downloaded)
86+
87+
output := buf.String()
88+
if tt.wantLines == 0 {
89+
assert.Empty(t, output)
90+
} else {
91+
lines := strings.Split(strings.TrimSpace(output), "\n")
92+
assert.Equal(t, tt.wantLines, len(lines))
93+
}
94+
if tt.wantContains != "" {
95+
assert.Contains(t, output, tt.wantContains)
96+
}
97+
})
98+
}
99+
}
100+
101+
func TestSetupNonTTYProgressWriter(t *testing.T) {
102+
tests := []struct {
103+
name string
104+
progress chan types.ProgressProperties
105+
progressInterval time.Duration
106+
wantProgressSet bool
107+
wantIntervalSet bool
108+
wantMinInterval time.Duration
109+
}{
110+
{
111+
name: "nil channel gets default setup",
112+
progress: nil,
113+
progressInterval: 0,
114+
wantProgressSet: true,
115+
wantIntervalSet: true,
116+
wantMinInterval: nonTTYProgressInterval,
117+
},
118+
{
119+
name: "unbuffered channel gets replaced",
120+
progress: make(chan types.ProgressProperties),
121+
progressInterval: 0,
122+
wantProgressSet: true,
123+
wantIntervalSet: true,
124+
wantMinInterval: nonTTYProgressInterval,
125+
},
126+
{
127+
name: "buffered channel is kept",
128+
progress: make(chan types.ProgressProperties, 5),
129+
progressInterval: 0,
130+
wantProgressSet: false,
131+
wantIntervalSet: false,
132+
},
133+
{
134+
name: "caller interval larger than default is respected",
135+
progress: nil,
136+
progressInterval: 2 * time.Second,
137+
wantProgressSet: true,
138+
wantIntervalSet: false,
139+
wantMinInterval: 2 * time.Second,
140+
},
141+
{
142+
name: "caller interval smaller than default is kept",
143+
progress: nil,
144+
progressInterval: 100 * time.Millisecond,
145+
wantProgressSet: true,
146+
wantIntervalSet: false,
147+
},
148+
}
149+
150+
for _, tt := range tests {
151+
t.Run(tt.name, func(t *testing.T) {
152+
var buf bytes.Buffer
153+
opts := &Options{
154+
Progress: tt.progress,
155+
ProgressInterval: tt.progressInterval,
156+
}
157+
originalProgress := opts.Progress
158+
159+
cleanup := setupNonTTYProgressWriter(&buf, opts)
160+
defer cleanup()
161+
162+
if tt.wantProgressSet {
163+
assert.NotNil(t, opts.Progress)
164+
assert.Greater(t, cap(opts.Progress), 0)
165+
if originalProgress != nil {
166+
assert.NotEqual(t, originalProgress, opts.Progress)
167+
}
168+
} else {
169+
assert.Equal(t, originalProgress, opts.Progress)
170+
}
171+
172+
if tt.wantIntervalSet {
173+
assert.Equal(t, nonTTYProgressInterval, opts.ProgressInterval)
174+
}
175+
176+
if tt.wantMinInterval > 0 {
177+
assert.GreaterOrEqual(t, opts.ProgressInterval, tt.wantMinInterval)
178+
}
179+
})
180+
}
181+
}

0 commit comments

Comments
 (0)