Add new reporting interface that can be used to create progress tracker

go.mod

@@ -8,10 +8,13 @@ require (
 	github.com/chrismellard/docker-credential-acr-env v0.0.0-20220327082430-c57b701bfc08
 	github.com/cppforlife/cobrautil v0.0.0-20221021151949-d60711905d65
 	github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14
+	github.com/docker/go-units v0.5.0
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/google/go-containerregistry v0.20.3
 	github.com/mattn/go-isatty v0.0.20
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2
+	github.com/moby/term v0.0.0-20221205130635-1aeaba878587
+	github.com/morikuni/aec v1.0.0
 	github.com/spf13/cobra v1.9.1
 	github.com/stretchr/testify v1.10.0
 	golang.org/x/sync v0.12.0
@@ -22,6 +25,7 @@ require (
 require (
 	cloud.google.com/go v0.99.0 // indirect
 	github.com/Azure/azure-sdk-for-go v55.0.0+incompatible // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.18 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
@@ -48,7 +52,6 @@ require (
 	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
 	github.com/cppforlife/color v1.9.1-0.20200716202919-6706ac40b835 // indirect
-	github.com/creack/pty v1.1.11 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.0 // indirect
 	github.com/docker/cli v27.5.0+incompatible // indirect

go.sum

@@ -47,6 +47,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go v55.0.0+incompatible h1:L4/vUGbg1Xkw5L20LZD+hJI5I+ibWSytqQ68lTCfLwY=
 github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.6/go.mod h1:V6p3pKZx1KKkJubbxnDWrzNhEIfOy/pTGasLqzHIPHs=
@@ -130,8 +132,8 @@ github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14 h1:MjRdR01xh0
 github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14/go.mod h1:AlgTssDlstr4mf92TR4DPITLfl5+7wEY4cKStCmeeto=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
-github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
-github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -144,6 +146,8 @@ github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBi
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=
 github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -271,6 +275,10 @@ github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlz
 github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=
+github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

pkg/imgpkg/cmd/copy.go

@@ -119,7 +119,7 @@ func (c *CopyOptions) Run() error {
 	}
 
 	imageSet := ctlimgset.NewImageSet(c.Concurrency, levelLogger, tagGen)
-	tarImageSet := ctlimgset.NewTarImageSet(imageSet, c.Concurrency, levelLogger)
+	tarImageSet := ctlimgset.NewTarImageSet(imageSet, c.Concurrency, levelLogger, &v1.NoopProgressReporter{})
 
 	var signatureRetriever v1.SignatureFetcher
 	if c.SignatureFlags.CopyCosignSignatures {

pkg/imgpkg/imageset/tar_image_set.go

@@ -18,14 +18,15 @@ import (
 )
 
 type TarImageSet struct {
-	imageSet    ImageSet
-	concurrency int
-	logger      Logger
+	imageSet         ImageSet
+	concurrency      int
+	logger           Logger
+	progressReporter imagetar.ProgressReporter
 }
 
 // NewTarImageSet provides export/import operations on a tarball for a set of images
-func NewTarImageSet(imageSet ImageSet, concurrency int, logger Logger) TarImageSet {
-	return TarImageSet{imageSet, concurrency, logger}
+func NewTarImageSet(imageSet ImageSet, concurrency int, logger Logger, report imagetar.ProgressReporter) TarImageSet {
+	return TarImageSet{imageSet: imageSet, concurrency: concurrency, logger: logger, progressReporter: report}
 }
 
 // Export Creates a Tar with the provided Images
@@ -105,7 +106,7 @@ func (i *TarImageSet) Export(foundImages *UnprocessedImageRefs, outputPath strin
 
 	opts := imagetar.TarWriterOpts{Concurrency: i.concurrency}
 
-	err = imagetar.NewTarWriter(ids, outputFileOpener, opts, i.logger, imageLayerWriterCheck, alreadyDownloadedLayers).Write()
+	err = imagetar.NewTarWriter(ids, outputFileOpener, opts, i.logger, imageLayerWriterCheck, alreadyDownloadedLayers, i.progressReporter).Write()
 	return ids, err
 }
 

pkg/imgpkg/imagetar/tar_writer.go

@@ -23,6 +23,14 @@ type Logger interface {
 	Logf(str string, args ...interface{})
 }
 
+// ProgressReporter used to report the current status of the read/write
+type ProgressReporter interface {
+	StartReporting(id string, total int64) error
+	Report(id string, completed int64, total int64, err error) error
+	Finish(id string, total int64) error
+	ActiveReporter() bool
+}
+
 type TarWriterOpts struct {
 	Concurrency int
 }
@@ -31,9 +39,10 @@ type TarWriter struct {
 	ids       *imagedesc.ImageRefDescriptors
 	dstOpener func() (io.WriteCloser, error)
 
-	dst           io.WriteCloser
-	tf            *tar.Writer
-	layersToWrite []imagedesc.ImageLayerDescriptor
+	dst              io.WriteCloser
+	tf               *tar.Writer
+	layersToWrite    []imagedesc.ImageLayerDescriptor
+	progressReporter ProgressReporter
 
 	opts                  TarWriterOpts
 	logger                Logger
@@ -44,7 +53,7 @@ type TarWriter struct {
 // NewTarWriter constructor returning a mechanism to write image refs / layers to a tarball on disk.
 func NewTarWriter(ids *imagedesc.ImageRefDescriptors, dstOpener func() (io.WriteCloser, error),
 	opts TarWriterOpts, logger Logger, imageLayerWriterCheck ImageLayerWriterFilter,
-	layersFromOtherSource []regv1.Layer) *TarWriter {
+	layersFromOtherSource []regv1.Layer, reporter ProgressReporter) *TarWriter {
 	knownlayers := map[string]regv1.Layer{}
 	for _, layer := range layersFromOtherSource {
 		d, err := layer.Digest()
@@ -53,13 +62,17 @@ func NewTarWriter(ids *imagedesc.ImageRefDescriptors, dstOpener func() (io.Write
 		}
 		knownlayers[d.String()] = layer
 	}
+	if reporter == nil {
+		panic(fmt.Sprintf("Internal inconsistency: A Progress Reported need to be provided"))
+	}
 	return &TarWriter{
 		ids:                   ids,
 		dstOpener:             dstOpener,
 		opts:                  opts,
 		logger:                logger,
 		imageLayerWriterCheck: imageLayerWriterCheck,
 		layersFromOtherSource: knownlayers,
+		progressReporter:      reporter,
 	}
 }
 
@@ -138,6 +151,29 @@ func (w *TarWriter) writeImage(td imagedesc.ImageDescriptor) error {
 	return nil
 }
 
+type readerWithProgressLogger struct {
+	reader           io.Reader
+	progressReporter ProgressReporter
+	layerID          string
+	gotSize          int64
+	totalSize        int64
+}
+
+func (r *readerWithProgressLogger) Read(p []byte) (n int, err error) {
+	readSize, err := r.reader.Read(p)
+	r.gotSize += int64(readSize)
+	if err != nil {
+		if err == io.EOF {
+			r.progressReporter.Finish(r.layerID, r.totalSize)
+			return readSize, err
+		}
+		r.progressReporter.Report(r.layerID, r.gotSize, r.totalSize, err)
+		return 0, err
+	}
+	r.progressReporter.Report(r.layerID, r.gotSize, r.totalSize, err)
+	return readSize, nil
+}
+
 type writtenLayer struct {
 	Name   string
 	Offset int64
@@ -188,10 +224,19 @@ func (w *TarWriter) writeLayers() error {
 			stream = nil
 		} else {
 			if sourceLayer, ok := w.layersFromOtherSource[digest.String()]; ok {
-				stream, err = sourceLayer.Compressed()
+				rLayer, err := sourceLayer.Compressed()
 				if err != nil {
 					return fmt.Errorf("failed to get compressed stuff: %s", err)
 				}
+				w.progressReporter.StartReporting(digest.Hex, imgLayer.Size)
+				progressReader := &readerWithProgressLogger{
+					reader:           rLayer,
+					progressReporter: w.progressReporter,
+					layerID:          digest.Hex,
+					gotSize:          0,
+					totalSize:        imgLayer.Size,
+				}
+				stream = progressReader
 			}
 
 			if stream == nil {
@@ -201,10 +246,19 @@ func (w *TarWriter) writeLayers() error {
 					return err
 				}
 
-				stream, err = foundLayer.Open()
+				rLayer, err := foundLayer.Open()
 				if err != nil {
 					return err
 				}
+				w.progressReporter.StartReporting(digest.Hex, imgLayer.Size)
+				progressReader := &readerWithProgressLogger{
+					reader:           rLayer,
+					progressReporter: w.progressReporter,
+					layerID:          digest.Hex,
+					gotSize:          0,
+					totalSize:        imgLayer.Size,
+				}
+				stream = progressReader
 			} else {
 				w.logger.Debugf("reusing layer: %s", digest.String())
 			}
@@ -317,7 +371,16 @@ func (w *TarWriter) fillInLayer(wl writtenLayer) error {
 	}
 	w.logger.Tracef("took %s to prepare layer %s to be written", time.Since(startFillingLayer), wl.Layer.Digest)
 
-	err = w.writeTarEntry(tw, wl.Name, stream, wl.Layer.Size)
+	w.progressReporter.StartReporting(wl.Layer.Digest, wl.Layer.Size)
+	progressReader := &readerWithProgressLogger{
+		reader:           stream,
+		progressReporter: w.progressReporter,
+		layerID:          wl.Layer.Digest,
+		gotSize:          0,
+		totalSize:        wl.Layer.Size,
+	}
+
+	err = w.writeTarEntry(tw, wl.Name, progressReader, wl.Layer.Size)
 	if err != nil {
 		return fmt.Errorf("Rewriting tar entry (%s): %s", wl.Name, err)
 	}
@@ -353,7 +416,9 @@ func (w *TarWriter) writeTarEntry(tw *tar.Writer, path string, r io.Reader, size
 	}
 
 	if !zerosFill {
-		w.logger.Logf("done: file '%s' (%s)\n", path, time.Since(t1))
+		if !w.progressReporter.ActiveReporter() {
+			w.logger.Logf("done: file '%s' (%s)\n", path, time.Since(t1))
+		}
 	}
 
 	return nil

pkg/imgpkg/v1/copy.go

@@ -28,6 +28,7 @@ type CopyOpts struct {
 	SignatureRetriever      SignatureFetcher
 	IncludeNonDistributable bool
 	Resume                  bool
+	ProgressTracker         ProgressTracker
 }
 
 // CopyOrigin abstracts the original location to copy from
@@ -47,6 +48,9 @@ func CopyToTar(origin CopyOrigin, outputTarPath string, opts CopyOpts, reg regis
 		return nil, err
 	}
 
+	if opts.ProgressTracker != nil {
+		opts.ProgressTracker.StartDisplay()
+	}
 	opts.Logger.Tracef("Exporting images to tar\n")
 	ids, err := opts.TarImageSet.Export(unprocessedImageRefs, outputTarPath, reg, imagetar.NewImageLayerWriterCheck(opts.IncludeNonDistributable), opts.Resume)
 	if err != nil {

pkg/imgpkg/v1/copy_test.go

@@ -40,11 +40,12 @@ func testSetup(registryBuild *helpers.FakeTestRegistryBuilder, imageName string,
 	uiLogger := util.NewUILevelLogger(util.LogWarn, util.NewBufferLogger(stdOut))
 	tagGen := ctlimg.DefaultTagGenerator{}
 	imageSet := imageset.NewImageSet(1, uiLogger, tagGen)
+	noopProgress := v1.NoopProgressReporter{}
 
 	opts := v1.CopyOpts{
 		Logger:             uiLogger,
 		ImageSet:           imageSet,
-		TarImageSet:        imageset.NewTarImageSet(imageSet, 1, uiLogger),
+		TarImageSet:        imageset.NewTarImageSet(imageSet, 1, uiLogger, &noopProgress),
 		Concurrency:        1,
 		SignatureRetriever: &fakeSignatureRetriever{},
 		Resume:             false,