Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor blob copying and compression #1579

Merged
merged 35 commits into from
Jun 16, 2022
Merged
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
84115a5
Move imageCopier.copyBlobFromStream into a new copy/blob.go
mtrmac Jun 15, 2022
345b1c0
Move errorAnnotationReader
mtrmac Jun 15, 2022
52c06a9
Rename the srcStream parameter to srcReader
mtrmac Jun 15, 2022
54c4a9e
Introduce sourceStream, use it for the first pipeline stage
mtrmac Jun 15, 2022
acdd036
Use stream.reader instead of destStream
mtrmac Jun 15, 2022
2ee1587
Eliminate inputInfo
mtrmac Jun 15, 2022
11c0fe0
Move the OCI blob decryption pipeline step into copier.blobPipelineDe…
mtrmac Jun 15, 2022
338d64d
Beautify blobPipelineDecryptionStep a bit
mtrmac Jun 15, 2022
5472fa9
Rename newDesc to desc
mtrmac Jun 15, 2022
651679a
Beautify the DecryptLayer usage a bit
mtrmac Jun 15, 2022
8dbbe7c
Move the OCI blob encryption pipeline step into copier.blobPipelineEn…
mtrmac Jun 15, 2022
014c23d
Simplify blobPipelineEncryptionStep
mtrmac Jun 15, 2022
e8c3881
Exit from updateCryptoOperationAndAnnotations early if not encrypting
mtrmac Jun 15, 2022
d6fccbd
Beautify blobPipelineEncryptionStep
mtrmac Jun 15, 2022
0f854d0
Rename copy/encrypt.go to copy/encryption.go
mtrmac Jun 15, 2022
f0b0bf4
Move the compression detection step into blobPipelineDetectCompressio…
mtrmac Jun 15, 2022
bf130bc
Beautify blobPipelineDetectCompressionStep
mtrmac Jun 15, 2022
003d340
Move the compression annotation update closer to the other compressio…
mtrmac Jun 15, 2022
6af0c82
Move copier.compressGoroutine to compression.go
mtrmac Jun 15, 2022
e83976e
Move the compression/decompression step to blobPipelineCompressionStep
mtrmac Jun 15, 2022
00a66fe
Rename detectedCompession to detected
mtrmac Jun 15, 2022
23a031f
Rename compresisonOperation to operation
mtrmac Jun 15, 2022
78fd34d
Rename uploadCompressionFormat to uploadedAlgorithm
mtrmac Jun 15, 2022
ce4d066
Rename uploadCompressorName to uploadedCompressorName
mtrmac Jun 15, 2022
68e0d9d
Rename compressionMetadata to uploadedAnnotations.
mtrmac Jun 15, 2022
4f0548a
Compute srcCompressorName already in blobPipelineDetectCompressionStep
mtrmac Jun 15, 2022
f93ebce
Return a fresh &bpCompressionStepData in each case
mtrmac Jun 15, 2022
7a81dce
Move uploadedAnnotations inside the individual cases.
mtrmac Jun 15, 2022
302967d
Move uploadedAlgorithm inside the individual cases.
mtrmac Jun 15, 2022
c39fe5b
Clean up the control flow of blobPipelineCompressionStep
mtrmac Jun 15, 2022
b6bb69b
Rename s to decompressed in the re-compress case
mtrmac Jun 15, 2022
f70a390
Factor out copier.compressedStream from copier.blobPipelineCompressio…
mtrmac Jun 15, 2022
73bf1ae
Split blobPipelineCompressionStep into individual functions
mtrmac Jun 15, 2022
20f0638
Eliminate the closers array in bpcRecompressCompressed
mtrmac Jun 15, 2022
2756d70
Use a loop for the alternatives in blobPipelineCompressionStep
mtrmac Jun 15, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Rename detectedCompession to detected
The compression meaning is clear in context here.

Should not change behavior.

Signed-off-by: Miloslav Trmač <[email protected]>
  • Loading branch information
mtrmac committed Jun 15, 2022
commit 00a66fee8c0f901b7ea2d89bd039334f9e4197fe
22 changes: 11 additions & 11 deletions copy/compression.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,15 @@ type bpCompressionStepData struct {
// Returns data for other steps; the caller should eventually call updateCompressionEdits and perhaps recordValidatedBlobData,
// and must eventually call close.
func (c *copier) blobPipelineCompressionStep(stream *sourceStream, canModifyBlob bool,
detectedCompression bpDetectCompressionStepData) (*bpCompressionStepData, error) {
detected bpDetectCompressionStepData) (*bpCompressionStepData, error) {
// WARNING: If you are adding new reasons to change the blob, update also the OptimizeDestinationImageAlreadyExists
// short-circuit conditions
compressionMetadata := map[string]string{}
var compressionOperation types.LayerCompression
var uploadCompressionFormat *compressiontypes.Algorithm
srcCompressorName := internalblobinfocache.Uncompressed
if detectedCompression.isCompressed {
srcCompressorName = detectedCompression.format.Name()
if detected.isCompressed {
srcCompressorName = detected.format.Name()
}
var uploadCompressorName string
var closers []io.Closer
Expand All @@ -82,7 +82,7 @@ func (c *copier) blobPipelineCompressionStep(stream *sourceStream, canModifyBlob
srcCompressorName = internalblobinfocache.UnknownCompression
uploadCompressionFormat = nil
uploadCompressorName = internalblobinfocache.UnknownCompression
} else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Compress && !detectedCompression.isCompressed {
} else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Compress && !detected.isCompressed {
logrus.Debugf("Compressing blob on the fly")
compressionOperation = types.Compress
pipeReader, pipeWriter := io.Pipe()
Expand All @@ -103,14 +103,14 @@ func (c *copier) blobPipelineCompressionStep(stream *sourceStream, canModifyBlob
Size: -1,
}
uploadCompressorName = uploadCompressionFormat.Name()
} else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Compress && detectedCompression.isCompressed &&
c.compressionFormat != nil && c.compressionFormat.Name() != detectedCompression.format.Name() {
} else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Compress && detected.isCompressed &&
c.compressionFormat != nil && c.compressionFormat.Name() != detected.format.Name() {
// When the blob is compressed, but the desired format is different, it first needs to be decompressed and finally
// re-compressed using the desired format.
logrus.Debugf("Blob will be converted")

compressionOperation = types.PreserveOriginal
s, err := detectedCompression.decompressor(stream.reader)
s, err := detected.decompressor(stream.reader)
if err != nil {
return nil, err
}
Expand All @@ -128,10 +128,10 @@ func (c *copier) blobPipelineCompressionStep(stream *sourceStream, canModifyBlob
Size: -1,
}
uploadCompressorName = uploadCompressionFormat.Name()
} else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Decompress && detectedCompression.isCompressed {
} else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Decompress && detected.isCompressed {
logrus.Debugf("Blob will be decompressed")
compressionOperation = types.Decompress
s, err := detectedCompression.decompressor(stream.reader)
s, err := detected.decompressor(stream.reader)
if err != nil {
return nil, err
}
Expand All @@ -151,8 +151,8 @@ func (c *copier) blobPipelineCompressionStep(stream *sourceStream, canModifyBlob
// LayerInfosForCopy() returned something that differs from what was in the
// source's manifest, and UpdatedImage() needs to call UpdateLayerInfos(),
// it will be able to correctly derive the MediaType for the copied blob.
if detectedCompression.isCompressed {
uploadCompressionFormat = &detectedCompression.format
if detected.isCompressed {
uploadCompressionFormat = &detected.format
} else {
uploadCompressionFormat = nil
}
Expand Down