vendor: update buildkit to v0.8
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
2
vendor/github.com/docker/docker/pkg/system/rm.go
generated
vendored
2
vendor/github.com/docker/docker/pkg/system/rm.go
generated
vendored
@@ -1,3 +1,5 @@
|
||||
// +build !darwin,!windows
|
||||
|
||||
package system // import "github.com/docker/docker/pkg/system"
|
||||
|
||||
import (
|
||||
|
||||
6
vendor/github.com/docker/docker/pkg/system/rm_windows.go
generated
vendored
Normal file
6
vendor/github.com/docker/docker/pkg/system/rm_windows.go
generated
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
package system
|
||||
|
||||
import "os"
|
||||
|
||||
// EnsureRemoveAll is an alias to os.RemoveAll on Windows
|
||||
var EnsureRemoveAll = os.RemoveAll
|
||||
@@ -1,3 +1,5 @@
|
||||
// +build freebsd netbsd
|
||||
|
||||
package system // import "github.com/docker/docker/pkg/system"
|
||||
|
||||
import "syscall"
|
||||
13
vendor/github.com/docker/docker/pkg/system/stat_solaris.go
generated
vendored
Normal file
13
vendor/github.com/docker/docker/pkg/system/stat_solaris.go
generated
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
package system // import "github.com/docker/docker/pkg/system"
|
||||
|
||||
import "syscall"
|
||||
|
||||
// fromStatT converts a syscall.Stat_t type to a system.Stat_t type
|
||||
func fromStatT(s *syscall.Stat_t) (*StatT, error) {
|
||||
return &StatT{size: s.Size,
|
||||
mode: s.Mode,
|
||||
uid: s.Uid,
|
||||
gid: s.Gid,
|
||||
rdev: s.Rdev,
|
||||
mtim: s.Mtim}, nil
|
||||
}
|
||||
21
vendor/github.com/docker/docker/pkg/tarsum/builder_context.go
generated
vendored
21
vendor/github.com/docker/docker/pkg/tarsum/builder_context.go
generated
vendored
@@ -1,21 +0,0 @@
|
||||
package tarsum // import "github.com/docker/docker/pkg/tarsum"
|
||||
|
||||
// BuilderContext is an interface extending TarSum by adding the Remove method.
|
||||
// In general there was concern about adding this method to TarSum itself
|
||||
// so instead it is being added just to "BuilderContext" which will then
|
||||
// only be used during the .dockerignore file processing
|
||||
// - see builder/evaluator.go
|
||||
type BuilderContext interface {
|
||||
TarSum
|
||||
Remove(string)
|
||||
}
|
||||
|
||||
func (bc *tarSum) Remove(filename string) {
|
||||
for i, fis := range bc.sums {
|
||||
if fis.Name() == filename {
|
||||
bc.sums = append(bc.sums[:i], bc.sums[i+1:]...)
|
||||
// Note, we don't just return because there could be
|
||||
// more than one with this name
|
||||
}
|
||||
}
|
||||
}
|
||||
133
vendor/github.com/docker/docker/pkg/tarsum/fileinfosums.go
generated
vendored
133
vendor/github.com/docker/docker/pkg/tarsum/fileinfosums.go
generated
vendored
@@ -1,133 +0,0 @@
|
||||
package tarsum // import "github.com/docker/docker/pkg/tarsum"
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FileInfoSumInterface provides an interface for accessing file checksum
|
||||
// information within a tar file. This info is accessed through interface
|
||||
// so the actual name and sum cannot be melded with.
|
||||
type FileInfoSumInterface interface {
|
||||
// File name
|
||||
Name() string
|
||||
// Checksum of this particular file and its headers
|
||||
Sum() string
|
||||
// Position of file in the tar
|
||||
Pos() int64
|
||||
}
|
||||
|
||||
type fileInfoSum struct {
|
||||
name string
|
||||
sum string
|
||||
pos int64
|
||||
}
|
||||
|
||||
func (fis fileInfoSum) Name() string {
|
||||
return fis.name
|
||||
}
|
||||
func (fis fileInfoSum) Sum() string {
|
||||
return fis.sum
|
||||
}
|
||||
func (fis fileInfoSum) Pos() int64 {
|
||||
return fis.pos
|
||||
}
|
||||
|
||||
// FileInfoSums provides a list of FileInfoSumInterfaces.
|
||||
type FileInfoSums []FileInfoSumInterface
|
||||
|
||||
// GetFile returns the first FileInfoSumInterface with a matching name.
|
||||
func (fis FileInfoSums) GetFile(name string) FileInfoSumInterface {
|
||||
// We do case insensitive matching on Windows as c:\APP and c:\app are
|
||||
// the same. See issue #33107.
|
||||
for i := range fis {
|
||||
if (runtime.GOOS == "windows" && strings.EqualFold(fis[i].Name(), name)) ||
|
||||
(runtime.GOOS != "windows" && fis[i].Name() == name) {
|
||||
return fis[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAllFile returns a FileInfoSums with all matching names.
|
||||
func (fis FileInfoSums) GetAllFile(name string) FileInfoSums {
|
||||
f := FileInfoSums{}
|
||||
for i := range fis {
|
||||
if fis[i].Name() == name {
|
||||
f = append(f, fis[i])
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// GetDuplicatePaths returns a FileInfoSums with all duplicated paths.
|
||||
func (fis FileInfoSums) GetDuplicatePaths() (dups FileInfoSums) {
|
||||
seen := make(map[string]int, len(fis)) // allocate earl. no need to grow this map.
|
||||
for i := range fis {
|
||||
f := fis[i]
|
||||
if _, ok := seen[f.Name()]; ok {
|
||||
dups = append(dups, f)
|
||||
} else {
|
||||
seen[f.Name()] = 0
|
||||
}
|
||||
}
|
||||
return dups
|
||||
}
|
||||
|
||||
// Len returns the size of the FileInfoSums.
|
||||
func (fis FileInfoSums) Len() int { return len(fis) }
|
||||
|
||||
// Swap swaps two FileInfoSum values if a FileInfoSums list.
|
||||
func (fis FileInfoSums) Swap(i, j int) { fis[i], fis[j] = fis[j], fis[i] }
|
||||
|
||||
// SortByPos sorts FileInfoSums content by position.
|
||||
func (fis FileInfoSums) SortByPos() {
|
||||
sort.Sort(byPos{fis})
|
||||
}
|
||||
|
||||
// SortByNames sorts FileInfoSums content by name.
|
||||
func (fis FileInfoSums) SortByNames() {
|
||||
sort.Sort(byName{fis})
|
||||
}
|
||||
|
||||
// SortBySums sorts FileInfoSums content by sums.
|
||||
func (fis FileInfoSums) SortBySums() {
|
||||
dups := fis.GetDuplicatePaths()
|
||||
if len(dups) > 0 {
|
||||
sort.Sort(bySum{fis, dups})
|
||||
} else {
|
||||
sort.Sort(bySum{fis, nil})
|
||||
}
|
||||
}
|
||||
|
||||
// byName is a sort.Sort helper for sorting by file names.
|
||||
// If names are the same, order them by their appearance in the tar archive
|
||||
type byName struct{ FileInfoSums }
|
||||
|
||||
func (bn byName) Less(i, j int) bool {
|
||||
if bn.FileInfoSums[i].Name() == bn.FileInfoSums[j].Name() {
|
||||
return bn.FileInfoSums[i].Pos() < bn.FileInfoSums[j].Pos()
|
||||
}
|
||||
return bn.FileInfoSums[i].Name() < bn.FileInfoSums[j].Name()
|
||||
}
|
||||
|
||||
// bySum is a sort.Sort helper for sorting by the sums of all the fileinfos in the tar archive
|
||||
type bySum struct {
|
||||
FileInfoSums
|
||||
dups FileInfoSums
|
||||
}
|
||||
|
||||
func (bs bySum) Less(i, j int) bool {
|
||||
if bs.dups != nil && bs.FileInfoSums[i].Name() == bs.FileInfoSums[j].Name() {
|
||||
return bs.FileInfoSums[i].Pos() < bs.FileInfoSums[j].Pos()
|
||||
}
|
||||
return bs.FileInfoSums[i].Sum() < bs.FileInfoSums[j].Sum()
|
||||
}
|
||||
|
||||
// byPos is a sort.Sort helper for sorting by the sums of all the fileinfos by their original order
|
||||
type byPos struct{ FileInfoSums }
|
||||
|
||||
func (bp byPos) Less(i, j int) bool {
|
||||
return bp.FileInfoSums[i].Pos() < bp.FileInfoSums[j].Pos()
|
||||
}
|
||||
301
vendor/github.com/docker/docker/pkg/tarsum/tarsum.go
generated
vendored
301
vendor/github.com/docker/docker/pkg/tarsum/tarsum.go
generated
vendored
@@ -1,301 +0,0 @@
|
||||
// Package tarsum provides algorithms to perform checksum calculation on
|
||||
// filesystem layers.
|
||||
//
|
||||
// The transportation of filesystems, regarding Docker, is done with tar(1)
|
||||
// archives. There are a variety of tar serialization formats [2], and a key
|
||||
// concern here is ensuring a repeatable checksum given a set of inputs from a
|
||||
// generic tar archive. Types of transportation include distribution to and from a
|
||||
// registry endpoint, saving and loading through commands or Docker daemon APIs,
|
||||
// transferring the build context from client to Docker daemon, and committing the
|
||||
// filesystem of a container to become an image.
|
||||
//
|
||||
// As tar archives are used for transit, but not preserved in many situations, the
|
||||
// focus of the algorithm is to ensure the integrity of the preserved filesystem,
|
||||
// while maintaining a deterministic accountability. This includes neither
|
||||
// constraining the ordering or manipulation of the files during the creation or
|
||||
// unpacking of the archive, nor include additional metadata state about the file
|
||||
// system attributes.
|
||||
package tarsum // import "github.com/docker/docker/pkg/tarsum"
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
buf8K = 8 * 1024
|
||||
buf16K = 16 * 1024
|
||||
buf32K = 32 * 1024
|
||||
)
|
||||
|
||||
// NewTarSum creates a new interface for calculating a fixed time checksum of a
|
||||
// tar archive.
|
||||
//
|
||||
// This is used for calculating checksums of layers of an image, in some cases
|
||||
// including the byte payload of the image's json metadata as well, and for
|
||||
// calculating the checksums for buildcache.
|
||||
func NewTarSum(r io.Reader, dc bool, v Version) (TarSum, error) {
|
||||
return NewTarSumHash(r, dc, v, DefaultTHash)
|
||||
}
|
||||
|
||||
// NewTarSumHash creates a new TarSum, providing a THash to use rather than
|
||||
// the DefaultTHash.
|
||||
func NewTarSumHash(r io.Reader, dc bool, v Version, tHash THash) (TarSum, error) {
|
||||
headerSelector, err := getTarHeaderSelector(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ts := &tarSum{Reader: r, DisableCompression: dc, tarSumVersion: v, headerSelector: headerSelector, tHash: tHash}
|
||||
err = ts.initTarSum()
|
||||
return ts, err
|
||||
}
|
||||
|
||||
// NewTarSumForLabel creates a new TarSum using the provided TarSum version+hash label.
|
||||
func NewTarSumForLabel(r io.Reader, disableCompression bool, label string) (TarSum, error) {
|
||||
parts := strings.SplitN(label, "+", 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, errors.New("tarsum label string should be of the form: {tarsum_version}+{hash_name}")
|
||||
}
|
||||
|
||||
versionName, hashName := parts[0], parts[1]
|
||||
|
||||
version, ok := tarSumVersionsByName[versionName]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown TarSum version name: %q", versionName)
|
||||
}
|
||||
|
||||
hashConfig, ok := standardHashConfigs[hashName]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown TarSum hash name: %q", hashName)
|
||||
}
|
||||
|
||||
tHash := NewTHash(hashConfig.name, hashConfig.hash.New)
|
||||
|
||||
return NewTarSumHash(r, disableCompression, version, tHash)
|
||||
}
|
||||
|
||||
// TarSum is the generic interface for calculating fixed time
|
||||
// checksums of a tar archive.
|
||||
type TarSum interface {
|
||||
io.Reader
|
||||
GetSums() FileInfoSums
|
||||
Sum([]byte) string
|
||||
Version() Version
|
||||
Hash() THash
|
||||
}
|
||||
|
||||
// tarSum struct is the structure for a Version0 checksum calculation.
|
||||
type tarSum struct {
|
||||
io.Reader
|
||||
tarR *tar.Reader
|
||||
tarW *tar.Writer
|
||||
writer writeCloseFlusher
|
||||
bufTar *bytes.Buffer
|
||||
bufWriter *bytes.Buffer
|
||||
bufData []byte
|
||||
h hash.Hash
|
||||
tHash THash
|
||||
sums FileInfoSums
|
||||
fileCounter int64
|
||||
currentFile string
|
||||
finished bool
|
||||
first bool
|
||||
DisableCompression bool // false by default. When false, the output gzip compressed.
|
||||
tarSumVersion Version // this field is not exported so it can not be mutated during use
|
||||
headerSelector tarHeaderSelector // handles selecting and ordering headers for files in the archive
|
||||
}
|
||||
|
||||
func (ts tarSum) Hash() THash {
|
||||
return ts.tHash
|
||||
}
|
||||
|
||||
func (ts tarSum) Version() Version {
|
||||
return ts.tarSumVersion
|
||||
}
|
||||
|
||||
// THash provides a hash.Hash type generator and its name.
|
||||
type THash interface {
|
||||
Hash() hash.Hash
|
||||
Name() string
|
||||
}
|
||||
|
||||
// NewTHash is a convenience method for creating a THash.
|
||||
func NewTHash(name string, h func() hash.Hash) THash {
|
||||
return simpleTHash{n: name, h: h}
|
||||
}
|
||||
|
||||
type tHashConfig struct {
|
||||
name string
|
||||
hash crypto.Hash
|
||||
}
|
||||
|
||||
var (
|
||||
// NOTE: DO NOT include MD5 or SHA1, which are considered insecure.
|
||||
standardHashConfigs = map[string]tHashConfig{
|
||||
"sha256": {name: "sha256", hash: crypto.SHA256},
|
||||
"sha512": {name: "sha512", hash: crypto.SHA512},
|
||||
}
|
||||
)
|
||||
|
||||
// DefaultTHash is default TarSum hashing algorithm - "sha256".
|
||||
var DefaultTHash = NewTHash("sha256", sha256.New)
|
||||
|
||||
type simpleTHash struct {
|
||||
n string
|
||||
h func() hash.Hash
|
||||
}
|
||||
|
||||
func (sth simpleTHash) Name() string { return sth.n }
|
||||
func (sth simpleTHash) Hash() hash.Hash { return sth.h() }
|
||||
|
||||
func (ts *tarSum) encodeHeader(h *tar.Header) error {
|
||||
for _, elem := range ts.headerSelector.selectHeaders(h) {
|
||||
// Ignore these headers to be compatible with versions
|
||||
// before go 1.10
|
||||
if elem[0] == "gname" || elem[0] == "uname" {
|
||||
elem[1] = ""
|
||||
}
|
||||
if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ts *tarSum) initTarSum() error {
|
||||
ts.bufTar = bytes.NewBuffer([]byte{})
|
||||
ts.bufWriter = bytes.NewBuffer([]byte{})
|
||||
ts.tarR = tar.NewReader(ts.Reader)
|
||||
ts.tarW = tar.NewWriter(ts.bufTar)
|
||||
if !ts.DisableCompression {
|
||||
ts.writer = gzip.NewWriter(ts.bufWriter)
|
||||
} else {
|
||||
ts.writer = &nopCloseFlusher{Writer: ts.bufWriter}
|
||||
}
|
||||
if ts.tHash == nil {
|
||||
ts.tHash = DefaultTHash
|
||||
}
|
||||
ts.h = ts.tHash.Hash()
|
||||
ts.h.Reset()
|
||||
ts.first = true
|
||||
ts.sums = FileInfoSums{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ts *tarSum) Read(buf []byte) (int, error) {
|
||||
if ts.finished {
|
||||
return ts.bufWriter.Read(buf)
|
||||
}
|
||||
if len(ts.bufData) < len(buf) {
|
||||
switch {
|
||||
case len(buf) <= buf8K:
|
||||
ts.bufData = make([]byte, buf8K)
|
||||
case len(buf) <= buf16K:
|
||||
ts.bufData = make([]byte, buf16K)
|
||||
case len(buf) <= buf32K:
|
||||
ts.bufData = make([]byte, buf32K)
|
||||
default:
|
||||
ts.bufData = make([]byte, len(buf))
|
||||
}
|
||||
}
|
||||
buf2 := ts.bufData[:len(buf)]
|
||||
|
||||
n, err := ts.tarR.Read(buf2)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
if _, err := ts.h.Write(buf2[:n]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !ts.first {
|
||||
ts.sums = append(ts.sums, fileInfoSum{name: ts.currentFile, sum: hex.EncodeToString(ts.h.Sum(nil)), pos: ts.fileCounter})
|
||||
ts.fileCounter++
|
||||
ts.h.Reset()
|
||||
} else {
|
||||
ts.first = false
|
||||
}
|
||||
|
||||
if _, err := ts.tarW.Write(buf2[:n]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
currentHeader, err := ts.tarR.Next()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
if err := ts.tarW.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := io.Copy(ts.writer, ts.bufTar); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := ts.writer.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ts.finished = true
|
||||
return ts.bufWriter.Read(buf)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
ts.currentFile = path.Join(".", path.Join("/", currentHeader.Name))
|
||||
if err := ts.encodeHeader(currentHeader); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := ts.tarW.WriteHeader(currentHeader); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(ts.writer, ts.bufTar); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ts.writer.Flush()
|
||||
|
||||
return ts.bufWriter.Read(buf)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Filling the hash buffer
|
||||
if _, err = ts.h.Write(buf2[:n]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Filling the tar writer
|
||||
if _, err = ts.tarW.Write(buf2[:n]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Filling the output writer
|
||||
if _, err = io.Copy(ts.writer, ts.bufTar); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ts.writer.Flush()
|
||||
|
||||
return ts.bufWriter.Read(buf)
|
||||
}
|
||||
|
||||
func (ts *tarSum) Sum(extra []byte) string {
|
||||
ts.sums.SortBySums()
|
||||
h := ts.tHash.Hash()
|
||||
if extra != nil {
|
||||
h.Write(extra)
|
||||
}
|
||||
for _, fis := range ts.sums {
|
||||
h.Write([]byte(fis.Sum()))
|
||||
}
|
||||
checksum := ts.Version().String() + "+" + ts.tHash.Name() + ":" + hex.EncodeToString(h.Sum(nil))
|
||||
return checksum
|
||||
}
|
||||
|
||||
func (ts *tarSum) GetSums() FileInfoSums {
|
||||
return ts.sums
|
||||
}
|
||||
230
vendor/github.com/docker/docker/pkg/tarsum/tarsum_spec.md
generated
vendored
230
vendor/github.com/docker/docker/pkg/tarsum/tarsum_spec.md
generated
vendored
@@ -1,230 +0,0 @@
|
||||
page_title: TarSum checksum specification
|
||||
page_description: Documentation for algorithms used in the TarSum checksum calculation
|
||||
page_keywords: docker, checksum, validation, tarsum
|
||||
|
||||
# TarSum Checksum Specification
|
||||
|
||||
## Abstract
|
||||
|
||||
This document describes the algorithms used in performing the TarSum checksum
|
||||
calculation on filesystem layers, the need for this method over existing
|
||||
methods, and the versioning of this calculation.
|
||||
|
||||
## Warning
|
||||
|
||||
This checksum algorithm is for best-effort comparison of file trees with fuzzy logic.
|
||||
|
||||
This is _not_ a cryptographic attestation, and should not be considered secure.
|
||||
|
||||
## Introduction
|
||||
|
||||
The transportation of filesystems, regarding Docker, is done with tar(1)
|
||||
archives. There are a variety of tar serialization formats [2], and a key
|
||||
concern here is ensuring a repeatable checksum given a set of inputs from a
|
||||
generic tar archive. Types of transportation include distribution to and from a
|
||||
registry endpoint, saving and loading through commands or Docker daemon APIs,
|
||||
transferring the build context from client to Docker daemon, and committing the
|
||||
filesystem of a container to become an image.
|
||||
|
||||
As tar archives are used for transit, but not preserved in many situations, the
|
||||
focus of the algorithm is to ensure the integrity of the preserved filesystem,
|
||||
while maintaining a deterministic accountability. This includes neither
|
||||
constraining the ordering or manipulation of the files during the creation or
|
||||
unpacking of the archive, nor include additional metadata state about the file
|
||||
system attributes.
|
||||
|
||||
## Intended Audience
|
||||
|
||||
This document is outlining the methods used for consistent checksum calculation
|
||||
for filesystems transported via tar archives.
|
||||
|
||||
Auditing these methodologies is an open and iterative process. This document
|
||||
should accommodate the review of source code. Ultimately, this document should
|
||||
be the starting point of further refinements to the algorithm and its future
|
||||
versions.
|
||||
|
||||
## Concept
|
||||
|
||||
The checksum mechanism must ensure the integrity and assurance of the
|
||||
filesystem payload.
|
||||
|
||||
## Checksum Algorithm Profile
|
||||
|
||||
A checksum mechanism must define the following operations and attributes:
|
||||
|
||||
* Associated hashing cipher - used to checksum each file payload and attribute
|
||||
information.
|
||||
* Checksum list - each file of the filesystem archive has its checksum
|
||||
calculated from the payload and attributes of the file. The final checksum is
|
||||
calculated from this list, with specific ordering.
|
||||
* Version - as the algorithm adapts to requirements, there are behaviors of the
|
||||
algorithm to manage by versioning.
|
||||
* Archive being calculated - the tar archive having its checksum calculated
|
||||
|
||||
## Elements of TarSum checksum
|
||||
|
||||
The calculated sum output is a text string. The elements included in the output
|
||||
of the calculated sum comprise the information needed for validation of the sum
|
||||
(TarSum version and hashing cipher used) and the expected checksum in hexadecimal
|
||||
form.
|
||||
|
||||
There are two delimiters used:
|
||||
* '+' separates TarSum version from hashing cipher
|
||||
* ':' separates calculation mechanics from expected hash
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
"tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e"
|
||||
| | \ |
|
||||
| | \ |
|
||||
|_version_|_cipher__|__ |
|
||||
| \ |
|
||||
|_calculation_mechanics_|______________________expected_sum_______________________|
|
||||
```
|
||||
|
||||
## Versioning
|
||||
|
||||
Versioning was introduced [0] to accommodate differences in calculation needed,
|
||||
and ability to maintain reverse compatibility.
|
||||
|
||||
The general algorithm will be describe further in the 'Calculation'.
|
||||
|
||||
### Version0
|
||||
|
||||
This is the initial version of TarSum.
|
||||
|
||||
Its element in the TarSum checksum string is `tarsum`.
|
||||
|
||||
### Version1
|
||||
|
||||
Its element in the TarSum checksum is `tarsum.v1`.
|
||||
|
||||
The notable changes in this version:
|
||||
* Exclusion of file `mtime` from the file information headers, in each file
|
||||
checksum calculation
|
||||
* Inclusion of extended attributes (`xattrs`. Also seen as `SCHILY.xattr.` prefixed Pax
|
||||
tar file info headers) keys and values in each file checksum calculation
|
||||
|
||||
### VersionDev
|
||||
|
||||
*Do not use unless validating refinements to the checksum algorithm*
|
||||
|
||||
Its element in the TarSum checksum is `tarsum.dev`.
|
||||
|
||||
This is a floating place holder for a next version and grounds for testing
|
||||
changes. The methods used for calculation are subject to change without notice,
|
||||
and this version is for testing and not for production use.
|
||||
|
||||
## Ciphers
|
||||
|
||||
The official default and standard hashing cipher used in the calculation mechanic
|
||||
is `sha256`. This refers to SHA256 hash algorithm as defined in FIPS 180-4.
|
||||
|
||||
Though the TarSum algorithm itself is not exclusively bound to the single
|
||||
hashing cipher `sha256`, support for alternate hashing ciphers was later added
|
||||
[1]. Use cases for alternate cipher could include future-proofing TarSum
|
||||
checksum format and using faster cipher hashes for tar filesystem checksums.
|
||||
|
||||
## Calculation
|
||||
|
||||
### Requirement
|
||||
|
||||
As mentioned earlier, the calculation is such that it takes into consideration
|
||||
the lifecycle of the tar archive. In that the tar archive is not an immutable,
|
||||
permanent artifact. Otherwise options like relying on a known hashing cipher
|
||||
checksum of the archive itself would be reliable enough. The tar archive of the
|
||||
filesystem is used as a transportation medium for Docker images, and the
|
||||
archive is discarded once its contents are extracted. Therefore, for consistent
|
||||
validation items such as order of files in the tar archive and time stamps are
|
||||
subject to change once an image is received.
|
||||
|
||||
### Process
|
||||
|
||||
The method is typically iterative due to reading tar info headers from the
|
||||
archive stream, though this is not a strict requirement.
|
||||
|
||||
#### Files
|
||||
|
||||
Each file in the tar archive have their contents (headers and body) checksummed
|
||||
individually using the designated associated hashing cipher. The ordered
|
||||
headers of the file are written to the checksum calculation first, and then the
|
||||
payload of the file body.
|
||||
|
||||
The resulting checksum of the file is appended to the list of file sums. The
|
||||
sum is encoded as a string of the hexadecimal digest. Additionally, the file
|
||||
name and position in the archive is kept as reference for special ordering.
|
||||
|
||||
#### Headers
|
||||
|
||||
The following headers are read, in this
|
||||
order ( and the corresponding representation of its value):
|
||||
* 'name' - string
|
||||
* 'mode' - string of the base10 integer
|
||||
* 'uid' - string of the integer
|
||||
* 'gid' - string of the integer
|
||||
* 'size' - string of the integer
|
||||
* 'mtime' (_Version0 only_) - string of integer of the seconds since 1970-01-01 00:00:00 UTC
|
||||
* 'typeflag' - string of the char
|
||||
* 'linkname' - string
|
||||
* 'uname' - string
|
||||
* 'gname' - string
|
||||
* 'devmajor' - string of the integer
|
||||
* 'devminor' - string of the integer
|
||||
|
||||
For >= Version1, the extended attribute headers ("SCHILY.xattr." prefixed pax
|
||||
headers) included after the above list. These xattrs key/values are first
|
||||
sorted by the keys.
|
||||
|
||||
#### Header Format
|
||||
|
||||
The ordered headers are written to the hash in the format of
|
||||
|
||||
"{.key}{.value}"
|
||||
|
||||
with no newline.
|
||||
|
||||
#### Body
|
||||
|
||||
After the order headers of the file have been added to the checksum for the
|
||||
file, the body of the file is written to the hash.
|
||||
|
||||
#### List of file sums
|
||||
|
||||
The list of file sums is sorted by the string of the hexadecimal digest.
|
||||
|
||||
If there are two files in the tar with matching paths, the order of occurrence
|
||||
for that path is reflected for the sums of the corresponding file header and
|
||||
body.
|
||||
|
||||
#### Final Checksum
|
||||
|
||||
Begin with a fresh or initial state of the associated hash cipher. If there is
|
||||
additional payload to include in the TarSum calculation for the archive, it is
|
||||
written first. Then each checksum from the ordered list of file sums is written
|
||||
to the hash.
|
||||
|
||||
The resulting digest is formatted per the Elements of TarSum checksum,
|
||||
including the TarSum version, the associated hash cipher and the hexadecimal
|
||||
encoded checksum digest.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
The initial version of TarSum has undergone one update that could invalidate
|
||||
handcrafted tar archives. The tar archive format supports appending of files
|
||||
with same names as prior files in the archive. The latter file will clobber the
|
||||
prior file of the same path. Due to this the algorithm now accounts for files
|
||||
with matching paths, and orders the list of file sums accordingly [3].
|
||||
|
||||
## Footnotes
|
||||
|
||||
* [0] Versioning https://github.com/docker/docker/commit/747f89cd327db9d50251b17797c4d825162226d0
|
||||
* [1] Alternate ciphers https://github.com/docker/docker/commit/4e9925d780665149b8bc940d5ba242ada1973c4e
|
||||
* [2] Tar http://en.wikipedia.org/wiki/Tar_%28computing%29
|
||||
* [3] Name collision https://github.com/docker/docker/commit/c5e6362c53cbbc09ddbabd5a7323e04438b57d31
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
Joffrey F (shin-) and Guillaume J. Charmes (creack) on the initial work of the
|
||||
TarSum calculation.
|
||||
|
||||
158
vendor/github.com/docker/docker/pkg/tarsum/versioning.go
generated
vendored
158
vendor/github.com/docker/docker/pkg/tarsum/versioning.go
generated
vendored
@@ -1,158 +0,0 @@
|
||||
package tarsum // import "github.com/docker/docker/pkg/tarsum"
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"errors"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Version is used for versioning of the TarSum algorithm
|
||||
// based on the prefix of the hash used
|
||||
// i.e. "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b"
|
||||
type Version int
|
||||
|
||||
// Prefix of "tarsum"
|
||||
const (
|
||||
Version0 Version = iota
|
||||
Version1
|
||||
// VersionDev this constant will be either the latest or an unsettled next-version of the TarSum calculation
|
||||
VersionDev
|
||||
)
|
||||
|
||||
// WriteV1Header writes a tar header to a writer in V1 tarsum format.
|
||||
func WriteV1Header(h *tar.Header, w io.Writer) {
|
||||
for _, elem := range v1TarHeaderSelect(h) {
|
||||
w.Write([]byte(elem[0] + elem[1]))
|
||||
}
|
||||
}
|
||||
|
||||
// VersionLabelForChecksum returns the label for the given tarsum
|
||||
// checksum, i.e., everything before the first `+` character in
|
||||
// the string or an empty string if no label separator is found.
|
||||
func VersionLabelForChecksum(checksum string) string {
|
||||
// Checksums are in the form: {versionLabel}+{hashID}:{hex}
|
||||
sepIndex := strings.Index(checksum, "+")
|
||||
if sepIndex < 0 {
|
||||
return ""
|
||||
}
|
||||
return checksum[:sepIndex]
|
||||
}
|
||||
|
||||
// GetVersions gets a list of all known tarsum versions.
|
||||
func GetVersions() []Version {
|
||||
v := []Version{}
|
||||
for k := range tarSumVersions {
|
||||
v = append(v, k)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
var (
|
||||
tarSumVersions = map[Version]string{
|
||||
Version0: "tarsum",
|
||||
Version1: "tarsum.v1",
|
||||
VersionDev: "tarsum.dev",
|
||||
}
|
||||
tarSumVersionsByName = map[string]Version{
|
||||
"tarsum": Version0,
|
||||
"tarsum.v1": Version1,
|
||||
"tarsum.dev": VersionDev,
|
||||
}
|
||||
)
|
||||
|
||||
func (tsv Version) String() string {
|
||||
return tarSumVersions[tsv]
|
||||
}
|
||||
|
||||
// GetVersionFromTarsum returns the Version from the provided string.
|
||||
func GetVersionFromTarsum(tarsum string) (Version, error) {
|
||||
tsv := tarsum
|
||||
if strings.Contains(tarsum, "+") {
|
||||
tsv = strings.SplitN(tarsum, "+", 2)[0]
|
||||
}
|
||||
for v, s := range tarSumVersions {
|
||||
if s == tsv {
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
return -1, ErrNotVersion
|
||||
}
|
||||
|
||||
// Errors that may be returned by functions in this package
|
||||
var (
|
||||
ErrNotVersion = errors.New("string does not include a TarSum Version")
|
||||
ErrVersionNotImplemented = errors.New("TarSum Version is not yet implemented")
|
||||
)
|
||||
|
||||
// tarHeaderSelector is the interface which different versions
|
||||
// of tarsum should use for selecting and ordering tar headers
|
||||
// for each item in the archive.
|
||||
type tarHeaderSelector interface {
|
||||
selectHeaders(h *tar.Header) (orderedHeaders [][2]string)
|
||||
}
|
||||
|
||||
type tarHeaderSelectFunc func(h *tar.Header) (orderedHeaders [][2]string)
|
||||
|
||||
func (f tarHeaderSelectFunc) selectHeaders(h *tar.Header) (orderedHeaders [][2]string) {
|
||||
return f(h)
|
||||
}
|
||||
|
||||
func v0TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) {
|
||||
return [][2]string{
|
||||
{"name", h.Name},
|
||||
{"mode", strconv.FormatInt(h.Mode, 10)},
|
||||
{"uid", strconv.Itoa(h.Uid)},
|
||||
{"gid", strconv.Itoa(h.Gid)},
|
||||
{"size", strconv.FormatInt(h.Size, 10)},
|
||||
{"mtime", strconv.FormatInt(h.ModTime.UTC().Unix(), 10)},
|
||||
{"typeflag", string([]byte{h.Typeflag})},
|
||||
{"linkname", h.Linkname},
|
||||
{"uname", h.Uname},
|
||||
{"gname", h.Gname},
|
||||
{"devmajor", strconv.FormatInt(h.Devmajor, 10)},
|
||||
{"devminor", strconv.FormatInt(h.Devminor, 10)},
|
||||
}
|
||||
}
|
||||
|
||||
func v1TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) {
|
||||
// Get extended attributes.
|
||||
xAttrKeys := make([]string, len(h.Xattrs))
|
||||
for k := range h.Xattrs {
|
||||
xAttrKeys = append(xAttrKeys, k)
|
||||
}
|
||||
sort.Strings(xAttrKeys)
|
||||
|
||||
// Make the slice with enough capacity to hold the 11 basic headers
|
||||
// we want from the v0 selector plus however many xattrs we have.
|
||||
orderedHeaders = make([][2]string, 0, 11+len(xAttrKeys))
|
||||
|
||||
// Copy all headers from v0 excluding the 'mtime' header (the 5th element).
|
||||
v0headers := v0TarHeaderSelect(h)
|
||||
orderedHeaders = append(orderedHeaders, v0headers[0:5]...)
|
||||
orderedHeaders = append(orderedHeaders, v0headers[6:]...)
|
||||
|
||||
// Finally, append the sorted xattrs.
|
||||
for _, k := range xAttrKeys {
|
||||
orderedHeaders = append(orderedHeaders, [2]string{k, h.Xattrs[k]})
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var registeredHeaderSelectors = map[Version]tarHeaderSelectFunc{
|
||||
Version0: v0TarHeaderSelect,
|
||||
Version1: v1TarHeaderSelect,
|
||||
VersionDev: v1TarHeaderSelect,
|
||||
}
|
||||
|
||||
func getTarHeaderSelector(v Version) (tarHeaderSelector, error) {
|
||||
headerSelector, ok := registeredHeaderSelectors[v]
|
||||
if !ok {
|
||||
return nil, ErrVersionNotImplemented
|
||||
}
|
||||
|
||||
return headerSelector, nil
|
||||
}
|
||||
22
vendor/github.com/docker/docker/pkg/tarsum/writercloser.go
generated
vendored
22
vendor/github.com/docker/docker/pkg/tarsum/writercloser.go
generated
vendored
@@ -1,22 +0,0 @@
|
||||
package tarsum // import "github.com/docker/docker/pkg/tarsum"
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
type writeCloseFlusher interface {
|
||||
io.WriteCloser
|
||||
Flush() error
|
||||
}
|
||||
|
||||
type nopCloseFlusher struct {
|
||||
io.Writer
|
||||
}
|
||||
|
||||
func (n *nopCloseFlusher) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *nopCloseFlusher) Flush() error {
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user