publicWagsHome
/
buildx
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #800 from crazy-max/ulimit

Browse Source 
build: add ulimit support
pull/790/head
Tõnis Tiigi 3 years ago committed by GitHub
parent 461369748c b89e2f35df
commit 868610e0e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1055 additions and 536 deletions
Show Stats Download Patch File Download Diff File

10
build/build.go
Unescape Escape View File

@ -20,6 +20,7 @@ import (
"github.com/docker/buildx/util/imagetools"
"github.com/docker/buildx/util/progress"
clitypes "github.com/docker/cli/cli/config/types"
"github.com/docker/cli/opts"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
dockerclient "github.com/docker/docker/client"
@ -56,6 +57,7 @@ type Options struct {
ImageIDFile string
ExtraHosts []string
NetworkMode string
Ulimits *opts.UlimitOpt
NoCache bool
Target string
@ -554,6 +556,14 @@ func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Opti
}
so.FrontendAttrs["add-hosts"] = extraHosts
// setup ulimits
ulimits, err := toBuildkitUlimits(opt.Ulimits)
if err != nil {
return nil, nil, err
} else if len(ulimits) > 0 {
so.FrontendAttrs["ulimit"] = ulimits
}
return &so, releaseF, nil
}

13
build/utils.go
Unescape Escape View File

@ -7,6 +7,7 @@ import (
"os"
"strings"
"github.com/docker/cli/opts"
"github.com/pkg/errors"
)
@ -53,3 +54,15 @@ func toBuildkitExtraHosts(inp []string) (string, error) {
}
return strings.Join(hosts, ","), nil
}
// toBuildkitUlimits converts ulimits from docker type=soft:hard format to buildkit's csv format
func toBuildkitUlimits(inp *opts.UlimitOpt) (string, error) {
if inp == nil || len(inp.GetList()) == 0 {
return "", nil
}
ulimits := make([]string, 0, len(inp.GetList()))
for _, ulimit := range inp.GetList() {
ulimits = append(ulimits, ulimit.String())
}
return strings.Join(ulimits, ","), nil
}

17
commands/build.go
Unescape Escape View File

@ -15,7 +15,9 @@ import (
"github.com/docker/buildx/util/tracing"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/opts"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/go-units"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/util/appcontext"
@ -45,6 +47,7 @@ type buildOptions struct {
extraHosts []string
networkMode string
quiet bool
ulimits *opts.UlimitOpt
// unimplemented
squash bool
@ -53,7 +56,6 @@ type buildOptions struct {
// hidden
// untrusted bool
// ulimits *opts.UlimitOpt
// memory opts.MemBytes
// memorySwap opts.MemSwapBytes
// shmSize opts.MemBytes
@ -125,6 +127,7 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
ImageIDFile: in.imageIDFile,
ExtraHosts: in.extraHosts,
NetworkMode: in.networkMode,
Ulimits: in.ulimits,
}
platforms, err := platformutil.Parse(in.platforms)
@ -260,8 +263,15 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
return resp[defaultTargetName].ExporterResponse["containerimage.digest"], err
}
func newBuildOptions() buildOptions {
ulimits := make(map[string]*units.Ulimit)
return buildOptions{
ulimits: opts.NewUlimitOpt(&ulimits),
}
}
func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
var options buildOptions
options := newBuildOptions()
cmd := &cobra.Command{
Use: "build [OPTIONS] PATH | URL | -",
@ -303,6 +313,7 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
flags.StringSliceVar(&options.extraHosts, "add-host", []string{}, "Add a custom host-to-IP mapping (format: `host:ip`)")
flags.SetAnnotation("add-host", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#add-entries-to-container-hosts-file---add-host"})
flags.StringVar(&options.imageIDFile, "iidfile", "", "Write the image ID to the file")
flags.Var(options.ulimits, "ulimit", "Ulimit options")
// not implemented
flags.BoolVar(&options.squash, "squash", false, "Squash newly built layers into a single new layer")
@ -313,8 +324,6 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
var ignoreSlice []string
var ignoreBool bool
var ignoreInt int64
flags.StringVar(&ignore, "ulimit", "", "Ulimit options")
flags.MarkHidden("ulimit")
flags.StringSliceVar(&ignoreSlice, "security-opt", []string{}, "Security options")
flags.MarkHidden("security-opt")
flags.BoolVar(&ignoreBool, "compress", false, "Compress the build context using gzip")

16
docs/reference/buildx_build.md
Unescape Escape View File

@ -38,6 +38,7 @@ Start a build
| `--ssh stringArray` | SSH agent socket or keys to expose to the build (format: `default\|<id>[=<socket>\|<key>[,<key>]]`) |
| [`-t`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t), [`--tag stringArray`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t) | Name and optionally a tag (format: `name:tag`) |
| [`--target string`](https://docs.docker.com/engine/reference/commandline/build/#specifying-target-build-stage---target) | Set the target build stage to build. |
| [`--ulimit ulimit`](#ulimit) | Ulimit options |
<!---MARKER_GEN_END-->
@ -317,3 +318,18 @@ with `--allow-insecure-entitlement` (see [`create --buildkitd-flags`](buildx_cre
$ docker buildx create --use --name insecure-builder --buildkitd-flags '--allow-insecure-entitlement security.insecure'
$ docker buildx build --allow security.insecure .
```
### <a name="ulimit"></a> Set ulimits (--ulimit)
`--ulimit` is specified with a soft and hard limit as such:
`<type>=<soft limit>[:<hard limit>]`, for example:
```console
$ docker buildx build --ulimit nofile=1024:1024 .
```
> **Note**
>
> If you do not provide a `hard limit`, the `soft limit` is used
> for both values. If no `ulimits` are set, they are inherited from
> the default `ulimits` set on the daemon.

3
go.mod
Unescape Escape View File

@ -17,6 +17,7 @@ require (
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/docker v20.10.7+incompatible
github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
github.com/docker/go-units v0.4.0
github.com/docker/libtrust v0.0.0-20150526203908-9cbd2a1374f4 // indirect
github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
github.com/fvbommel/sortorder v1.0.1 // indirect
@ -30,7 +31,7 @@ require (
github.com/jinzhu/gorm v1.9.2 // indirect
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/moby/buildkit v0.9.1-0.20210927192002-23c8affb1ffe
github.com/moby/buildkit v0.9.1-0.20211008210008-ba673bbdab4f
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
github.com/pkg/errors v0.9.1

7
go.sum
Unescape Escape View File

@ -404,6 +404,7 @@ github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e h1:n81KvOMrLZa+VWHwST7dun9f0G98X3zREHS1ztYzZKU=
github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e/go.mod h1:xpWTC2KnJMiDLkoawhsPQcXjvwATEBcbq0xevG2YR9M=
github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
@ -906,8 +907,8 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ=
github.com/moby/buildkit v0.9.1-0.20210927192002-23c8affb1ffe h1:SA9l85GRngGomGpyDmVdziovizKPH8peFyUUtsluh34=
github.com/moby/buildkit v0.9.1-0.20210927192002-23c8affb1ffe/go.mod h1:BmktjI4PV4waDqQWB19oWso7X4t9VTcak6q7ynntmkQ=
github.com/moby/buildkit v0.9.1-0.20211008210008-ba673bbdab4f h1:RCFnbn0W1+InqTusV4raeqAyxvRMzuVoyT+8yKOMbWs=
github.com/moby/buildkit v0.9.1-0.20211008210008-ba673bbdab4f/go.mod h1:be+M6HNNl/Et0cUrFWRwEnXC1OChl1wPxXO5uEg4w6A=
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
@ -1203,7 +1204,7 @@ github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dS
github.com/tonistiigi/fsutil v0.0.0-20201103201449-0834f99b7b85/go.mod h1:a7cilN64dG941IOXfhJhlH0qB92hxJ9A1ewrdUmJ6xo=
github.com/tonistiigi/fsutil v0.0.0-20210818161904-4442383b5028 h1:uEkkUFMCPtzz1HVOa42u15OHems1ugiRt172tSRTWSk=
github.com/tonistiigi/fsutil v0.0.0-20210818161904-4442383b5028/go.mod h1:E6osHKls9ix67jofYQ61RQKwlJhqJOZM2hintp+49iI=
github.com/tonistiigi/go-actions-cache v0.0.0-20210714033416-b93d7f1b2e70/go.mod h1:dNS+PPTqGnSl80x3wEyWWCHeON5xiBGtcM0uD6CgHNU=
github.com/tonistiigi/go-actions-cache v0.0.0-20211002214948-4d48f2ff622a/go.mod h1:YiIBjH5gP7mao3t0dBrNNBGuKYdeJmcAJjYLXr43k6A=
github.com/tonistiigi/opentelemetry-go-contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.0.0-20210714055410-d010b05b4939 h1:s6wDMZYNyWt8KvkjhrMpOthFPgI3JB8ipJS+eCV/psg=
github.com/tonistiigi/opentelemetry-go-contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.0.0-20210714055410-d010b05b4939/go.mod h1:Vm5u/mtkj1OMhtao0v+BGo2LUoLCgHYXvRmj0jWITlE=
github.com/tonistiigi/opentelemetry-go-contrib/instrumentation/net/http/httptrace/otelhttptrace v0.0.0-20210714055410-d010b05b4939 h1:ZZ1KHKvs97BcRoblbm6RhrDzs/OejFv7miYSIcZI7Ds=

58
vendor/github.com/moby/buildkit/client/llb/exec.go generated vendored
Unescape Escape View File

@ -192,6 +192,7 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
User: user,
Hostname: hostname,
}
extraHosts, err := getExtraHosts(e.base)(ctx, c)
if err != nil {
return "", nil, nil, nil, err
@ -204,6 +205,31 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
meta.ExtraHosts = hosts
}
shmSize, err := getShmSize(e.base)(ctx, c)
if err != nil {
return "", nil, nil, nil, err
}
if shmSize != nil {
meta.ShmSize = *shmSize
}
ulimits, err := getUlimit(e.base)(ctx, c)
if err != nil {
return "", nil, nil, nil, err
}
if len(ulimits) > 0 {
addCap(&e.constraints, pb.CapExecMetaUlimit)
ul := make([]*pb.Ulimit, len(ulimits))
for i, u := range ulimits {
ul[i] = &pb.Ulimit{
Name: u.Name,
Soft: u.Soft,
Hard: u.Hard,
}
}
meta.Ulimit = ul
}
network, err := getNetwork(e.base)(ctx, c)
if err != nil {
return "", nil, nil, nil, err
@ -498,6 +524,18 @@ func AddExtraHost(host string, ip net.IP) RunOption {
})
}
func WithShmSize(kb int64) RunOption {
return runOptionFunc(func(ei *ExecInfo) {
ei.State = ei.State.WithShmSize(kb)
})
}
func AddUlimit(name UlimitName, soft int64, hard int64) RunOption {
return runOptionFunc(func(ei *ExecInfo) {
ei.State = ei.State.AddUlimit(name, soft, hard)
})
}
func With(so ...StateOption) RunOption {
return runOptionFunc(func(ei *ExecInfo) {
ei.State = ei.State.With(so...)
@ -667,3 +705,23 @@ const (
SecurityModeInsecure = pb.SecurityMode_INSECURE
SecurityModeSandbox = pb.SecurityMode_SANDBOX
)
type UlimitName string
const (
UlimitCore UlimitName = "core"
UlimitCPU UlimitName = "cpu"
UlimitData UlimitName = "data"
UlimitFsize UlimitName = "fsize"
UlimitLocks UlimitName = "locks"
UlimitMemlock UlimitName = "memlock"
UlimitMsgqueue UlimitName = "msgqueue"
UlimitNice UlimitName = "nice"
UlimitNofile UlimitName = "nofile"
UlimitNproc UlimitName = "nproc"
UlimitRss UlimitName = "rss"
UlimitRtprio UlimitName = "rtprio"
UlimitRttime UlimitName = "rttime"
UlimitSigpending UlimitName = "sigpending"
UlimitStack UlimitName = "stack"
)

62
vendor/github.com/moby/buildkit/client/llb/meta.go generated vendored
Unescape Escape View File

@ -18,12 +18,15 @@ var (
keyArgs = contextKeyT("llb.exec.args")
keyDir = contextKeyT("llb.exec.dir")
keyEnv = contextKeyT("llb.exec.env")
keyUser = contextKeyT("llb.exec.user")
keyHostname = contextKeyT("llb.exec.hostname")
keyExtraHost = contextKeyT("llb.exec.extrahost")
keyPlatform = contextKeyT("llb.platform")
keyNetwork = contextKeyT("llb.network")
keySecurity = contextKeyT("llb.security")
keyHostname = contextKeyT("llb.exec.hostname")
keyShmSize = contextKeyT("llb.exec.shmsize")
keyUlimit = contextKeyT("llb.exec.ulimit")
keyUser = contextKeyT("llb.exec.user")
keyPlatform = contextKeyT("llb.platform")
keyNetwork = contextKeyT("llb.network")
keySecurity = contextKeyT("llb.security")
)
func AddEnvf(key, value string, v ...interface{}) StateOption {
@ -232,6 +235,55 @@ type HostIP struct {
IP net.IP
}
func shmSize(kb int64) StateOption {
return func(s State) State {
return s.WithValue(keyShmSize, kb)
}
}
func getShmSize(s State) func(context.Context, *Constraints) (*int64, error) {
return func(ctx context.Context, c *Constraints) (*int64, error) {
v, err := s.getValue(keyShmSize)(ctx, c)
if err != nil {
return nil, err
}
if v != nil {
kb := v.(int64)
return &kb, nil
}
return nil, nil
}
}
func ulimit(name UlimitName, soft int64, hard int64) StateOption {
return func(s State) State {
return s.withValue(keyUlimit, func(ctx context.Context, c *Constraints) (interface{}, error) {
v, err := getUlimit(s)(ctx, c)
if err != nil {
return nil, err
}
return append(v, pb.Ulimit{
Name: string(name),
Soft: soft,
Hard: hard,
}), nil
})
}
}
func getUlimit(s State) func(context.Context, *Constraints) ([]pb.Ulimit, error) {
return func(ctx context.Context, c *Constraints) ([]pb.Ulimit, error) {
v, err := s.getValue(keyUlimit)(ctx, c)
if err != nil {
return nil, err
}
if v != nil {
return v.([]pb.Ulimit), nil
}
return nil, nil
}
}
func Network(v pb.NetMode) StateOption {
return func(s State) State {
return s.WithValue(keyNetwork, v)

8
vendor/github.com/moby/buildkit/client/llb/state.go generated vendored
Unescape Escape View File

@ -397,6 +397,14 @@ func (s State) AddExtraHost(host string, ip net.IP) State {
return extraHost(host, ip)(s)
}
func (s State) WithShmSize(kb int64) State {
return shmSize(kb)(s)
}
func (s State) AddUlimit(name UlimitName, soft int64, hard int64) State {
return ulimit(name, soft, hard)(s)
}
func (s State) isFileOpCopyInput() {}
type output struct {

36
vendor/github.com/moby/buildkit/solver/pb/caps.go generated vendored
Unescape Escape View File

@ -35,22 +35,22 @@ const (
CapBuildOpLLBFileName apicaps.CapID = "source.buildop.llbfilename"
CapExecMetaBase apicaps.CapID = "exec.meta.base"
CapExecMetaProxy apicaps.CapID = "exec.meta.proxyenv"
CapExecMetaNetwork apicaps.CapID = "exec.meta.network"
CapExecMetaSecurity apicaps.CapID = "exec.meta.security"
CapExecMetaSetsDefaultPath apicaps.CapID = "exec.meta.setsdefaultpath"
CapExecMountBind apicaps.CapID = "exec.mount.bind"
CapExecMountBindReadWriteNoOuput apicaps.CapID = "exec.mount.bind.readwrite-nooutput"
CapExecMountCache apicaps.CapID = "exec.mount.cache"
CapExecMountCacheSharing apicaps.CapID = "exec.mount.cache.sharing"
CapExecMountSelector apicaps.CapID = "exec.mount.selector"
CapExecMountTmpfs apicaps.CapID = "exec.mount.tmpfs"
CapExecMountSecret apicaps.CapID = "exec.mount.secret"
CapExecMountSSH apicaps.CapID = "exec.mount.ssh"
CapExecCgroupsMounted apicaps.CapID = "exec.cgroup"
CapExecMetaBase apicaps.CapID = "exec.meta.base"
CapExecMetaNetwork apicaps.CapID = "exec.meta.network"
CapExecMetaProxy apicaps.CapID = "exec.meta.proxyenv"
CapExecMetaSecurity apicaps.CapID = "exec.meta.security"
CapExecMetaSecurityDeviceWhitelistV1 apicaps.CapID = "exec.meta.security.devices.v1"
CapExecMetaSetsDefaultPath apicaps.CapID = "exec.meta.setsdefaultpath"
CapExecMetaUlimit apicaps.CapID = "exec.meta.ulimit"
CapExecMountBind apicaps.CapID = "exec.mount.bind"
CapExecMountBindReadWriteNoOuput apicaps.CapID = "exec.mount.bind.readwrite-nooutput"
CapExecMountCache apicaps.CapID = "exec.mount.cache"
CapExecMountCacheSharing apicaps.CapID = "exec.mount.cache.sharing"
CapExecMountSelector apicaps.CapID = "exec.mount.selector"
CapExecMountTmpfs apicaps.CapID = "exec.mount.tmpfs"
CapExecMountSecret apicaps.CapID = "exec.mount.secret"
CapExecMountSSH apicaps.CapID = "exec.mount.ssh"
CapExecCgroupsMounted apicaps.CapID = "exec.cgroup"
CapFileBase apicaps.CapID = "file.base"
CapFileRmWildcard apicaps.CapID = "file.rm.wildcard"
@ -236,6 +236,12 @@ func init() {
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapExecMetaUlimit,
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapExecMountBind,
Enabled: true,

1340
vendor/github.com/moby/buildkit/solver/pb/ops.pb.go generated vendored
View File

File diff suppressed because it is too large Load Diff

18
vendor/github.com/moby/buildkit/solver/pb/ops.proto generated vendored
Unescape Escape View File

@ -58,6 +58,19 @@ message Meta {
ProxyEnv proxy_env = 5;
repeated HostIP extraHosts = 6;
string hostname = 7;
int64 shmSize = 8;
repeated Ulimit ulimit = 9;
}
message HostIP {
string Host = 1;
string IP = 2;
}
message Ulimit {
string Name = 1;
int64 Soft = 2;
int64 Hard = 3;
}
enum NetMode {
@ -243,11 +256,6 @@ message Definition {
Source Source = 3;
}
message HostIP {
string Host = 1;
string IP = 2;
}
message FileOp {
repeated FileAction actions = 2;
}

3
vendor/modules.txt vendored
Unescape Escape View File

@ -168,6 +168,7 @@ github.com/docker/go-connections/tlsconfig
# github.com/docker/go-metrics v0.0.1
github.com/docker/go-metrics
# github.com/docker/go-units v0.4.0
## explicit
github.com/docker/go-units
# github.com/docker/libtrust v0.0.0-20150526203908-9cbd2a1374f4
## explicit
@ -281,7 +282,7 @@ github.com/miekg/pkcs11
github.com/mitchellh/go-wordwrap
# github.com/mitchellh/mapstructure v1.4.1
github.com/mitchellh/mapstructure
# github.com/moby/buildkit v0.9.1-0.20210927192002-23c8affb1ffe
# github.com/moby/buildkit v0.9.1-0.20211008210008-ba673bbdab4f
## explicit
github.com/moby/buildkit/api/services/control
github.com/moby/buildkit/api/types

Write Preview
Loading…
Cancel
Save