vendor: update git url handling with changes to buildkit

The git ssh url parsing API changed a bit and broke how buildx utilized
the API. This updates the affected method with the new method of
identifying an SSH url.

Signed-off-by: Jonathan A. Sternberg <jonathan.sternberg@docker.com>
This commit is contained in:
Jonathan A. Sternberg
2023-08-25 13:42:18 -05:00
parent da6662975f
commit fc7f066caa
28 changed files with 525 additions and 435 deletions

View File

@@ -63,14 +63,13 @@ func GetLogger(ctx context.Context) (l *logrus.Entry) {
return l
}
// LazyStackTrace lets you include a stack trace as a field's value in a log but only
// call it when the log level is actually enabled.
type LazyStackTrace struct{}
func (LazyStackTrace) String() string {
return string(debug.Stack())
}
func (LazyStackTrace) MarshalText() ([]byte, error) {
return debug.Stack(), nil
// TraceLevelOnlyStack returns a stack trace for the current goroutine only if
// trace level logs are enabled; otherwise it returns an empty string. This ensure
// we only pay the cost of generating a stack trace when the log entry will actually
// be emitted.
func TraceLevelOnlyStack() string {
if logrus.GetLevel() == logrus.TraceLevel {
return string(debug.Stack())
}
return ""
}

View File

@@ -1,46 +0,0 @@
package gitutil
import (
"strings"
"github.com/moby/buildkit/util/sshutil"
)
const (
HTTPProtocol = iota + 1
HTTPSProtocol
SSHProtocol
GitProtocol
UnknownProtocol
)
// ParseProtocol parses a git URL and returns the remote url and protocol type
func ParseProtocol(remote string) (string, int) {
prefixes := map[string]int{
"http://": HTTPProtocol,
"https://": HTTPSProtocol,
"git://": GitProtocol,
"ssh://": SSHProtocol,
}
protocolType := UnknownProtocol
for prefix, potentialType := range prefixes {
if strings.HasPrefix(remote, prefix) {
remote = strings.TrimPrefix(remote, prefix)
protocolType = potentialType
}
}
if protocolType == UnknownProtocol && sshutil.IsImplicitSSHTransport(remote) {
protocolType = SSHProtocol
}
// remove name from ssh
if protocolType == SSHProtocol {
parts := strings.SplitN(remote, "@", 2)
if len(parts) == 2 {
remote = parts[1]
}
}
return remote, protocolType
}

View File

@@ -1,10 +1,11 @@
package gitutil
import (
"regexp"
"net/url"
"strings"
"github.com/containerd/containerd/errdefs"
"github.com/pkg/errors"
)
// GitRef represents a git ref.
@@ -51,35 +52,51 @@ type GitRef struct {
func ParseGitRef(ref string) (*GitRef, error) {
res := &GitRef{}
var (
remote *url.URL
err error
)
if strings.HasPrefix(ref, "github.com/") {
res.IndistinguishableFromLocal = true // Deprecated
} else {
_, proto := ParseProtocol(ref)
switch proto {
case UnknownProtocol:
return nil, errdefs.ErrInvalidArgument
remote = &url.URL{
Scheme: "https",
Host: "github.com",
Path: strings.TrimPrefix(ref, "github.com/"),
}
switch proto {
} else {
remote, err = ParseURL(ref)
if errors.Is(err, ErrUnknownProtocol) {
remote, err = ParseURL("https://" + ref)
}
if err != nil {
return nil, err
}
switch remote.Scheme {
case HTTPProtocol, GitProtocol:
res.UnencryptedTCP = true // Discouraged, but not deprecated
}
switch proto {
switch remote.Scheme {
// An HTTP(S) URL is considered to be a valid git ref only when it has the ".git[...]" suffix.
case HTTPProtocol, HTTPSProtocol:
var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
if !gitURLPathWithFragmentSuffix.MatchString(ref) {
if !strings.HasSuffix(remote.Path, ".git") {
return nil, errdefs.ErrInvalidArgument
}
}
}
var fragment string
res.Remote, fragment, _ = strings.Cut(ref, "#")
if len(res.Remote) == 0 {
return res, errdefs.ErrInvalidArgument
res.Commit, res.SubDir = SplitGitFragment(remote.Fragment)
remote.Fragment = ""
res.Remote = remote.String()
if res.IndistinguishableFromLocal {
_, res.Remote, _ = strings.Cut(res.Remote, "://")
}
res.Commit, res.SubDir, _ = strings.Cut(fragment, ":")
repoSplitBySlash := strings.Split(res.Remote, "/")
res.ShortName = strings.TrimSuffix(repoSplitBySlash[len(repoSplitBySlash)-1], ".git")
return res, nil
}

View File

@@ -0,0 +1,71 @@
package gitutil
import (
"net/url"
"regexp"
"strings"
"github.com/moby/buildkit/util/sshutil"
"github.com/pkg/errors"
)
const (
HTTPProtocol string = "http"
HTTPSProtocol string = "https"
SSHProtocol string = "ssh"
GitProtocol string = "git"
)
var (
ErrUnknownProtocol = errors.New("unknown protocol")
ErrInvalidProtocol = errors.New("invalid protocol")
)
var supportedProtos = map[string]struct{}{
HTTPProtocol: {},
HTTPSProtocol: {},
SSHProtocol: {},
GitProtocol: {},
}
var protoRegexp = regexp.MustCompile(`^[a-zA-Z0-9]+://`)
// ParseURL parses a git URL and returns a parsed URL object.
//
// ParseURL understands implicit ssh URLs such as "git@host:repo", and
// returns the same response as if the URL were "ssh://git@host/repo".
func ParseURL(remote string) (*url.URL, error) {
if proto := protoRegexp.FindString(remote); proto != "" {
proto = strings.ToLower(strings.TrimSuffix(proto, "://"))
if _, ok := supportedProtos[proto]; !ok {
return nil, errors.Wrap(ErrInvalidProtocol, proto)
}
return url.Parse(remote)
}
if sshutil.IsImplicitSSHTransport(remote) {
remote, fragment, _ := strings.Cut(remote, "#")
remote, path, _ := strings.Cut(remote, ":")
user, host, _ := strings.Cut(remote, "@")
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
return &url.URL{
Scheme: SSHProtocol,
User: url.User(user),
Host: host,
Path: path,
Fragment: fragment,
}, nil
}
return nil, ErrUnknownProtocol
}
// SplitGitFragments splits a git URL fragment into its respective git
// reference and subdirectory components.
func SplitGitFragment(fragment string) (ref string, subdir string) {
ref, subdir, _ = strings.Cut(fragment, ":")
return ref, subdir
}

View File

@@ -65,7 +65,7 @@ func (ps *MultiWriter) Write(id string, v interface{}) error {
Sys: v,
meta: ps.meta,
}
return ps.WriteRawProgress(p)
return ps.writeRawProgress(p)
}
func (ps *MultiWriter) WriteRawProgress(p *Progress) error {

View File

@@ -3,15 +3,15 @@ syntax = "proto3";
package stack;
message Stack {
repeated Frame frames = 1;
repeated string cmdline = 2;
int32 pid = 3;
string version = 4;
string revision = 5;
repeated Frame frames = 1;
repeated string cmdline = 2;
int32 pid = 3;
string version = 4;
string revision = 5;
}
message Frame {
string Name = 1;
string File = 2;
int32 Line = 3;
string Name = 1;
string File = 2;
int32 Line = 3;
}

View File

@@ -13,6 +13,7 @@ import (
"github.com/google/shlex"
"github.com/moby/buildkit/util/bklog"
"github.com/pkg/errors"
)
const buildkitdConfigFile = "buildkitd.toml"
@@ -126,7 +127,7 @@ func newSandbox(ctx context.Context, w Worker, mirror string, mv matrixValue) (s
}
func RootlessSupported(uid int) bool {
cmd := exec.Command("sudo", "-E", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "exec", "unshare", "-U", "true") //nolint:gosec // test utility
cmd := exec.Command("sudo", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "exec", "unshare", "-U", "true") //nolint:gosec // test utility
b, err := cmd.CombinedOutput()
if err != nil {
bklog.L.Warnf("rootless mode is not supported on this host: %v (%s)", err, string(b))
@@ -156,6 +157,13 @@ func FormatLogs(m map[string]*bytes.Buffer) string {
}
func CheckFeatureCompat(t *testing.T, sb Sandbox, features map[string]struct{}, reason ...string) {
t.Helper()
if err := HasFeatureCompat(t, sb, features, reason...); err != nil {
t.Skipf(err.Error())
}
}
func HasFeatureCompat(t *testing.T, sb Sandbox, features map[string]struct{}, reason ...string) error {
t.Helper()
if len(reason) == 0 {
t.Fatal("no reason provided")
@@ -172,6 +180,7 @@ func CheckFeatureCompat(t *testing.T, sb Sandbox, features map[string]struct{},
}
}
if len(ereasons) > 0 {
t.Skipf("%s worker can not currently run this test due to missing features (%s)", sb.Name(), strings.Join(ereasons, ", "))
return errors.Errorf("%s worker can not currently run this test due to missing features (%s)", sb.Name(), strings.Join(ereasons, ", "))
}
return nil
}

View File

@@ -168,7 +168,7 @@ disabled_plugins = ["cri"]
containerdArgs := []string{c.Containerd, "--config", configFile}
rootlessKitState := filepath.Join(tmpdir, "rootlesskit-containerd")
if rootless {
containerdArgs = append(append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", c.UID), "-i",
containerdArgs = append(append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.UID), "-i",
fmt.Sprintf("CONTAINERD_ROOTLESS_ROOTLESSKIT_STATE_DIR=%s", rootlessKitState),
// Integration test requires the access to localhost of the host network namespace.
// TODO: remove these configurations
@@ -211,7 +211,7 @@ disabled_plugins = ["cri"]
if err != nil {
return nil, nil, err
}
buildkitdArgs = append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", c.UID), "-i", "--", "exec",
buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.UID), "-i", "--", "exec",
"nsenter", "-U", "--preserve-credentials", "-m", "-t", fmt.Sprintf("%d", pid)},
append(buildkitdArgs, "--containerd-worker-snapshotter=native")...)
}

View File

@@ -61,3 +61,7 @@ var features = map[string]struct{}{
func CheckFeatureCompat(t *testing.T, sb integration.Sandbox, reason ...string) {
integration.CheckFeatureCompat(t, sb, features, reason...)
}
func HasFeatureCompat(t *testing.T, sb integration.Sandbox, reason ...string) error {
return integration.HasFeatureCompat(t, sb, features, reason...)
}

View File

@@ -66,7 +66,7 @@ func (s *OCI) New(ctx context.Context, cfg *integration.BackendConfig) (integrat
return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", s.UID, s.GID)
}
// TODO: make sure the user exists and subuid/subgid are configured.
buildkitdArgs = append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", s.UID), "-i", "--", "exec", "rootlesskit"}, buildkitdArgs...)
buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", s.UID), "-i", "--", "exec", "rootlesskit"}, buildkitdArgs...)
}
var extraEnv []string

View File

@@ -54,9 +54,9 @@ func runBuildkitd(ctx context.Context, conf *integration.BackendConfig, args []s
address = getBuildkitdAddr(tmpdir)
args = append(args, "--root", tmpdir, "--addr", address, "--debug")
args = append(args, "--root", tmpdir, "--addr", address, "--otel-socket-path", getTraceSocketPath(tmpdir), "--debug")
cmd := exec.Command(args[0], args[1:]...) //nolint:gosec // test utility
cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "BUILDKIT_TRACE_SOCKET="+getTraceSocketPath(tmpdir), "TMPDIR="+filepath.Join(tmpdir, "tmp"))
cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp"))
cmd.Env = append(cmd.Env, extraEnv...)
cmd.SysProcAttr = getSysProcAttr()