@ -17,7 +17,10 @@
package loader
import (
"bytes"
"context"
"fmt"
"io"
"os"
paths "path"
"path/filepath"
@ -68,6 +71,16 @@ type Options struct {
projectNameImperativelySet bool
// Profiles set profiles to enable
Profiles [ ] string
// ResourceLoaders manages support for remote resources
ResourceLoaders [ ] ResourceLoader
}
// ResourceLoader is a plugable remote resource resolver
type ResourceLoader interface {
// Accept returns `true` is the resource reference matches ResourceLoader supported protocol(s)
Accept ( path string ) bool
// Load returns the path to a local copy of remote resource identified by `path`.
Load ( ctx context . Context , path string ) ( string , error )
}
func ( o * Options ) clone ( ) * Options {
@ -85,6 +98,7 @@ func (o *Options) clone() *Options {
projectName : o . projectName ,
projectNameImperativelySet : o . projectNameImperativelySet ,
Profiles : o . Profiles ,
ResourceLoaders : o . ResourceLoaders ,
}
}
@ -154,7 +168,9 @@ func WithProfiles(profiles []string) func(*Options) {
// ParseYAML reads the bytes from a file, parses the bytes into a mapping
// structure, and returns it.
func ParseYAML ( source [ ] byte ) ( map [ string ] interface { } , error ) {
m , _ , err := parseYAML ( source )
r := bytes . NewReader ( source )
decoder := yaml . NewDecoder ( r )
m , _ , err := parseYAML ( decoder )
return m , err
}
@ -167,11 +183,11 @@ type PostProcessor interface {
Apply ( config * types . Config ) error
}
func parseYAML ( source [ ] byte ) ( map [ string ] interface { } , PostProcessor , error ) {
func parseYAML ( decoder * yaml . Decoder ) ( map [ string ] interface { } , PostProcessor , error ) {
var cfg interface { }
processor := ResetProcessor { target : & cfg }
if err := yaml. Unmarshal ( source , & processor ) ; err != nil {
if err := decoder. Decode ( & processor ) ; err != nil {
return nil , nil , err
}
stringMap , ok := cfg . ( map [ string ] interface { } )
@ -193,8 +209,14 @@ func parseYAML(source []byte) (map[string]interface{}, PostProcessor, error) {
return converted . ( map [ string ] interface { } ) , & processor , nil
}
// Load reads a ConfigDetails and returns a fully loaded configuration
// Load reads a ConfigDetails and returns a fully loaded configuration.
// Deprecated: use LoadWithContext.
func Load ( configDetails types . ConfigDetails , options ... func ( * Options ) ) ( * types . Project , error ) {
return LoadWithContext ( context . Background ( ) , configDetails , options ... )
}
// LoadWithContext reads a ConfigDetails and returns a fully loaded configuration
func LoadWithContext ( ctx context . Context , configDetails types . ConfigDetails , options ... func ( * Options ) ) ( * types . Project , error ) {
if len ( configDetails . ConfigFiles ) < 1 {
return nil , errors . Errorf ( "No files specified" )
}
@ -217,10 +239,10 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
return nil , err
}
opts . projectName = projectName
return load ( c onfigDetails, opts , nil )
return load ( c tx, c onfigDetails, opts , nil )
}
func load ( c onfigDetails types . ConfigDetails , opts * Options , loaded [ ] string ) ( * types . Project , error ) {
func load ( c tx context . Context , c onfigDetails types . ConfigDetails , opts * Options , loaded [ ] string ) ( * types . Project , error ) {
var model * types . Config
mainFile := configDetails . ConfigFiles [ 0 ] . Filename
@ -232,63 +254,87 @@ func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*t
}
loaded = append ( loaded , mainFile )
for i , file := range configDetails . ConfigFiles {
includeRefs := make ( map [ string ] [ ] types . IncludeConfig )
first := true
for _ , file := range configDetails . ConfigFiles {
var postProcessor PostProcessor
configDict := file . Config
if configDict == nil {
if len ( file . Content ) == 0 {
content , err := os . ReadFile ( file . Filename )
if err nil {
return nil , err
processYaml := func ( ) error {
if ! opts . SkipValidation {
if err := schema . Validate ( configDict ) ; err != nil {
return fmt . Errorf ( "validating %s: %w" , file . Filename , err )
}
file . Content = content
}
dict , p , err := parseConfig ( file . Content , opts )
configDict = groupXFieldsIntoExtensions ( configDict )
cfg , err := loadSections ( ctx , file . Filename , configDict , configDetails , opts )
if err != nil {
return nil , fmt . Errorf ( "parsing %s: %w" , file . Filename , err )
return err
}
configDict = dict
file . Config = dict
configDetails . ConfigFiles [ i ] = file
postProcessor = p
}
if ! opts . SkipValidation {
if err := schema . Validate ( configDict ) ; err != nil {
return nil , fmt . Errorf ( "validating %s: %w" , file . Filename , err )
if ! opts . SkipInclude {
var included map [ string ] [ ] types . IncludeConfig
cfg , included , err = loadInclude ( ctx , file . Filename , configDetails , cfg , opts , loaded )
if err != nil {
return err
}
for k , v := range included {
includeRefs [ k ] = append ( includeRefs [ k ] , v ... )
}
}
}
configDict = groupXFieldsIntoExtensions ( configDict )
cfg , err := loadSections ( file . Filename , configDict , configDetails , opts )
if err != nil {
return nil , err
}
if ! opts . SkipInclude {
cfg , err = loadInclude ( configDetails , cfg , opts , loaded )
if first {
first = false
model = cfg
return nil
}
merged , err := merge ( [ ] * types . Config { model , cfg } )
if err != nil {
return nil , err
return err
}
if postProcessor != nil {
err = postProcessor . Apply ( merged )
if err != nil {
return err
}
}
model = merged
return nil
}
if i == 0 {
model = cfg
continue
}
if configDict == nil {
if len ( file . Content ) == 0 {
content , err := os . ReadFile ( file . Filename )
if err != nil {
return nil , err
}
file . Content = content
}
merged , err := merge ( [ ] * types . Config { model , cfg } )
if err != nil {
return nil , err
}
if postProcessor != nil {
err = postProcessor . Apply ( merged )
if err != nil {
r := bytes . NewReader ( file . Content )
decoder := yaml . NewDecoder ( r )
for {
dict , p , err := parseConfig ( decoder , opts )
if err != nil {
if err != io . EOF {
return nil , fmt . Errorf ( "parsing %s: %w" , file . Filename , err )
}
break
}
configDict = dict
postProcessor = p
if err := processYaml ( ) ; err != nil {
return nil , err
}
}
} else {
if err := processYaml ( ) ; err != nil {
return nil , err
}
}
model = merged
}
project := & types . Project {
@ -303,6 +349,10 @@ func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*t
Extensions : model . Extensions ,
}
if len ( includeRefs ) != 0 {
project . IncludeReferences = includeRefs
}
if ! opts . SkipNormalization {
err := Normalize ( project )
if err != nil {
@ -333,9 +383,6 @@ func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*t
}
}
if profiles , ok := project . Environment [ consts . ComposeProfiles ] ; ok && len ( opts . Profiles ) == 0 {
opts . Profiles = strings . Split ( profiles , "," )
}
project . ApplyProfiles ( opts . Profiles )
err := project . ResolveServicesEnvironment ( opts . discardEnvFiles )
@ -422,8 +469,8 @@ func NormalizeProjectName(s string) string {
return strings . TrimLeft ( s , "_-" )
}
func parseConfig ( b [ ] byte , opts * Options ) ( map [ string ] interface { } , PostProcessor , error ) {
yml , postProcessor , err := parseYAML ( b )
func parseConfig ( decoder * yaml . Decoder , opts * Options ) ( map [ string ] interface { } , PostProcessor , error ) {
yml , postProcessor , err := parseYAML ( decoder )
if err != nil {
return nil , nil , err
}
@ -453,7 +500,7 @@ func groupXFieldsIntoExtensions(dict map[string]interface{}) map[string]interfac
return dict
}
func loadSections ( string , config map [ string ] interface { } , configDetails types . ConfigDetails , opts * Options ) ( * types . Config , error ) {
func loadSections ( ctx context . Context , filename string , config map [ string ] interface { } , configDetails types . ConfigDetails , opts * Options ) ( * types . Config , error ) {
var err error
cfg := types . Config {
Filename : filename ,
@ -466,7 +513,7 @@ func loadSections(filename string, config map[string]interface{}, configDetails
}
}
cfg . Name = name
cfg . Services , err = LoadServices ( , getSection ( config , "services" ) , configDetails . WorkingDir , configDetails . LookupEnv , opts )
cfg . Services , err = LoadServices ( ctx, filename, getSection ( config , "services" ) , configDetails . WorkingDir , configDetails . LookupEnv , opts )
if err != nil {
return nil , err
}
@ -659,7 +706,7 @@ func formatInvalidKeyError(keyPrefix string, key interface{}) error {
// LoadServices produces a ServiceConfig map from a compose file Dict
// the servicesDict is not validated if directly used. Use Load() to enable validation
func LoadServices ( string , servicesDict map [ string ] interface { } , workingDir string , lookupEnv template . Mapping , opts * Options ) ( [ ] types . ServiceConfig , error ) {
func LoadServices ( ctx context . Context , filename string , servicesDict map [ string ] interface { } , workingDir string , lookupEnv template . Mapping , opts * Options ) ( [ ] types . ServiceConfig , error ) {
var services [ ] types . ServiceConfig
x , ok := servicesDict [ extensions ]
@ -672,7 +719,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD
}
for name := range servicesDict {
serviceConfig , err := loadServiceWithExtends ( , name , servicesDict , workingDir , lookupEnv , opts , & cycleTracker { } )
serviceConfig , err := loadServiceWithExtends ( ctx, filename, name , servicesDict , workingDir , lookupEnv , opts , & cycleTracker { } )
if err != nil {
return nil , err
}
@ -683,7 +730,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD
return services , nil
}
func loadServiceWithExtends ( , name string , servicesDict map [ string ] interface { } , workingDir string , lookupEnv template . Mapping , opts * Options , ct * cycleTracker ) ( * types . ServiceConfig , error ) {
func loadServiceWithExtends ( ctx context . Context , filename, name string , servicesDict map [ string ] interface { } , workingDir string , lookupEnv template . Mapping , opts * Options , ct * cycleTracker ) ( * types . ServiceConfig , error ) {
if err := ct . Add ( filename , name ) ; err != nil {
return nil , err
}
@ -707,11 +754,21 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
var baseService * types . ServiceConfig
file := serviceConfig . Extends . File
if file == "" {
baseService , err = loadServiceWithExtends ( , baseServiceName , servicesDict , workingDir , lookupEnv , opts , ct )
baseService , err = loadServiceWithExtends ( ctx, filename, baseServiceName , servicesDict , workingDir , lookupEnv , opts , ct )
if err != nil {
return nil , err
}
} else {
for _ , loader := range opts . ResourceLoaders {
if loader . Accept ( file ) {
path , err := loader . Load ( ctx , file )
if err != nil {
return nil , err
}
file = path
break
}
}
// Resolve the path to the imported file, and load it.
baseFilePath := absPath ( workingDir , file )
@ -720,13 +777,16 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
return nil , err
}
baseFile , _ , err := parseConfig ( b , opts )
r := bytes . NewReader ( b )
decoder := yaml . NewDecoder ( r )
baseFile , _ , err := parseConfig ( decoder , opts )
if err != nil {
return nil , err
}
baseFileServices := getSection ( baseFile , "services" )
baseService , err = loadServiceWithExtends ( , baseServiceName , baseFileServices , filepath . Dir ( baseFilePath ) , lookupEnv , opts , ct )
baseService , err = loadServiceWithExtends ( ctx, baseFilePath, baseServiceName , baseFileServices , filepath . Dir ( baseFilePath ) , lookupEnv , opts , ct )
if err != nil {
return nil , err
}
@ -1038,7 +1098,12 @@ var transformServiceDeviceRequest TransformerFunc = func(data interface{}) (inte
value [ "count" ] = - 1
return value , nil
}
return data , errors . Errorf ( "invalid string value for 'count' (the only value allowed is 'all')" )
i , err := strconv . ParseInt ( val , 10 , 64 )
if err == nil {
value [ "count" ] = i
return value , nil
}
return data , errors . Errorf ( "invalid string value for 'count' (the only value allowed is 'all' or a number)" )
default :
return data , errors . Errorf ( "invalid type %T for device count" , val )
}
Sebastiaan van Stijn
1 year ago