From 122e732feec71fa9eb074ff6b52cdca02b93fa32 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 20 Jun 2024 17:06:07 -0600 Subject: [PATCH 01/40] Updated Makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 7b0fd12..2a69980 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ all: plugins exe # build the main executable to make configs main: exe +driver: exe exe: go build --tags=all -o configurator From 0e3eec733bc5003d5092f5a53ce9697445337a18 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 20 Jun 2024 17:06:19 -0600 Subject: [PATCH 02/40] Updated README.md --- README.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3103256..c012873 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # OpenCHAMI Configurator -The `configurator` (portmanteau of config + generator) is a tool that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate commonly used config files. The tool is also capable of some templating using the Jinja 2 syntax with generator plugins. +The `configurator` (portmanteau of config + generator) is an extensible tool that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate commonly used config files based on Jinja 2 template files. The tool and generator plugins are written in Go and plugins can be written by following the ["Creating Generator Plugins"](#creating-generator-plugins) section of this README. ## Building and Usage @@ -32,16 +32,18 @@ These commands will build the default plugins and store them in the "lib" direct This will generate a new `dnsmasq` config file based on the Jinja 2 template specified in the config file for "dnsmasq". The `--target` flag specifies the type of config file to generate by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `configurator` tool requires a valid access token when making requests to an instance of SMD that has protected routes. -The tool can also run as a microservice: +The tool can also run as a service to generate files for clients: ```bash ./configurator serve --config config.yaml ``` -Once the server is up and listening for HTTP requests, you can try making a request to it with curl: +Once the server is up and listening for HTTP requests, you can try making a request to it with `curl` or `configurator fetch`. Both commands below are essentially equivalent: ```bash -curl http://127.0.0.1:3334/target?type=dhcp&template=dnsmasq +curl http://127.0.0.1:3334/generate?target=dnsmasq +# ...or... +./configurator fetch --target dnsmasq --host http://127.0.0.1 --port 3334 ``` This will do the same thing as the `generate` subcommand, but remotely. @@ -108,30 +110,30 @@ Now your plugin should be available to use with the `configurator` main driver. Here is an example config file to start using configurator: ```yaml -server: +server: # server settings when using as service host: 127.0.0.1 port: 3334 - jwks: + jwks: # set URL for JWKS to enable auth uri: "" retries: 5 -smd: +smd: # settings for SMD service host: http://127.0.0.1 port: 27779 -templates: +templates: # template mappings to generator plugins (by name) dnsmasq: templates/dnsmasq.jinja coredhcp: templates/coredhcp.jinja syslog: templates/syslog.jinja ansible: templates/ansible.jinja powerman: templates/powerman.jinja conman: templates/conman.jinja -groups: +groups: # (WIP) setting to allow creating configs by groups warewulf: - dnsmasq - syslog - ansible - powerman - conman -plugins: +plugins: # path to plugin directories (may change to include files as well) - "lib/" ``` @@ -143,6 +145,6 @@ The `server` section sets the properties for running the `configurator` tool as ## TODO -- Add group functionality -- Extend SMD client functionality -- Redo service API with authorization +- Add group functionality to create by files by groups +- Extend SMD client functionality (or make extensible?) +- Handle authentication with `OAuthClient`'s correctly From 22195fa00a9833ef42e9da992fefdd287f6a742e Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 20 Jun 2024 17:09:02 -0600 Subject: [PATCH 03/40] Fixed server implementation and refactored --- cmd/generate.go | 194 +++++++------------------------- cmd/serve.go | 42 ++++++- internal/client.go | 54 +++++++-- internal/config.go | 4 +- internal/configurator.go | 34 ++++++ internal/generator/generator.go | 119 ++++++++++++++------ internal/server/server.go | 40 +++++-- internal/util/params.go | 17 +++ 8 files changed, 289 insertions(+), 215 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index d73d645..d9b7fa0 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -6,13 +6,10 @@ package cmd import ( "encoding/json" "fmt" - "maps" "os" "path/filepath" - configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/generator" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -25,28 +22,19 @@ var generateCmd = &cobra.Command{ Use: "generate", Short: "Generate a config file from state management", Run: func(cmd *cobra.Command, args []string) { - // load generator plugins to generate configs or to print - var ( - generators = make(map[string]generator.Generator) - client = configurator.SmdClient{ - Host: config.SmdClient.Host, - Port: config.SmdClient.Port, - AccessToken: config.AccessToken, - } - ) - for _, path := range pluginPaths { - if verbose { - fmt.Printf("loading plugins from '%s'\n", path) - } - gens, err := generator.LoadPlugins(path) + // if we have more than one target and output is set, create configs in directory + targetCount := len(targets) + if outputPath != "" && targetCount > 1 { + err := os.MkdirAll(outputPath, 0o755) if err != nil { - fmt.Printf("failed to load plugins: %v\n", err) - err = nil - continue + fmt.Printf("failed to make output directory: %v", err) + os.Exit(1) } + } - // add loaded generator plugins to set - maps.Copy(generators, gens) + // use config plugins if none supplied via CLI + if len(pluginPaths) <= 0 { + pluginPaths = append(pluginPaths, config.PluginDirs...) } // show config as JSON and generators if verbose @@ -58,149 +46,47 @@ var generateCmd = &cobra.Command{ fmt.Printf("%v\n", string(b)) } - // show available targets then exit - if len(args) == 0 && len(targets) == 0 { - for g := range generators { - fmt.Printf("\tplugin: %s, name:\n", g) + // generate config with each supplied target + for _, target := range targets { + params := generator.Params{ + Args: args, + PluginPaths: pluginPaths, + Target: target, + Verbose: verbose, } - os.Exit(0) - } - - // make sure that we have a token present before trying to make request - if config.AccessToken == "" { - // TODO: make request to check if request will need token - - // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead - accessToken := os.Getenv("ACCESS_TOKEN") - if accessToken != "" { - config.AccessToken = accessToken - } else { - // TODO: try and fetch token first if it is needed - if verbose { - fmt.Printf("No token found. Attempting to generate config without one...\n") - } + output, err := generator.Generate(&config, params) + if err != nil { + fmt.Printf("failed to generate config: %v\n", err) + os.Exit(1) } - } - if targets == nil { - logrus.Errorf("no target supplied (--target type:template)") - } else { - // if we have more than one target and output is set, create configs in directory - targetCount := len(targets) - if outputPath != "" && targetCount > 1 { - err := os.MkdirAll(outputPath, 0o755) + // write config output if no specific targetPath is set + if outputPath == "" { + // write only to stdout + fmt.Printf("%s\n", string(output)) + } else if outputPath != "" && targetCount == 1 { + // write just a single file using template name + err := os.WriteFile(outputPath, output, 0o644) if err != nil { - logrus.Errorf("failed to make output directory: %v", err) - return + fmt.Printf("failed to write config to file: %v", err) + os.Exit(1) + } + } else if outputPath != "" && targetCount > 1 { + // write multiple files in directory using template name + err := os.WriteFile(fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, ".conf"), output, 0o644) + if err != nil { + fmt.Printf("failed to write config to file: %v", err) + os.Exit(1) } } - - for _, target := range targets { - // split the target and type - // tmp := strings.Split(target, ":") - - // make sure each target has at least two args - // if len(tmp) < 2 { - // message := "target" - // if len(tmp) == 1 { - // message += fmt.Sprintf(" '%s'", tmp[0]) - // } - // message += " does not provide enough arguments (args: \"type:template\")" - // logrus.Errorf(message) - // continue - // } - // var ( - // _type = tmp[0] - // _template = tmp[1] - // ) - // g := generator.Generator{ - // Type: tmp[0], - // Template: tmp[1], - // } - - // check if another param is specified - // targetPath := "" - // if len(tmp) > 2 { - // targetPath = tmp[2] - // } - - // run the generator plugin from target passed - gen := generators[target] - if gen == nil { - fmt.Printf("invalid generator target (%s)\n", target) - continue - } - output, err := gen.Generate( - &config, - generator.WithTemplate(gen.GetName()), - generator.WithClient(client), - ) - if err != nil { - fmt.Printf("failed to generate config: %v\n", err) - continue - } - - // NOTE: we probably don't want to hardcode the types, but should do for now - // ext := "" - // contents := []byte{} - // if _type == "dhcp" { - // // fetch eths from SMD - // eths, err := client.FetchEthernetInterfaces() - // if err != nil { - // logrus.Errorf("failed to fetch DHCP metadata: %v\n", err) - // continue - // } - // if len(eths) <= 0 { - // continue - // } - // // generate a new config from that data - // contents, err = g.GenerateDHCP(&config, eths) - // if err != nil { - // logrus.Errorf("failed to generate DHCP config file: %v\n", err) - // continue - // } - // ext = "conf" - // } else if g.Type == "dns" { - // // TODO: fetch from SMD - // // TODO: generate config from pulled info - - // } else if g.Type == "syslog" { - - // } else if g.Type == "ansible" { - - // } else if g.Type == "warewulf" { - - // } - - // write config output if no specific targetPath is set - // if targetPath == "" { - if outputPath == "" { - // write only to stdout - fmt.Printf("%s\n", string(output)) - } else if outputPath != "" && targetCount == 1 { - // write just a single file using template name - err := os.WriteFile(outputPath, output, 0o644) - if err != nil { - logrus.Errorf("failed to write config to file: %v", err) - continue - } - } else if outputPath != "" && targetCount > 1 { - // write multiple files in directory using template name - err := os.WriteFile(fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, ".conf"), output, 0o644) - if err != nil { - logrus.Errorf("failed to write config to file: %v", err) - continue - } - } - // } - } // for targets } + }, } func init() { - generateCmd.Flags().StringSliceVar(&targets, "target", nil, "set the target configs to make") - generateCmd.Flags().StringSliceVar(&pluginPaths, "plugin", nil, "set the generator plugins directory path to shared libraries") + generateCmd.Flags().StringSliceVar(&targets, "target", []string{}, "set the target configs to make") + generateCmd.Flags().StringSliceVar(&pluginPaths, "plugins", []string{}, "set the generator plugins directory path") generateCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") generateCmd.Flags().IntVar(&tokenFetchRetries, "fetch-retries", 5, "set the number of retries to fetch an access token") diff --git a/cmd/serve.go b/cmd/serve.go index d60ad9f..998ff36 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -4,13 +4,14 @@ package cmd import ( + "encoding/json" "errors" "fmt" "net/http" "os" + "github.com/OpenCHAMI/configurator/internal/generator" "github.com/OpenCHAMI/configurator/internal/server" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -18,13 +19,41 @@ var serveCmd = &cobra.Command{ Use: "serve", Short: "Start configurator as a server and listen for requests", Run: func(cmd *cobra.Command, args []string) { + // use config plugins if none supplied via CLI + if len(pluginPaths) <= 0 { + pluginPaths = append(pluginPaths, config.PluginDirs...) + } + + // show config as JSON and generators if verbose + if verbose { + b, err := json.MarshalIndent(config, "", " ") + if err != nil { + fmt.Printf("failed to marshal config: %v\n", err) + } + fmt.Printf("%v\n", string(b)) + } + // set up the routes and start the server - server := server.New() - err := server.Start(&config) + server := server.Server{ + Server: &http.Server{ + Addr: fmt.Sprintf("%s:%d", config.Server.Host, config.Server.Port), + }, + Jwks: server.Jwks{ + Uri: config.Server.Jwks.Uri, + Retries: config.Server.Jwks.Retries, + }, + GeneratorParams: generator.Params{ + Args: args, + PluginPaths: pluginPaths, + // Target: target, // NOTE: targets are set via HTTP requests (ex: curl http://configurator:3334/generate?target=dnsmasq) + Verbose: verbose, + }, + } + err := server.Serve(&config) if errors.Is(err, http.ErrServerClosed) { fmt.Printf("Server closed.") } else if err != nil { - logrus.Errorf("failed to start server: %v", err) + fmt.Errorf("failed to start server: %v", err) os.Exit(1) } }, @@ -33,7 +62,8 @@ var serveCmd = &cobra.Command{ func init() { serveCmd.Flags().StringVar(&config.Server.Host, "host", config.Server.Host, "set the server host") serveCmd.Flags().IntVar(&config.Server.Port, "port", config.Server.Port, "set the server port") - serveCmd.Flags().StringVar(&config.Options.JwksUri, "jwks-uri", config.Options.JwksUri, "set the JWKS url to fetch public key") - serveCmd.Flags().IntVar(&config.Options.JwksRetries, "jwks-fetch-retries", config.Options.JwksRetries, "set the JWKS fetch retry count") + serveCmd.Flags().StringSliceVar(&pluginPaths, "plugins", nil, "set the generator plugins directory path") + serveCmd.Flags().StringVar(&config.Server.Jwks.Uri, "jwks-uri", config.Server.Jwks.Uri, "set the JWKS url to fetch public key") + serveCmd.Flags().IntVar(&config.Server.Jwks.Retries, "jwks-fetch-retries", config.Server.Jwks.Retries, "set the JWKS fetch retry count") rootCmd.AddCommand(serveCmd) } diff --git a/internal/client.go b/internal/client.go index 68038d9..4b750c6 100644 --- a/internal/client.go +++ b/internal/client.go @@ -11,7 +11,7 @@ import ( ) type SmdClient struct { - http.Client + http.Client `json:"-"` Host string `yaml:"host"` Port int `yaml:"port"` AccessToken string `yaml:"access-token"` @@ -20,7 +20,7 @@ type SmdClient struct { type Params = map[string]any type Option func(Params) -func WithVerbose() Option { +func WithVerbosity() Option { return func(p util.Params) { p["verbose"] = true } @@ -35,6 +35,11 @@ func NewParams() Params { // Fetch the ethernet interfaces from SMD service using its API. An access token may be required if the SMD // service SMD_JWKS_URL envirnoment variable is set. func (client *SmdClient) FetchEthernetInterfaces(opts ...util.Option) ([]EthernetInterface, error) { + var ( + params = util.GetParams(opts...) + verbose = util.Get[bool](params, "verbose") + eths = []EthernetInterface{} + ) // make request to SMD endpoint b, err := client.makeRequest("/Inventory/EthernetInterfaces") if err != nil { @@ -42,16 +47,14 @@ func (client *SmdClient) FetchEthernetInterfaces(opts ...util.Option) ([]Etherne } // unmarshal response body JSON and extract in object - eths := []EthernetInterface{} // []map[string]any{} err = json.Unmarshal(b, ðs) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) } // print what we got if verbose is set - params := util.GetParams(opts...) - if verbose, ok := params["verbose"].(bool); ok { - if verbose { + if verbose != nil { + if *verbose { fmt.Printf("Ethernet Interfaces: %v\n", string(b)) } } @@ -62,23 +65,26 @@ func (client *SmdClient) FetchEthernetInterfaces(opts ...util.Option) ([]Etherne // Fetch the components from SMD using its API. An access token may be required if the SMD // service SMD_JWKS_URL envirnoment variable is set. func (client *SmdClient) FetchComponents(opts ...util.Option) ([]Component, error) { + var ( + params = util.GetParams(opts...) + verbose = util.Get[bool](params, "verbose") + comps = []Component{} + ) // make request to SMD endpoint b, err := client.makeRequest("/State/Components") if err != nil { - return nil, fmt.Errorf("failed to read HTTP response: %v", err) + return nil, fmt.Errorf("failed to make HTTP request: %v", err) } // unmarshal response body JSON and extract in object - comps := []Component{} err = json.Unmarshal(b, &comps) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) } // print what we got if verbose is set - params := util.GetParams(opts...) - if verbose, ok := params["verbose"].(bool); ok { - if verbose { + if verbose != nil { + if *verbose { fmt.Printf("Components: %v\n", string(b)) } } @@ -86,6 +92,32 @@ func (client *SmdClient) FetchComponents(opts ...util.Option) ([]Component, erro return comps, nil } +func (client *SmdClient) FetchRedfishEndpoints(opts ...util.Option) ([]RedfishEndpoint, error) { + var ( + params = util.GetParams(opts...) + verbose = util.Get[bool](params, "verbose") + rfs = []RedfishEndpoint{} + ) + + b, err := client.makeRequest("/Inventory/RedfishEndpoints") + if err != nil { + return nil, fmt.Errorf("failed to make HTTP resquest: %v", err) + } + + err = json.Unmarshal(b, &rfs) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + + if verbose != nil { + if *verbose { + fmt.Printf("Redfish endpoints: %v\n", string(b)) + } + } + + return rfs, nil +} + func (client *SmdClient) makeRequest(endpoint string) ([]byte, error) { if client == nil { return nil, fmt.Errorf("client is nil") diff --git a/internal/config.go b/internal/config.go index 7aabb0a..f8cdaa4 100644 --- a/internal/config.go +++ b/internal/config.go @@ -27,7 +27,7 @@ type Config struct { SmdClient SmdClient `yaml:"smd"` AccessToken string `yaml:"access-token"` TemplatePaths map[string]string `yaml:"templates"` - Plugins []string `yaml:"plugins"` + PluginDirs []string `yaml:"plugins"` Options Options `yaml:"options"` } @@ -45,7 +45,7 @@ func NewConfig() Config { "powerman": "templates/powerman.jinja", "conman": "templates/conman.jinja", }, - Plugins: []string{}, + PluginDirs: []string{}, Server: Server{ Host: "127.0.0.1", Port: 3334, diff --git a/internal/configurator.go b/internal/configurator.go index a20d5e3..68dae99 100644 --- a/internal/configurator.go +++ b/internal/configurator.go @@ -1,5 +1,7 @@ package configurator +import "encoding/json" + type IPAddr struct { IpAddress string `json:"IPAddress"` Network string `json:"Network"` @@ -16,6 +18,38 @@ type EthernetInterface struct { } type Component struct { + ID string `json:"ID"` + Type string `json:"Type"` + State string `json:"State,omitempty"` + Flag string `json:"Flag,omitempty"` + Enabled *bool `json:"Enabled,omitempty"` + SwStatus string `json:"SoftwareStatus,omitempty"` + Role string `json:"Role,omitempty"` + SubRole string `json:"SubRole,omitempty"` + NID json.Number `json:"NID,omitempty"` + Subtype string `json:"Subtype,omitempty"` + NetType string `json:"NetType,omitempty"` + Arch string `json:"Arch,omitempty"` + Class string `json:"Class,omitempty"` + ReservationDisabled bool `json:"ReservationDisabled,omitempty"` + Locked bool `json:"Locked,omitempty"` +} + +type RedfishEndpoint struct { + ID string `json:"ID"` + Type string `json:"Type"` + Name string `json:"Name,omitempty"` // user supplied descriptive name + Hostname string `json:"Hostname"` + Domain string `json:"Domain"` + FQDN string `json:"FQDN"` + Enabled bool `json:"Enabled"` + UUID string `json:"UUID,omitempty"` + User string `json:"User"` + Password string `json:"Password"` // Temporary until more secure method + UseSSDP bool `json:"UseSSDP,omitempty"` + MACRequired bool `json:"MACRequired,omitempty"` + MACAddr string `json:"MACAddr,omitempty"` + IPAddr string `json:"IPAddress,omitempty"` } type Node struct { diff --git a/internal/generator/generator.go b/internal/generator/generator.go index fabfe0c..ef41281 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -3,6 +3,7 @@ package generator import ( "bytes" "fmt" + "maps" "os" "plugin" @@ -10,6 +11,7 @@ import ( "github.com/OpenCHAMI/configurator/internal/util" "github.com/nikolalohinski/gonja/v2" "github.com/nikolalohinski/gonja/v2/exec" + "github.com/sirupsen/logrus" ) type Mappings = map[string]any @@ -19,6 +21,13 @@ type Generator interface { Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) } +type Params struct { + Args []string + PluginPaths []string + Target string + Verbose bool +} + func LoadPlugin(path string) (Generator, error) { p, err := plugin.Open(path) if err != nil { @@ -45,53 +54,33 @@ func LoadPlugins(dirpath string, opts ...util.Option) (map[string]Generator, err ) items, _ := os.ReadDir(dirpath) - var LoadGenerator = func(path string) (Generator, error) { - // load each generator plugin - p, err := plugin.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to load plugin: %v", err) - } - - // lookup symbol in plugin - symbol, err := p.Lookup("Generator") - if err != nil { - return nil, fmt.Errorf("failed to look up symbol: %v", err) - } - - // assert that the loaded symbol is the correct type - gen, ok := symbol.(Generator) - if !ok { - return nil, fmt.Errorf("failed to load the correct symbol type") - } - return gen, nil - } for _, item := range items { if item.IsDir() { subitems, _ := os.ReadDir(item.Name()) for _, subitem := range subitems { if !subitem.IsDir() { - gen, err := LoadGenerator(subitem.Name()) + gen, err := LoadPlugin(subitem.Name()) if err != nil { fmt.Printf("failed to load generator in directory '%s': %v\n", item.Name(), err) continue } if verbose, ok := params["verbose"].(bool); ok { if verbose { - fmt.Printf("found plugin '%s'\n", item.Name()) + fmt.Printf("-- found plugin '%s'\n", item.Name()) } } gens[gen.GetName()] = gen } } } else { - gen, err := LoadGenerator(dirpath + item.Name()) + gen, err := LoadPlugin(dirpath + item.Name()) if err != nil { fmt.Printf("failed to load generator: %v\n", err) continue } if verbose, ok := params["verbose"].(bool); ok { if verbose { - fmt.Printf("found plugin '%s'\n", dirpath+item.Name()) + fmt.Printf("-- found plugin '%s'\n", dirpath+item.Name()) } } gens[gen.GetName()] = gen @@ -123,17 +112,15 @@ func WithClient(client configurator.SmdClient) util.Option { } } -// Syntactic sugar generic function to get parameter from util.Params. -func Get[T any](params util.Params, key string) *T { - if v, ok := params[key].(T); ok { - return &v +func WithOption(key string, value any) util.Option { + return func(p util.Params) { + p[key] = value } - return nil } // Helper function to get client in generator plugins. func GetClient(params util.Params) *configurator.SmdClient { - return Get[configurator.SmdClient](params, "client") + return util.Get[configurator.SmdClient](params, "client") } func GetParams(opts ...util.Option) util.Params { @@ -144,10 +131,6 @@ func GetParams(opts ...util.Option) util.Params { return params } -func Generate(g Generator, config *configurator.Config, opts ...util.Option) { - g.Generate(config, opts...) -} - func ApplyTemplate(path string, mappings map[string]any) ([]byte, error) { data := exec.NewContext(mappings) @@ -165,3 +148,71 @@ func ApplyTemplate(path string, mappings map[string]any) ([]byte, error) { return b.Bytes(), nil } + +func Generate(config *configurator.Config, params Params) ([]byte, error) { + // load generator plugins to generate configs or to print + var ( + generators = make(map[string]Generator) + client = configurator.SmdClient{ + Host: config.SmdClient.Host, + Port: config.SmdClient.Port, + AccessToken: config.AccessToken, + } + ) + + // make sure that we have a token present before trying to make request + if config.AccessToken == "" { + // TODO: make request to check if request will need token + + // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead + accessToken := os.Getenv("ACCESS_TOKEN") + if accessToken != "" { + config.AccessToken = accessToken + } else { + // TODO: try and fetch token first if it is needed + if params.Verbose { + fmt.Printf("No token found. Attempting to generate config without one...\n") + } + } + } + + // load all plugins from params + for _, path := range params.PluginPaths { + if params.Verbose { + fmt.Printf("loading plugins from '%s'\n", path) + } + gens, err := LoadPlugins(path) + if err != nil { + fmt.Printf("failed to load plugins: %v\n", err) + err = nil + continue + } + + // add loaded generator plugins to set + maps.Copy(generators, gens) + } + + // show available targets then exit + if len(params.Args) == 0 && params.Target == "" { + for g := range generators { + fmt.Printf("-- found generator plugin \"%s\"\n", g) + } + return nil, nil + } + + if params.Target == "" { + logrus.Errorf("no target supplied (--target name)") + } else { + // run the generator plugin from target passed + gen := generators[params.Target] + if gen == nil { + return nil, fmt.Errorf("invalid generator target (%s)", params.Target) + } + return gen.Generate( + config, + WithTemplate(gen.GetName()), + WithClient(client), + ) + } + return nil, fmt.Errorf("an unknown error has occurred") +} diff --git a/internal/server/server.go b/internal/server/server.go index 5f57073..b5ce16b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -9,6 +9,7 @@ import ( "time" configurator "github.com/OpenCHAMI/configurator/internal" + "github.com/OpenCHAMI/configurator/internal/generator" "github.com/OpenCHAMI/jwtauth/v5" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -19,9 +20,15 @@ var ( tokenAuth *jwtauth.JWTAuth = nil ) +type Jwks struct { + Uri string + Retries int +} type Server struct { *http.Server - JwksUri string `yaml:"jwks-uri"` + Jwks Jwks `yaml:"jwks"` + GeneratorParams generator.Params + TokenAuth *jwtauth.JWTAuth } func New() *Server { @@ -29,11 +36,14 @@ func New() *Server { Server: &http.Server{ Addr: "localhost:3334", }, - JwksUri: "", + Jwks: Jwks{ + Uri: "", + Retries: 5, + }, } } -func (s *Server) Start(config *configurator.Config) error { +func (s *Server) Serve(config *configurator.Config) error { // create client just for the server to use to fetch data from SMD _ = &configurator.SmdClient{ Host: config.SmdClient.Host, @@ -56,6 +66,12 @@ func (s *Server) Start(config *configurator.Config) error { } } + var WriteError = func(w http.ResponseWriter, format string, a ...any) { + errmsg := fmt.Sprintf(format, a...) + fmt.Printf(errmsg) + w.Write([]byte(errmsg)) + } + // create new go-chi router with its routes router := chi.NewRouter() router.Use(middleware.RedirectSlashes) @@ -67,11 +83,19 @@ func (s *Server) Start(config *configurator.Config) error { jwtauth.Authenticator(tokenAuth), ) } - r.HandleFunc("/target", func(w http.ResponseWriter, r *http.Request) { - // g := generator.Generator{ - // Type: r.URL.Query().Get("type"), - // Template: r.URL.Query().Get("template"), - // } + r.HandleFunc("/generate", func(w http.ResponseWriter, r *http.Request) { + s.GeneratorParams.Target = r.URL.Query().Get("target") + output, err := generator.Generate(config, s.GeneratorParams) + if err != nil { + WriteError(w, "failed to generate config: %v\n", err) + return + } + + _, err = w.Write(output) + if err != nil { + WriteError(w, "failed to write response: %v", err) + return + } // NOTE: we probably don't want to hardcode the types, but should do for now // if _type == "dhcp" { diff --git a/internal/util/params.go b/internal/util/params.go index cc006fc..2417083 100644 --- a/internal/util/params.go +++ b/internal/util/params.go @@ -35,3 +35,20 @@ func AssertOptionsExist(params Params, opts ...string) []string { } return foundKeys } + +func WithDefault[T any](v T) Option { + return func(p Params) { + p["default"] = v + } +} + +// Syntactic sugar generic function to get parameter from util.Params. +func Get[T any](params Params, key string, opts ...Option) *T { + if v, ok := params[key].(T); ok { + return &v + } + if defaultValue, ok := params["default"].(T); ok { + return &defaultValue + } + return nil +} From 121c7b9f9c47bcba4ebe6950d07d8cb09a742db1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 20 Jun 2024 17:09:31 -0600 Subject: [PATCH 04/40] Updated generator plugin implementations --- internal/generator/plugins/conman/conman.go | 49 ++++++++++++------- .../generator/plugins/coredhcp/coredhcp.go | 4 +- internal/generator/plugins/dnsmasq/dnsmasq.go | 10 ++-- .../generator/plugins/powerman/powerman.go | 4 +- internal/generator/plugins/syslog/syslog.go | 4 +- 5 files changed, 47 insertions(+), 24 deletions(-) diff --git a/internal/generator/plugins/conman/conman.go b/internal/generator/plugins/conman/conman.go index 1477604..c9427dc 100644 --- a/internal/generator/plugins/conman/conman.go +++ b/internal/generator/plugins/conman/conman.go @@ -1,14 +1,11 @@ package main import ( - "bytes" "fmt" configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/generator" "github.com/OpenCHAMI/configurator/internal/util" - "github.com/nikolalohinski/gonja/v2" - "github.com/nikolalohinski/gonja/v2/exec" ) type Conman struct{} @@ -22,25 +19,43 @@ func (g *Conman) GetGroups() []string { } func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { - params := generator.GetParams(opts...) var ( - template = params["template"].(string) - path = config.TemplatePaths[template] + params = generator.GetParams(opts...) + client = generator.GetClient(params) + template = params["template"].(string) // required param + path = config.TemplatePaths[template] + eps []configurator.RedfishEndpoint = nil + err error = nil + // serverOpts = "" + // globalOpts = "" + consoles = "" ) - data := exec.NewContext(map[string]any{}) - t, err := gonja.FromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %v", err) - } - output := "# ========== GENERATED BY OCHAMI CONFIGURATOR ==========\n" - output += "# ======================================================" - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %v", err) + // fetch required data from SMD to create config + if client != nil { + eps, err = client.FetchRedfishEndpoints(opts...) + if err != nil { + return nil, fmt.Errorf("failed to fetch redfish endpoints with client: %v", err) + } } - return b.Bytes(), nil + // add any additional conman or server opts + // if extraOpts, ok := params["opts"].(map[string]any); ok { + + // } + + // format output to write to config file + consoles = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" + for _, ep := range eps { + consoles += fmt.Sprintf("CONSOLE name=%s dev=ipmi:%s-bmc ipmiopts=U:%s,P:%s,W:solpayloadsize\n", ep.Name, ep.Name, ep.User, ep.Password) + } + consoles += "# =====================================================================" + + // apply template substitutions and return output as byte array + return generator.ApplyTemplate(path, generator.Mappings{ + "server_opts": "", + "global_opts": "", + }) } var Generator Conman diff --git a/internal/generator/plugins/coredhcp/coredhcp.go b/internal/generator/plugins/coredhcp/coredhcp.go index d1d0154..c2682f5 100644 --- a/internal/generator/plugins/coredhcp/coredhcp.go +++ b/internal/generator/plugins/coredhcp/coredhcp.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/util" ) @@ -16,7 +18,7 @@ func (g *CoreDhcp) GetGroups() []string { } func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { - return nil, nil + return nil, fmt.Errorf("plugin does not implement generation function") } var Generator CoreDhcp diff --git a/internal/generator/plugins/dnsmasq/dnsmasq.go b/internal/generator/plugins/dnsmasq/dnsmasq.go index c7eb39f..aa125cb 100644 --- a/internal/generator/plugins/dnsmasq/dnsmasq.go +++ b/internal/generator/plugins/dnsmasq/dnsmasq.go @@ -40,6 +40,7 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] // set all the defaults for variables var ( params = generator.GetParams(opts...) + client = generator.GetClient(params) template = params["template"].(string) // required param path = config.TemplatePaths[template] eths []configurator.EthernetInterface = nil @@ -47,7 +48,7 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] ) // if we have a client, try making the request for the ethernet interfaces - if client, ok := params["client"].(configurator.SmdClient); ok { + if client != nil { eths, err = client.FetchEthernetInterfaces(opts...) if err != nil { return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) @@ -70,7 +71,7 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] } // format output to write to config file - output := "# ========== GENERATED BY OCHAMI CONFIGURATOR ==========\n" + output := "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" for _, eth := range eths { if eth.Type == "NodeBMC" { output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n" @@ -78,11 +79,12 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n" } } - output += "# ======================================================" + output += "# =====================================================================" // apply template substitutions and return output as byte array return generator.ApplyTemplate(path, generator.Mappings{ - "hosts": output, + "name": g.GetName(), + "output": output, }) } diff --git a/internal/generator/plugins/powerman/powerman.go b/internal/generator/plugins/powerman/powerman.go index 6c2b5eb..f1444a8 100644 --- a/internal/generator/plugins/powerman/powerman.go +++ b/internal/generator/plugins/powerman/powerman.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/util" ) @@ -16,7 +18,7 @@ func (g *Powerman) GetGroups() []string { } func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { - return nil, nil + return nil, fmt.Errorf("plugin does not implement generation function") } var Generator Powerman diff --git a/internal/generator/plugins/syslog/syslog.go b/internal/generator/plugins/syslog/syslog.go index db59ea6..04387fa 100644 --- a/internal/generator/plugins/syslog/syslog.go +++ b/internal/generator/plugins/syslog/syslog.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/util" ) @@ -16,7 +18,7 @@ func (g *Syslog) GetGroups() []string { } func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { - return nil, nil + return nil, fmt.Errorf("plugin does not implement generation function") } var Generator Syslog From d1bc04865f8c8344b6569ee59aa2a390caa5f4c2 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 20 Jun 2024 17:10:16 -0600 Subject: [PATCH 05/40] Added fetch command to interact with server --- cmd/fetch.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 cmd/fetch.go diff --git a/cmd/fetch.go b/cmd/fetch.go new file mode 100644 index 0000000..110f03c --- /dev/null +++ b/cmd/fetch.go @@ -0,0 +1,54 @@ +//go:build client || all +// +build client all + +package cmd + +import ( + "fmt" + "net/http" + + "github.com/OpenCHAMI/configurator/internal/util" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var ( + remoteHost string + remotePort int +) + +var fetchCmd = &cobra.Command{ + Use: "fetch", + Short: "Fetch a config file from a remote instance of configurator", + Run: func(cmd *cobra.Command, args []string) { + // make sure a host is set + if remoteHost == "" { + logrus.Errorf("no '--host' argument set") + return + } + for _, target := range targets { + + // make a request for each target + url := fmt.Sprintf("%s:%d/generate?target=%s", remoteHost, remotePort, target) + res, body, err := util.MakeRequest(url, http.MethodGet, nil, nil) + if err != nil { + logrus.Errorf("failed to make request: %v", err) + return + } + if res != nil { + if res.StatusCode == http.StatusOK { + fmt.Printf("%s\n", string(body)) + } + } + } + }, +} + +func init() { + fetchCmd.Flags().StringVar(&remoteHost, "host", "", "set the remote configurator host") + fetchCmd.Flags().IntVar(&remotePort, "port", 3334, "set the remote configurator port") + fetchCmd.Flags().StringSliceVar(&targets, "target", nil, "set the target configs to make") + fetchCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") + + rootCmd.AddCommand(fetchCmd) +} From a9ca2de3fc2c1d728fe0b755ec51940bf455712c Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 20 Jun 2024 17:44:52 -0600 Subject: [PATCH 06/40] Updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c012873..4912528 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,7 @@ The `server` section sets the properties for running the `configurator` tool as ## Known Issues - Adds a new `OAuthClient` with every token request +- Plugins are being loaded each time a file is generated ## TODO From 88f31d7e263ee9c60b7dadee653f47a9d9c3a4e0 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 20 Jun 2024 17:46:03 -0600 Subject: [PATCH 07/40] Fixed issues with using access token incorrectly --- cmd/generate.go | 16 ++++++++++++++++ cmd/serve.go | 16 ++++++++++++++++ internal/client.go | 32 +++++++++++++++++++++++++++++--- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index d9b7fa0..ef9a666 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -32,6 +32,22 @@ var generateCmd = &cobra.Command{ } } + // make sure that we have a token present before trying to make request + if config.AccessToken == "" { + // TODO: make request to check if request will need token + + // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead + accessToken := os.Getenv("ACCESS_TOKEN") + if accessToken != "" { + config.AccessToken = accessToken + } else { + // TODO: try and fetch token first if it is needed + if verbose { + fmt.Printf("No token found. Attempting to generate config without one...\n") + } + } + } + // use config plugins if none supplied via CLI if len(pluginPaths) <= 0 { pluginPaths = append(pluginPaths, config.PluginDirs...) diff --git a/cmd/serve.go b/cmd/serve.go index 998ff36..e150b34 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -19,6 +19,22 @@ var serveCmd = &cobra.Command{ Use: "serve", Short: "Start configurator as a server and listen for requests", Run: func(cmd *cobra.Command, args []string) { + // make sure that we have a token present before trying to make request + if config.AccessToken == "" { + // TODO: make request to check if request will need token + + // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead + accessToken := os.Getenv("ACCESS_TOKEN") + if accessToken != "" { + config.AccessToken = accessToken + } else { + // TODO: try and fetch token first if it is needed + if verbose { + fmt.Printf("No token found. Attempting to generate config without one...\n") + } + } + } + // use config plugins if none supplied via CLI if len(pluginPaths) <= 0 { pluginPaths = append(pluginPaths, config.PluginDirs...) diff --git a/internal/client.go b/internal/client.go index 4b750c6..42a7e6e 100644 --- a/internal/client.go +++ b/internal/client.go @@ -76,7 +76,21 @@ func (client *SmdClient) FetchComponents(opts ...util.Option) ([]Component, erro return nil, fmt.Errorf("failed to make HTTP request: %v", err) } + // make sure our response is actually JSON + if !json.Valid(b) { + return nil, fmt.Errorf("expected valid JSON response: %v", string(b)) + } + // unmarshal response body JSON and extract in object + var tmp map[string]any + err = json.Unmarshal(b, &tmp) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + b, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) + if err != nil { + return nil, fmt.Errorf("failed to marshal JSON: %v", err) + } err = json.Unmarshal(b, &comps) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) @@ -96,15 +110,27 @@ func (client *SmdClient) FetchRedfishEndpoints(opts ...util.Option) ([]RedfishEn var ( params = util.GetParams(opts...) verbose = util.Get[bool](params, "verbose") - rfs = []RedfishEndpoint{} + eps = []RedfishEndpoint{} ) b, err := client.makeRequest("/Inventory/RedfishEndpoints") if err != nil { return nil, fmt.Errorf("failed to make HTTP resquest: %v", err) } + if !json.Valid(b) { + return nil, fmt.Errorf("expected valid JSON response: %v", string(b)) + } + var tmp map[string]any + err = json.Unmarshal(b, &tmp) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } - err = json.Unmarshal(b, &rfs) + b, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) + if err != nil { + return nil, fmt.Errorf("failed to marshal JSON: %v", err) + } + err = json.Unmarshal(b, &eps) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) } @@ -115,7 +141,7 @@ func (client *SmdClient) FetchRedfishEndpoints(opts ...util.Option) ([]RedfishEn } } - return rfs, nil + return eps, nil } func (client *SmdClient) makeRequest(endpoint string) ([]byte, error) { From c4c52d111126e3398e68377b61f23efa95ca1fdf Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 20 Jun 2024 17:46:47 -0600 Subject: [PATCH 08/40] Fixed issue with unmarshaling certain JSON responses --- internal/generator/generator.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/internal/generator/generator.go b/internal/generator/generator.go index ef41281..427a475 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -160,22 +160,6 @@ func Generate(config *configurator.Config, params Params) ([]byte, error) { } ) - // make sure that we have a token present before trying to make request - if config.AccessToken == "" { - // TODO: make request to check if request will need token - - // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead - accessToken := os.Getenv("ACCESS_TOKEN") - if accessToken != "" { - config.AccessToken = accessToken - } else { - // TODO: try and fetch token first if it is needed - if params.Verbose { - fmt.Printf("No token found. Attempting to generate config without one...\n") - } - } - } - // load all plugins from params for _, path := range params.PluginPaths { if params.Verbose { From 235e6178de028e26e1e10273af24bcedd6c6dd8b Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 20 Jun 2024 17:47:34 -0600 Subject: [PATCH 09/40] Updated .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0bb9720..2455759 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ **configurator** **.yaml **.yml -**.so \ No newline at end of file +**.so +**.conf From fad5dcb8a9812440cd3933ff2776317dc64c05c1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 10:10:50 -0600 Subject: [PATCH 10/40] Updated Makefile plguins and added clean rule --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 2a69980..84b094c 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,15 @@ plugins: mkdir -p lib go build -buildmode=plugin -o lib/conman.so internal/generator/plugins/conman/conman.go go build -buildmode=plugin -o lib/coredhcp.so internal/generator/plugins/coredhcp/coredhcp.go + go build -buildmode=plugin -o lib/dhcpd.so internal/generator/plugins/dhcpd/dhcpd.go go build -buildmode=plugin -o lib/dnsmasq.so internal/generator/plugins/dnsmasq/dnsmasq.go + go build -buildmode=plugin -o lib/hostfile.so internal/generator/plugins/hostfile/hostfile.go go build -buildmode=plugin -o lib/powerman.so internal/generator/plugins/powerman/powerman.go go build -buildmode=plugin -o lib/syslog.so internal/generator/plugins/syslog/syslog.go + go build -buildmode=plugin -o lib/warewulf.so internal/generator/plugins/warewulf/warewulf.go + +# remove executable and all plugins +clean: + rm configurator + rm lib/* + From 2a2185ded0932ba57e12e262b2db2a54583e57ff Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 10:11:16 -0600 Subject: [PATCH 11/40] Updated README.md --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4912528..9422771 100644 --- a/README.md +++ b/README.md @@ -71,11 +71,7 @@ func (g *MyGenerator) GetName() string { return "my-generator" } -func (g *MyGenerator) GetGroups() []string { - return []string{ "my-generator" } -} - -func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { +func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { // do config generation stuff here... var ( params = generator.GetParams(opts...) @@ -103,7 +99,7 @@ Finally, build the plugin and put it somewhere specified by `plugins` in your co go build -buildmode=plugin -o lib/mygenerator.so path/to/mygenerator.go ``` -Now your plugin should be available to use with the `configurator` main driver. +Now your plugin should be available to use with the `configurator` main driver. If you get an error about not loading the correct symbol type, make sure that you generator function definitions match the `Generator` interface exactly. ## Configuration From 926062124a7211eb761559187b3c833dbfcdf1b0 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 10:14:40 -0600 Subject: [PATCH 12/40] Refactored generator and config --- cmd/generate.go | 40 ++++++++++++------ internal/config.go | 40 ++++++++++++------ internal/config.yaml | 56 ------------------------ internal/generator/generator.go | 75 ++++++++++++++++++++++++--------- 4 files changed, 108 insertions(+), 103 deletions(-) delete mode 100644 internal/config.yaml diff --git a/cmd/generate.go b/cmd/generate.go index ef9a666..cdc49e7 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -10,6 +10,7 @@ import ( "path/filepath" "github.com/OpenCHAMI/configurator/internal/generator" + "github.com/OpenCHAMI/configurator/internal/util" "github.com/spf13/cobra" ) @@ -70,29 +71,42 @@ var generateCmd = &cobra.Command{ Target: target, Verbose: verbose, } - output, err := generator.Generate(&config, params) + outputBytes, err := generator.Generate(&config, params) if err != nil { fmt.Printf("failed to generate config: %v\n", err) os.Exit(1) } - // write config output if no specific targetPath is set + outputMap := util.ConvertMapOutput(outputBytes) + // b, err := json.Marshal(outputMap) + // if err != nil { + // fmt.Printf("failed to marshal output: %v\n", err) + // os.Exit(1) + // } if outputPath == "" { - // write only to stdout - fmt.Printf("%s\n", string(output)) - } else if outputPath != "" && targetCount == 1 { + // write only to stdout by default + for _, contents := range outputMap { + fmt.Printf("%s\n", string(contents)) + } + } else if outputPath != "" && targetCount == 1 && len(outputMap) == 1 { // write just a single file using template name - err := os.WriteFile(outputPath, output, 0o644) - if err != nil { - fmt.Printf("failed to write config to file: %v", err) - os.Exit(1) + for _, contents := range outputBytes { + // FIXME: fix output paths to not overwrite each other with multiple templates + err := os.WriteFile(outputPath, contents, 0o644) + if err != nil { + fmt.Printf("failed to write config to file: %v", err) + os.Exit(1) + } } } else if outputPath != "" && targetCount > 1 { // write multiple files in directory using template name - err := os.WriteFile(fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, ".conf"), output, 0o644) - if err != nil { - fmt.Printf("failed to write config to file: %v", err) - os.Exit(1) + for _, contents := range outputBytes { + // FIXME: fix output paths to not overwrite each other with multiple templates + err := os.WriteFile(fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, ".conf"), contents, 0o644) + if err != nil { + fmt.Printf("failed to write config to file: %v", err) + os.Exit(1) + } } } } diff --git a/internal/config.go b/internal/config.go index f8cdaa4..5a7b653 100644 --- a/internal/config.go +++ b/internal/config.go @@ -10,6 +10,12 @@ import ( type Options struct{} +type Target struct { + Templates []string `yaml:"templates,omitempty"` + FilePaths []string `yaml:"files,omitempty"` + RunTargets []string `yaml:"targets,omitempty"` +} + type Jwks struct { Uri string `yaml:"uri"` Retries int `yaml:"retries"` @@ -22,13 +28,13 @@ type Server struct { } type Config struct { - Version string `yaml:"version"` - Server Server `yaml:"server"` - SmdClient SmdClient `yaml:"smd"` - AccessToken string `yaml:"access-token"` - TemplatePaths map[string]string `yaml:"templates"` - PluginDirs []string `yaml:"plugins"` - Options Options `yaml:"options"` + Version string `yaml:"version"` + Server Server `yaml:"server"` + SmdClient SmdClient `yaml:"smd"` + AccessToken string `yaml:"access-token"` + Targets map[string]Target `yaml:"targets"` + PluginDirs []string `yaml:"plugins"` + Options Options `yaml:"options"` } func NewConfig() Config { @@ -38,13 +44,21 @@ func NewConfig() Config { Host: "http://127.0.0.1", Port: 27779, }, - TemplatePaths: map[string]string{ - "dnsmasq": "templates/dnsmasq.jinja", - "syslog": "templates/syslog.jinja", - "ansible": "templates/ansible.jinja", - "powerman": "templates/powerman.jinja", - "conman": "templates/conman.jinja", + Targets: map[string]Target{ + "dnsmasq": Target{ + Templates: []string{}, + }, + "conman": Target{ + Templates: []string{}, + }, + "warewulf": Target{ + Templates: []string{ + "templates/warewulf/defaults/node.jinja", + "templates/warewulf/defaults/provision.jinja", + }, + }, }, + PluginDirs: []string{}, Server: Server{ Host: "127.0.0.1", diff --git a/internal/config.yaml b/internal/config.yaml deleted file mode 100644 index 48308d3..0000000 --- a/internal/config.yaml +++ /dev/null @@ -1,56 +0,0 @@ -version: "0.0.1" -server: - host: "127.0.0.1" - port: 3333 - callback: "/oidc/callback" - -providers: - facebook: "http://facebook.com" - forgejo: "http://git.towk.local:3000" - gitlab: "https://gitlab.newmexicoconsortium.org" - github: "https://github.com" - -authentication: - clients: - - id: "7527e7b4-c96a-4df0-8fc5-00fde18bb65d" - secret: "gto_cc5uvpb5lsdczkwnbarvwmbpv5kcjwg7nhbc75zt65yrfh2ldenq" - name: "forgejo" - issuer: "http://git.towk.local:3000" - scope: - - "openid" - - "profile" - - "read" - - "email" - redirect-uris: - - "http://127.0.0.1:3333/oidc/callback" - - id: "7c0fab1153674a258a705976fcb9468350df3addd91de4ec622fc9ed24bfbcdd" - secret: "a9a8bc55b0cd99236756093adc00ab17855fa507ce106b8038e7f9390ef2ad99" - name: "gitlab" - issuer: "http://gitlab.newmexicoconsortium.org" - scope: - - "openid" - - "profile" - - "email" - redirect-uris: - - "http://127.0.0.1:3333/oidc/callback" - flows: - authorization-code: - state: "" - client-credentials: - -authorization: - urls: - #identities: http://127.0.0.1:4434/admin/identities - trusted-issuers: http://127.0.0.1:4445/admin/trust/grants/jwt-bearer/issuers - login: http://127.0.0.1:4433/self-service/login/api - clients: http://127.0.0.1:4445/admin/clients - authorize: http://127.0.0.1:4444/oauth2/auth - register: http://127.0.0.1:4444/oauth2/register - token: http://127.0.0.1:4444/oauth2/token - - -options: - decode-id-token: true - decode-access-token: true - run-once: true - open-browser: false diff --git a/internal/generator/generator.go b/internal/generator/generator.go index 427a475..a12acdf 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -5,6 +5,7 @@ import ( "fmt" "maps" "os" + "path/filepath" "plugin" configurator "github.com/OpenCHAMI/configurator/internal" @@ -15,10 +16,10 @@ import ( ) type Mappings = map[string]any +type Files = map[string][]byte type Generator interface { GetName() string - GetGroups() []string - Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) + Generate(config *configurator.Config, opts ...util.Option) (Files, error) } type Params struct { @@ -34,14 +35,16 @@ func LoadPlugin(path string) (Generator, error) { return nil, fmt.Errorf("failed to load plugin: %v", err) } + // load the "Generator" symbol from plugin symbol, err := p.Lookup("Generator") if err != nil { - return nil, fmt.Errorf("failed to look up symbol: %v", err) + return nil, fmt.Errorf("failed to look up symbol at path '%s': %v", path, err) } + // assert that the plugin loaded has a valid generator gen, ok := symbol.(Generator) if !ok { - return nil, fmt.Errorf("failed to load the correct symbol type") + return nil, fmt.Errorf("failed to load the correct symbol type at path '%s'", path) } return gen, nil } @@ -90,10 +93,10 @@ func LoadPlugins(dirpath string, opts ...util.Option) (map[string]Generator, err return gens, nil } -func WithTemplate(_template string) util.Option { +func WithTarget(target string) util.Option { return func(p util.Params) { if p != nil { - p["template"] = _template + p["target"] = target } } } @@ -123,6 +126,10 @@ func GetClient(params util.Params) *configurator.SmdClient { return util.Get[configurator.SmdClient](params, "client") } +func GetTarget(config *configurator.Config, key string) configurator.Target { + return config.Targets[key] +} + func GetParams(opts ...util.Option) util.Params { params := util.Params{} for _, opt := range opts { @@ -131,25 +138,51 @@ func GetParams(opts ...util.Option) util.Params { return params } -func ApplyTemplate(path string, mappings map[string]any) ([]byte, error) { - data := exec.NewContext(mappings) +func ApplyTemplates(mappings map[string]any, paths ...string) (Files, error) { + var ( + data = exec.NewContext(mappings) + outputs = Files{} + ) - // load jinja template from file - t, err := gonja.FromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %v", err) + for _, path := range paths { + // load jinja template from file + t, err := gonja.FromFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read template from file: %v", err) + } + + // execute/render jinja template + b := bytes.Buffer{} + if err = t.Execute(&b, data); err != nil { + return nil, fmt.Errorf("failed to execute: %v", err) + } + outputs[path] = b.Bytes() } - // execute/render jinja template - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %v", err) - } - - return b.Bytes(), nil + return outputs, nil } -func Generate(config *configurator.Config, params Params) ([]byte, error) { +func LoadFiles(paths ...string) (Files, error) { + var outputs = Files{} + for _, path := range paths { + expandedPaths, err := filepath.Glob(path) + if err != nil { + return nil, fmt.Errorf("failed to expand path: %v", err) + } + for _, expandedPath := range expandedPaths { + b, err := os.ReadFile(expandedPath) + if err != nil { + return nil, fmt.Errorf("failed to read file: %v", err) + } + + outputs[path] = b + } + } + + return outputs, nil +} + +func Generate(config *configurator.Config, params Params) (Files, error) { // load generator plugins to generate configs or to print var ( generators = make(map[string]Generator) @@ -194,7 +227,7 @@ func Generate(config *configurator.Config, params Params) ([]byte, error) { } return gen.Generate( config, - WithTemplate(gen.GetName()), + WithTarget(gen.GetName()), WithClient(client), ) } From 398031790ef23f28fb23c1de70df9b3be1a1c92b Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 10:15:31 -0600 Subject: [PATCH 13/40] Added util function --- internal/util/util.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/util/util.go b/internal/util/util.go index cc37bec..3faf93d 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -26,7 +26,7 @@ func MakeRequest(url string, httpMethod string, body []byte, headers map[string] if err != nil { return nil, nil, fmt.Errorf("could not create new HTTP request: %v", err) } - req.Header.Add("User-Agent", "magellan") + req.Header.Add("User-Agent", "configurator") for k, v := range headers { req.Header.Add(k, v) } @@ -41,3 +41,11 @@ func MakeRequest(url string, httpMethod string, body []byte, headers map[string] } return res, b, err } + +func ConvertMapOutput(m map[string][]byte) map[string]string { + n := make(map[string]string, len(m)) + for k, v := range m { + n[k] = string(v) + } + return n +} From 56be39ff99264c8fdf70b0146badd380db722895 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 10:15:54 -0600 Subject: [PATCH 14/40] Refactor server implementation --- internal/server/server.go | 44 ++++++++++++++------------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index b5ce16b..bf7534d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -4,6 +4,7 @@ package server import ( + "encoding/json" "fmt" "net/http" "time" @@ -85,42 +86,29 @@ func (s *Server) Serve(config *configurator.Config) error { } r.HandleFunc("/generate", func(w http.ResponseWriter, r *http.Request) { s.GeneratorParams.Target = r.URL.Query().Get("target") - output, err := generator.Generate(config, s.GeneratorParams) + outputs, err := generator.Generate(config, s.GeneratorParams) if err != nil { - WriteError(w, "failed to generate config: %v\n", err) + WriteError(w, "failed to generate config: %v", err) return } - _, err = w.Write(output) + // convert byte arrays to string + tmp := map[string]string{} + for path, output := range outputs { + tmp[path] = string(output) + } + + // marshal output to JSON then send + b, err := json.Marshal(tmp) + if err != nil { + WriteError(w, "failed to marshal output: %v", err) + return + } + _, err = w.Write(b) if err != nil { WriteError(w, "failed to write response: %v", err) return } - - // NOTE: we probably don't want to hardcode the types, but should do for now - // if _type == "dhcp" { - // // fetch eths from SMD - // eths, err := client.FetchEthernetInterfaces() - // if err != nil { - // logrus.Errorf("failed to fetch DHCP metadata: %v\n", err) - // w.Write([]byte("An error has occurred")) - // return - // } - // if len(eths) <= 0 { - // logrus.Warnf("no ethernet interfaces found") - // w.Write([]byte("no ethernet interfaces found")) - // return - // } - // // generate a new config from that data - - // // b, err := g.GenerateDHCP(config, eths) - // if err != nil { - // logrus.Errorf("failed to generate DHCP: %v", err) - // w.Write([]byte("An error has occurred.")) - // return - // } - // w.Write(b) - // } }) r.HandleFunc("/templates", func(w http.ResponseWriter, r *http.Request) { // TODO: handle GET request From f443558b509374b193c4b216cf8c925526bd4d6f Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 10:17:59 -0600 Subject: [PATCH 15/40] Updated generator plugins to reflect generator changes --- internal/generator/plugins/conman/conman.go | 20 +++--- .../generator/plugins/coredhcp/coredhcp.go | 6 +- internal/generator/plugins/dnsmasq/dnsmasq.go | 38 ++++-------- .../generator/plugins/powerman/powerman.go | 2 +- internal/generator/plugins/syslog/syslog.go | 6 +- .../generator/plugins/warewulf/warewulf.go | 61 +++++++++++++------ 6 files changed, 68 insertions(+), 65 deletions(-) diff --git a/internal/generator/plugins/conman/conman.go b/internal/generator/plugins/conman/conman.go index c9427dc..450f383 100644 --- a/internal/generator/plugins/conman/conman.go +++ b/internal/generator/plugins/conman/conman.go @@ -15,17 +15,17 @@ func (g *Conman) GetName() string { } func (g *Conman) GetGroups() []string { - return []string{"conman"} + return []string{""} } -func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { +func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { var ( - params = generator.GetParams(opts...) - client = generator.GetClient(params) - template = params["template"].(string) // required param - path = config.TemplatePaths[template] - eps []configurator.RedfishEndpoint = nil - err error = nil + params = generator.GetParams(opts...) + client = generator.GetClient(params) + targetKey = params["targets"].(string) // required param + target = config.Targets[targetKey] + eps []configurator.RedfishEndpoint = nil + err error = nil // serverOpts = "" // globalOpts = "" consoles = "" @@ -52,10 +52,10 @@ func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) ([]b consoles += "# =====================================================================" // apply template substitutions and return output as byte array - return generator.ApplyTemplate(path, generator.Mappings{ + return generator.ApplyTemplates(generator.Mappings{ "server_opts": "", "global_opts": "", - }) + }, target.Templates...) } var Generator Conman diff --git a/internal/generator/plugins/coredhcp/coredhcp.go b/internal/generator/plugins/coredhcp/coredhcp.go index c2682f5..ea0f2c1 100644 --- a/internal/generator/plugins/coredhcp/coredhcp.go +++ b/internal/generator/plugins/coredhcp/coredhcp.go @@ -13,11 +13,7 @@ func (g *CoreDhcp) GetName() string { return "coredhcp" } -func (g *CoreDhcp) GetGroups() []string { - return []string{"coredhcp"} -} - -func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { +func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/dnsmasq/dnsmasq.go b/internal/generator/plugins/dnsmasq/dnsmasq.go index aa125cb..2795623 100644 --- a/internal/generator/plugins/dnsmasq/dnsmasq.go +++ b/internal/generator/plugins/dnsmasq/dnsmasq.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strings" configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/generator" @@ -10,28 +11,11 @@ import ( type DnsMasq struct{} -func TestGenerateDnsMasq() { - var ( - g = DnsMasq{} - config = &configurator.Config{} - client = configurator.SmdClient{} - ) - g.Generate( - config, - generator.WithTemplate("dnsmasq"), - generator.WithClient(client), - ) -} - func (g *DnsMasq) GetName() string { return "dnsmasq" } -func (g *DnsMasq) GetGroups() []string { - return []string{"dnsmasq"} -} - -func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { +func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { // make sure we have a valid config first if config == nil { return nil, fmt.Errorf("invalid config (config is nil)") @@ -39,12 +23,12 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] // set all the defaults for variables var ( - params = generator.GetParams(opts...) - client = generator.GetClient(params) - template = params["template"].(string) // required param - path = config.TemplatePaths[template] - eths []configurator.EthernetInterface = nil - err error = nil + params = generator.GetParams(opts...) + client = generator.GetClient(params) + targetKey = params["target"].(string) // required param + target = config.Targets[targetKey] + eths []configurator.EthernetInterface = nil + err error = nil ) // if we have a client, try making the request for the ethernet interfaces @@ -66,7 +50,7 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] // print message if verbose param found if verbose, ok := params["verbose"].(bool); ok { if verbose { - fmt.Printf("path: %s\neth count: %v\n", path, len(eths)) + fmt.Printf("template: \n%s\n ethernet interfaces found: %v\n", strings.Join(target.Templates, "\n\t"), len(eths)) } } @@ -82,10 +66,10 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] output += "# =====================================================================" // apply template substitutions and return output as byte array - return generator.ApplyTemplate(path, generator.Mappings{ + return generator.ApplyTemplates(generator.Mappings{ "name": g.GetName(), "output": output, - }) + }, target.Templates...) } var Generator DnsMasq diff --git a/internal/generator/plugins/powerman/powerman.go b/internal/generator/plugins/powerman/powerman.go index f1444a8..37263e5 100644 --- a/internal/generator/plugins/powerman/powerman.go +++ b/internal/generator/plugins/powerman/powerman.go @@ -17,7 +17,7 @@ func (g *Powerman) GetGroups() []string { return []string{"powerman"} } -func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { +func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/syslog/syslog.go b/internal/generator/plugins/syslog/syslog.go index 04387fa..a50fba3 100644 --- a/internal/generator/plugins/syslog/syslog.go +++ b/internal/generator/plugins/syslog/syslog.go @@ -13,11 +13,7 @@ func (g *Syslog) GetName() string { return "syslog" } -func (g *Syslog) GetGroups() []string { - return []string{"log"} -} - -func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { +func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/warewulf/warewulf.go b/internal/generator/plugins/warewulf/warewulf.go index 9de624b..477815f 100644 --- a/internal/generator/plugins/warewulf/warewulf.go +++ b/internal/generator/plugins/warewulf/warewulf.go @@ -1,12 +1,12 @@ package main import ( - "bytes" "fmt" + "maps" configurator "github.com/OpenCHAMI/configurator/internal" - "github.com/nikolalohinski/gonja/v2" - "github.com/nikolalohinski/gonja/v2/exec" + "github.com/OpenCHAMI/configurator/internal/generator" + "github.com/OpenCHAMI/configurator/internal/util" ) type Warewulf struct{} @@ -19,26 +19,53 @@ func (g *Warewulf) GetGroups() []string { return []string{"warewulf"} } -func (g *Warewulf) Generate(config *configurator.Config, template string) ([]byte, error) { +func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { var ( - path = config.TemplatePaths[template] + params = generator.GetParams(opts...) + client = generator.GetClient(params) + targetKey = params["target"].(string) + target = config.Targets[targetKey] + outputs = make(generator.Files, len(target.FilePaths)+len(target.Templates)) ) - t, err := gonja.FromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %v", err) + // check if our client is included and is valid + if client == nil { + return nil, fmt.Errorf("invalid client (client is nil)") } - output := "# ========== GENERATED BY OCHAMI CONFIGURATOR ==========\n" - output += "# ======================================================" - data := exec.NewContext(map[string]any{ - "hosts": output, - }) - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %v", err) + // fetch redfish endpoints and handle errors + eps, err := client.FetchRedfishEndpoints(opts...) + if err != nil { + return nil, fmt.Errorf("failed to fetch redfish endpoints: %v", err) } - return nil, nil + if len(eps) <= 0 { + return nil, fmt.Errorf("no redfish endpoints found") + } + + // load files and templates and copy to outputs + files, err := generator.LoadFiles(target.FilePaths...) + if err != nil { + return nil, fmt.Errorf("failed to load files: %v", err) + } + templates, err := generator.ApplyTemplates(generator.Mappings{}, target.Templates...) + if err != nil { + return nil, fmt.Errorf("failed to load templates: %v", err) + } + + maps.Copy(outputs, files) + maps.Copy(outputs, templates) + + // print message if verbose param is found + if verbose, ok := params["verbose"].(bool); ok { + if verbose { + fmt.Printf("templates and files loaded: \n") + for path, _ := range outputs { + fmt.Printf("\t%s", path) + } + } + } + + return outputs, err } var Generator Warewulf From a7d78e8240d4fdea681fded5e7c766aa0c6e5ca0 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 10:19:19 -0600 Subject: [PATCH 16/40] Added DHCPD and hostfile plugins --- internal/generator/plugins/dhcpd/dhcpd.go | 65 +++++++++++++++++++ .../generator/plugins/hostfile/hostfile.go | 20 ++++++ .../plugins/hostfile/hostfile_test.go | 1 + 3 files changed, 86 insertions(+) create mode 100644 internal/generator/plugins/dhcpd/dhcpd.go create mode 100644 internal/generator/plugins/hostfile/hostfile.go create mode 100644 internal/generator/plugins/hostfile/hostfile_test.go diff --git a/internal/generator/plugins/dhcpd/dhcpd.go b/internal/generator/plugins/dhcpd/dhcpd.go new file mode 100644 index 0000000..edd09bb --- /dev/null +++ b/internal/generator/plugins/dhcpd/dhcpd.go @@ -0,0 +1,65 @@ +package main + +import ( + "fmt" + + configurator "github.com/OpenCHAMI/configurator/internal" + "github.com/OpenCHAMI/configurator/internal/generator" + "github.com/OpenCHAMI/configurator/internal/util" +) + +type Dhcpd struct{} + +func (g *Dhcpd) GetName() string { + return "dhcpd" +} + +func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { + var ( + params = generator.GetParams(opts...) + client = generator.GetClient(params) + targetKey = params["target"].(string) + target = config.Targets[targetKey] + compute_nodes = "" + eths []configurator.EthernetInterface = nil + err error = nil + ) + + // + if client != nil { + eths, err = client.FetchEthernetInterfaces(opts...) + if err != nil { + return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) + } + } + + // check if we have the required params first + if eths == nil { + return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") + } + if len(eths) <= 0 { + return nil, fmt.Errorf("no ethernet interfaces found") + } + + // format output to write to config file + compute_nodes = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" + for _, eth := range eths { + if len(eth.IpAddresses) == 0 { + continue + } + compute_nodes += fmt.Sprintf("host %s { hardware ethernet %s; fixed-address %s} ", eth.ComponentId, eth.MacAddress, eth.IpAddresses[0]) + } + compute_nodes += "# =====================================================================" + + if verbose, ok := params["verbose"].(bool); ok { + if verbose { + fmt.Printf("") + } + } + return generator.ApplyTemplates(generator.Mappings{ + "compute_nodes": compute_nodes, + "node_entries": "", + }, target.Templates...) +} + +var Generator Dhcpd diff --git a/internal/generator/plugins/hostfile/hostfile.go b/internal/generator/plugins/hostfile/hostfile.go new file mode 100644 index 0000000..ae3fba3 --- /dev/null +++ b/internal/generator/plugins/hostfile/hostfile.go @@ -0,0 +1,20 @@ +package main + +import ( + "fmt" + + configurator "github.com/OpenCHAMI/configurator/internal" + "github.com/OpenCHAMI/configurator/internal/util" +) + +type Hostfile struct{} + +func (g *Hostfile) GetName() string { + return "hostfile" +} + +func (g *Hostfile) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { + return nil, fmt.Errorf("plugin does not implement generation function") +} + +var Generator Hostfile diff --git a/internal/generator/plugins/hostfile/hostfile_test.go b/internal/generator/plugins/hostfile/hostfile_test.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/internal/generator/plugins/hostfile/hostfile_test.go @@ -0,0 +1 @@ +package main From cda5e71584b0e23e2e160ddd63b2d1e0d780d629 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 11:28:00 -0600 Subject: [PATCH 17/40] Added param for CA certs --- cmd/generate.go | 2 ++ internal/client.go | 60 +++++++++++++++++++++++++++++++++ internal/config.go | 1 + internal/generator/generator.go | 11 +++--- 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index cdc49e7..2c3a121 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -17,6 +17,7 @@ import ( var ( tokenFetchRetries int pluginPaths []string + cacertPath string ) var generateCmd = &cobra.Command{ @@ -118,6 +119,7 @@ func init() { generateCmd.Flags().StringSliceVar(&targets, "target", []string{}, "set the target configs to make") generateCmd.Flags().StringSliceVar(&pluginPaths, "plugins", []string{}, "set the generator plugins directory path") generateCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") + generateCmd.Flags().StringVar(&cacertPath, "ca-cert", "", "path to CA cert. (defaults to system CAs)") generateCmd.Flags().IntVar(&tokenFetchRetries, "fetch-retries", 5, "set the number of retries to fetch an access token") rootCmd.AddCommand(generateCmd) diff --git a/internal/client.go b/internal/client.go index 42a7e6e..1226095 100644 --- a/internal/client.go +++ b/internal/client.go @@ -2,10 +2,15 @@ package configurator import ( "bytes" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "io" + "net" "net/http" + "os" + "time" "github.com/OpenCHAMI/configurator/internal/util" ) @@ -19,6 +24,61 @@ type SmdClient struct { type Params = map[string]any type Option func(Params) +type ClientOption func(*SmdClient) + +func NewSmdClient(opts ...ClientOption) SmdClient { + client := SmdClient{} + for _, opt := range opts { + opt(&client) + } + return client +} + +func WithHost(host string) ClientOption { + return func(c *SmdClient) { + c.Host = host + } +} + +func WithPort(port int) ClientOption { + return func(c *SmdClient) { + c.Port = port + } +} + +func WithAccessToken(token string) ClientOption { + return func(c *SmdClient) { + c.AccessToken = token + } +} + +func WithCertPool(certPool *x509.CertPool) ClientOption { + return func(c *SmdClient) { + c.Client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + } +} + +func WithSecureTLS(certPath string) ClientOption { + if certPath == "" { + return func(sc *SmdClient) {} + } + cacert, _ := os.ReadFile(certPath) + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + return WithCertPool(certPool) +} func WithVerbosity() Option { return func(p util.Params) { diff --git a/internal/config.go b/internal/config.go index 5a7b653..df58cc5 100644 --- a/internal/config.go +++ b/internal/config.go @@ -34,6 +34,7 @@ type Config struct { AccessToken string `yaml:"access-token"` Targets map[string]Target `yaml:"targets"` PluginDirs []string `yaml:"plugins"` + CertPath string `yaml:"ca-cert"` Options Options `yaml:"options"` } diff --git a/internal/generator/generator.go b/internal/generator/generator.go index a12acdf..a1840a5 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -186,11 +186,12 @@ func Generate(config *configurator.Config, params Params) (Files, error) { // load generator plugins to generate configs or to print var ( generators = make(map[string]Generator) - client = configurator.SmdClient{ - Host: config.SmdClient.Host, - Port: config.SmdClient.Port, - AccessToken: config.AccessToken, - } + client = configurator.NewSmdClient( + configurator.WithHost(config.SmdClient.Host), + configurator.WithPort(config.SmdClient.Port), + configurator.WithAccessToken(config.AccessToken), + configurator.WithSecureTLS(config.CertPath), + ) ) // load all plugins from params From 30c8336ca648fc9e740770ce0694d17411d5b448 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 13:39:27 -0600 Subject: [PATCH 18/40] Added function to get commit as string --- internal/util/util.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/util/util.go b/internal/util/util.go index 3faf93d..3a990f6 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -7,6 +7,8 @@ import ( "io" "net/http" "os" + "os/exec" + "strings" ) func PathExists(path string) (bool, error) { @@ -49,3 +51,13 @@ func ConvertMapOutput(m map[string][]byte) map[string]string { } return n } + +func GitCommit() string { + c := exec.Command("git", "rev-parse", "HEAD") + stdout, err := c.Output() + if err != nil { + return "" + } + + return strings.TrimRight(string(stdout), "\n") +} From e92a883c89e9c64e6b1a3354588518a0189ea1bc Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 13:41:14 -0600 Subject: [PATCH 19/40] Update generator interface and plugins --- internal/generator/generator.go | 2 ++ internal/generator/plugins/conman/conman.go | 8 +++++ .../generator/plugins/coredhcp/coredhcp.go | 8 +++++ internal/generator/plugins/dhcpd/dhcpd.go | 8 +++++ internal/generator/plugins/dnsmasq/dnsmasq.go | 10 ++++++- .../generator/plugins/hostfile/hostfile.go | 8 +++++ .../generator/plugins/powerman/powerman.go | 8 +++-- internal/generator/plugins/syslog/syslog.go | 8 +++++ .../generator/plugins/warewulf/warewulf.go | 30 +++++++++++++++++-- 9 files changed, 85 insertions(+), 5 deletions(-) diff --git a/internal/generator/generator.go b/internal/generator/generator.go index a1840a5..45f1586 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -19,6 +19,8 @@ type Mappings = map[string]any type Files = map[string][]byte type Generator interface { GetName() string + GetVersion() string + GetDescription() string Generate(config *configurator.Config, opts ...util.Option) (Files, error) } diff --git a/internal/generator/plugins/conman/conman.go b/internal/generator/plugins/conman/conman.go index 450f383..4fcef2c 100644 --- a/internal/generator/plugins/conman/conman.go +++ b/internal/generator/plugins/conman/conman.go @@ -14,6 +14,14 @@ func (g *Conman) GetName() string { return "conman" } +func (g *Conman) GetVersion() string { + return util.GitCommit() +} + +func (g *Conman) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + func (g *Conman) GetGroups() []string { return []string{""} } diff --git a/internal/generator/plugins/coredhcp/coredhcp.go b/internal/generator/plugins/coredhcp/coredhcp.go index ea0f2c1..9b76227 100644 --- a/internal/generator/plugins/coredhcp/coredhcp.go +++ b/internal/generator/plugins/coredhcp/coredhcp.go @@ -13,6 +13,14 @@ func (g *CoreDhcp) GetName() string { return "coredhcp" } +func (g *CoreDhcp) GetVersion() string { + return util.GitCommit() +} + +func (g *CoreDhcp) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s' to generate config files. This plugin is not complete and still a WIP.", g.GetName()) +} + func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/dhcpd/dhcpd.go b/internal/generator/plugins/dhcpd/dhcpd.go index edd09bb..a56732e 100644 --- a/internal/generator/plugins/dhcpd/dhcpd.go +++ b/internal/generator/plugins/dhcpd/dhcpd.go @@ -14,6 +14,14 @@ func (g *Dhcpd) GetName() string { return "dhcpd" } +func (g *Dhcpd) GetVersion() string { + return util.GitCommit() +} + +func (g *Dhcpd) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { var ( params = generator.GetParams(opts...) diff --git a/internal/generator/plugins/dnsmasq/dnsmasq.go b/internal/generator/plugins/dnsmasq/dnsmasq.go index 2795623..b1b71a7 100644 --- a/internal/generator/plugins/dnsmasq/dnsmasq.go +++ b/internal/generator/plugins/dnsmasq/dnsmasq.go @@ -15,6 +15,14 @@ func (g *DnsMasq) GetName() string { return "dnsmasq" } +func (g *DnsMasq) GetVersion() string { + return util.GitCommit() +} + +func (g *DnsMasq) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { // make sure we have a valid config first if config == nil { @@ -50,7 +58,7 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (ma // print message if verbose param found if verbose, ok := params["verbose"].(bool); ok { if verbose { - fmt.Printf("template: \n%s\n ethernet interfaces found: %v\n", strings.Join(target.Templates, "\n\t"), len(eths)) + fmt.Printf("template: \n%s\nethernet interfaces found: %v\n", strings.Join(target.Templates, "\n\t"), len(eths)) } } diff --git a/internal/generator/plugins/hostfile/hostfile.go b/internal/generator/plugins/hostfile/hostfile.go index ae3fba3..9c66f3a 100644 --- a/internal/generator/plugins/hostfile/hostfile.go +++ b/internal/generator/plugins/hostfile/hostfile.go @@ -13,6 +13,14 @@ func (g *Hostfile) GetName() string { return "hostfile" } +func (g *Hostfile) GetVersion() string { + return util.GitCommit() +} + +func (g *Hostfile) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + func (g *Hostfile) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/powerman/powerman.go b/internal/generator/plugins/powerman/powerman.go index 37263e5..9f68dd7 100644 --- a/internal/generator/plugins/powerman/powerman.go +++ b/internal/generator/plugins/powerman/powerman.go @@ -13,8 +13,12 @@ func (g *Powerman) GetName() string { return "powerman" } -func (g *Powerman) GetGroups() []string { - return []string{"powerman"} +func (g *Powerman) GetVersion() string { + return util.GitCommit() +} + +func (g *Powerman) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { diff --git a/internal/generator/plugins/syslog/syslog.go b/internal/generator/plugins/syslog/syslog.go index a50fba3..f9caf40 100644 --- a/internal/generator/plugins/syslog/syslog.go +++ b/internal/generator/plugins/syslog/syslog.go @@ -13,6 +13,14 @@ func (g *Syslog) GetName() string { return "syslog" } +func (g *Syslog) GetVersion() string { + return util.GitCommit() +} + +func (g *Syslog) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/warewulf/warewulf.go b/internal/generator/plugins/warewulf/warewulf.go index 477815f..7999afa 100644 --- a/internal/generator/plugins/warewulf/warewulf.go +++ b/internal/generator/plugins/warewulf/warewulf.go @@ -3,6 +3,7 @@ package main import ( "fmt" "maps" + "strings" configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/generator" @@ -15,8 +16,12 @@ func (g *Warewulf) GetName() string { return "warewulf" } -func (g *Warewulf) GetGroups() []string { - return []string{"warewulf"} +func (g *Warewulf) GetVersion() string { + return util.GitCommit() +} + +func (g *Warewulf) GetDescription() string { + return "Configurator generator plugin for 'warewulf' config files." } func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { @@ -33,6 +38,27 @@ func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (g return nil, fmt.Errorf("invalid client (client is nil)") } + // if we have a client, try making the request for the ethernet interfaces + eths, err := client.FetchEthernetInterfaces(opts...) + if err != nil { + return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) + } + + // check if we have the required params first + if eths == nil { + return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") + } + if len(eths) <= 0 { + return nil, fmt.Errorf("no ethernet interfaces found") + } + + // print message if verbose param found + if verbose, ok := params["verbose"].(bool); ok { + if verbose { + fmt.Printf("template: \n%s\n ethernet interfaces found: %v\n", strings.Join(target.Templates, "\n\t"), len(eths)) + } + } + // fetch redfish endpoints and handle errors eps, err := client.FetchRedfishEndpoints(opts...) if err != nil { From 5845408d5d8eb7fc0b7fafa0c9fdfdf2b3870c1b Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 13:41:57 -0600 Subject: [PATCH 20/40] Added example generator plugin --- internal/generator/plugins/example/example.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 internal/generator/plugins/example/example.go diff --git a/internal/generator/plugins/example/example.go b/internal/generator/plugins/example/example.go new file mode 100644 index 0000000..044bcf7 --- /dev/null +++ b/internal/generator/plugins/example/example.go @@ -0,0 +1,32 @@ +package main + +import ( + "fmt" + + configurator "github.com/OpenCHAMI/configurator/internal" + "github.com/OpenCHAMI/configurator/internal/generator" + "github.com/OpenCHAMI/configurator/internal/util" +) + +type Example struct { + Message string +} + +func (g *Example) GetName() string { + return "example" +} + +func (g *Example) GetVersion() string { + return util.GitCommit() +} + +func (g *Example) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + +func (g *Example) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { + g.Message = ` + This is an example generator plugin. See the file in 'internal/generator/plugins/example/example.go' on + information about constructing plugins and plugin requirements.` + return generator.Files{"example": []byte(g.Message)}, nil +} From d2fcf10f0309627cacea7329232f0e93fcdcf04e Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 13:42:24 -0600 Subject: [PATCH 21/40] Added command to inspect generator plugins --- cmd/inspect.go | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 cmd/inspect.go diff --git a/cmd/inspect.go b/cmd/inspect.go new file mode 100644 index 0000000..e94906c --- /dev/null +++ b/cmd/inspect.go @@ -0,0 +1,64 @@ +package cmd + +import ( + "fmt" + "maps" + "strings" + + "github.com/OpenCHAMI/configurator/internal/generator" + "github.com/spf13/cobra" +) + +var ( + pluginDirs []string + generators map[string]generator.Generator +) + +var inspectCmd = &cobra.Command{ + Use: "inspect", + Short: "Inspect generator plugin information", + Run: func(cmd *cobra.Command, args []string) { + // load specific plugins from positional args + generators = make(map[string]generator.Generator) + for _, path := range args { + gen, err := generator.LoadPlugin(path) + if err != nil { + fmt.Printf("failed to load plugin at path '%s': %v\n", path, err) + continue + } + generators[path] = gen + } + + // load plugins and print all plugin details + if len(pluginDirs) > 0 { + + } else { + for _, pluginDir := range config.PluginDirs { + gens, err := generator.LoadPlugins(pluginDir) + if err != nil { + fmt.Printf("failed to load plugin: %v\n", err) + continue + } + maps.Copy(generators, gens) + } + } + + // print all generator information + const WIDTH = 40 + if len(generators) > 0 { + o := "" + for _, g := range generators { + o += fmt.Sprintf("- Name: %s\n", g.GetName()) + o += fmt.Sprintf(" Version: %s\n", g.GetVersion()) + o += fmt.Sprintf(" Description: %s\n", g.GetDescription()) + o += "\n" + } + o = strings.TrimRight(o, "\n") + fmt.Printf("%s", o) + } + }, +} + +func init() { + rootCmd.AddCommand(inspectCmd) +} From f3adc49c26d1ae8e346f73f0bedd1e13e56338c1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 14:01:53 -0600 Subject: [PATCH 22/40] Removed unused var --- cmd/inspect.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index e94906c..4ca4daf 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -44,7 +44,6 @@ var inspectCmd = &cobra.Command{ } // print all generator information - const WIDTH = 40 if len(generators) > 0 { o := "" for _, g := range generators { From 53f80fdd592f7410d186bfa4e9bd9cfe5f6b7992 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 14:02:13 -0600 Subject: [PATCH 23/40] Added small fix for CA cert path --- cmd/generate.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/generate.go b/cmd/generate.go index 2c3a121..10df8bb 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -50,6 +50,12 @@ var generateCmd = &cobra.Command{ } } + // use cert path from cobra if empty + // TODO: this needs to be checked for the correct desired behavior + if config.CertPath == "" { + config.CertPath = certPath + } + // use config plugins if none supplied via CLI if len(pluginPaths) <= 0 { pluginPaths = append(pluginPaths, config.PluginDirs...) From d3e4e32f88742d5d9e36ada9b198f3caef9037c8 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 14:02:57 -0600 Subject: [PATCH 24/40] Corrected var name for cert path --- cmd/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/generate.go b/cmd/generate.go index 10df8bb..2ef58a6 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -53,7 +53,7 @@ var generateCmd = &cobra.Command{ // use cert path from cobra if empty // TODO: this needs to be checked for the correct desired behavior if config.CertPath == "" { - config.CertPath = certPath + config.CertPath = cacertPath } // use config plugins if none supplied via CLI From 251088088f827ddc540d73fb0f77e3ee217f15da Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 14:17:09 -0600 Subject: [PATCH 25/40] Added check to skip directory paths --- internal/generator/generator.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/generator/generator.go b/internal/generator/generator.go index 45f1586..4161a81 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -172,6 +172,14 @@ func LoadFiles(paths ...string) (Files, error) { return nil, fmt.Errorf("failed to expand path: %v", err) } for _, expandedPath := range expandedPaths { + info, err := os.Stat(expandedPath) + if err != nil { + fmt.Println(err) + return nil, fmt.Errorf("failed to stat file or directory: %v", err) + } + if info.IsDir() { + continue + } b, err := os.ReadFile(expandedPath) if err != nil { return nil, fmt.Errorf("failed to read file: %v", err) From 87f35eef0f2d16a7269634a5c85d43c75d2f12e9 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 14:20:52 -0600 Subject: [PATCH 26/40] Fixed output path file name with multiple targets --- cmd/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/generate.go b/cmd/generate.go index 2ef58a6..2e973ba 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -109,7 +109,7 @@ var generateCmd = &cobra.Command{ // write multiple files in directory using template name for _, contents := range outputBytes { // FIXME: fix output paths to not overwrite each other with multiple templates - err := os.WriteFile(fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, ".conf"), contents, 0o644) + err := os.WriteFile(fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, "conf"), contents, 0o644) if err != nil { fmt.Printf("failed to write config to file: %v", err) os.Exit(1) From 9c51c0fa31204ab2e4e3b96842612bc59c8e80d1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 26 Jun 2024 16:57:59 -0600 Subject: [PATCH 27/40] Cleaned up how output is written to files --- cmd/generate.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 2e973ba..2f84a4b 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -85,18 +85,19 @@ var generateCmd = &cobra.Command{ } outputMap := util.ConvertMapOutput(outputBytes) - // b, err := json.Marshal(outputMap) - // if err != nil { - // fmt.Printf("failed to marshal output: %v\n", err) - // os.Exit(1) - // } if outputPath == "" { // write only to stdout by default - for _, contents := range outputMap { - fmt.Printf("%s\n", string(contents)) + if len(outputMap) == 1 { + for _, contents := range outputMap { + fmt.Printf("%s\n", string(contents)) + } + } else { + for path, contents := range outputMap { + fmt.Printf("-- file: %s, size: %d\n%s\n", path, len(contents), string(contents)) + } } } else if outputPath != "" && targetCount == 1 && len(outputMap) == 1 { - // write just a single file using template name + // write just a single file using provided name for _, contents := range outputBytes { // FIXME: fix output paths to not overwrite each other with multiple templates err := os.WriteFile(outputPath, contents, 0o644) @@ -104,16 +105,18 @@ var generateCmd = &cobra.Command{ fmt.Printf("failed to write config to file: %v", err) os.Exit(1) } + fmt.Printf("wrote file to '%s'\n", outputPath) } - } else if outputPath != "" && targetCount > 1 { + } else if outputPath != "" && targetCount > 1 || len(outputMap) > 1 { // write multiple files in directory using template name for _, contents := range outputBytes { - // FIXME: fix output paths to not overwrite each other with multiple templates - err := os.WriteFile(fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, "conf"), contents, 0o644) + cleanPath := fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, "conf") + err := os.WriteFile(cleanPath, contents, 0o644) if err != nil { fmt.Printf("failed to write config to file: %v", err) os.Exit(1) } + fmt.Printf("wrote file to '%s'\n", cleanPath) } } } From aea3c55dd8fa88020751875975a61ab474b027fb Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 27 Jun 2024 09:44:48 -0600 Subject: [PATCH 28/40] Fixed how multiple files are written for a target --- cmd/generate.go | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 2f84a4b..5455dcd 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -24,16 +24,6 @@ var generateCmd = &cobra.Command{ Use: "generate", Short: "Generate a config file from state management", Run: func(cmd *cobra.Command, args []string) { - // if we have more than one target and output is set, create configs in directory - targetCount := len(targets) - if outputPath != "" && targetCount > 1 { - err := os.MkdirAll(outputPath, 0o755) - if err != nil { - fmt.Printf("failed to make output directory: %v", err) - os.Exit(1) - } - } - // make sure that we have a token present before trying to make request if config.AccessToken == "" { // TODO: make request to check if request will need token @@ -85,6 +75,12 @@ var generateCmd = &cobra.Command{ } outputMap := util.ConvertMapOutput(outputBytes) + + // if we have more than one target and output is set, create configs in directory + var ( + targetCount = len(targets) + templateCount = len(outputMap) + ) if outputPath == "" { // write only to stdout by default if len(outputMap) == 1 { @@ -93,13 +89,12 @@ var generateCmd = &cobra.Command{ } } else { for path, contents := range outputMap { - fmt.Printf("-- file: %s, size: %d\n%s\n", path, len(contents), string(contents)) + fmt.Printf("-- file: %s, size: %d B\n%s\n", path, len(contents), string(contents)) } } - } else if outputPath != "" && targetCount == 1 && len(outputMap) == 1 { + } else if outputPath != "" && targetCount == 1 && templateCount == 1 { // write just a single file using provided name - for _, contents := range outputBytes { - // FIXME: fix output paths to not overwrite each other with multiple templates + for path, contents := range outputMap { err := os.WriteFile(outputPath, contents, 0o644) if err != nil { fmt.Printf("failed to write config to file: %v", err) @@ -107,10 +102,16 @@ var generateCmd = &cobra.Command{ } fmt.Printf("wrote file to '%s'\n", outputPath) } - } else if outputPath != "" && targetCount > 1 || len(outputMap) > 1 { + } else if outputPath != "" && targetCount > 1 || templateCount > 1 { // write multiple files in directory using template name - for _, contents := range outputBytes { - cleanPath := fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, "conf") + err := os.MkdirAll(filepath.Clean(outputPath), 0o755) + if err != nil { + fmt.Printf("failed to make output directory: %v", err) + os.Exit(1) + } + for path, contents := range outputBytes { + filename := filepath.Base(path) + cleanPath := fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), filename) err := os.WriteFile(cleanPath, contents, 0o644) if err != nil { fmt.Printf("failed to write config to file: %v", err) From a19bf5047165d3b2ff38426e09e42ec2417ed0db Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 27 Jun 2024 09:49:28 -0600 Subject: [PATCH 29/40] Fixed minor errors --- cmd/generate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 5455dcd..6f15e78 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -94,7 +94,7 @@ var generateCmd = &cobra.Command{ } } else if outputPath != "" && targetCount == 1 && templateCount == 1 { // write just a single file using provided name - for path, contents := range outputMap { + for _, contents := range outputBytes { err := os.WriteFile(outputPath, contents, 0o644) if err != nil { fmt.Printf("failed to write config to file: %v", err) @@ -111,7 +111,7 @@ var generateCmd = &cobra.Command{ } for path, contents := range outputBytes { filename := filepath.Base(path) - cleanPath := fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), filename) + cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) err := os.WriteFile(cleanPath, contents, 0o644) if err != nil { fmt.Printf("failed to write config to file: %v", err) From 2154657d34d518c3e8b44bbef3d847c943806625 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 27 Jun 2024 10:03:35 -0600 Subject: [PATCH 30/40] Fixed path used in LoadFiles --- internal/generator/generator.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/generator/generator.go b/internal/generator/generator.go index 4161a81..9b5a173 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -177,6 +177,7 @@ func LoadFiles(paths ...string) (Files, error) { fmt.Println(err) return nil, fmt.Errorf("failed to stat file or directory: %v", err) } + // skip any directories found if info.IsDir() { continue } @@ -185,7 +186,7 @@ func LoadFiles(paths ...string) (Files, error) { return nil, fmt.Errorf("failed to read file: %v", err) } - outputs[path] = b + outputs[expandedPath] = b } } From cccbb1ad25414eb87e0d5d71259766ee7ad9433a Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 27 Jun 2024 11:19:16 -0600 Subject: [PATCH 31/40] Minor refactor to run additional targets recursively --- cmd/generate.go | 115 +++++++++++++++++++++++------------------- internal/util/util.go | 18 +++++++ 2 files changed, 81 insertions(+), 52 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 6f15e78..f921fc2 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" + configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/generator" "github.com/OpenCHAMI/configurator/internal/util" "github.com/spf13/cobra" @@ -60,69 +61,79 @@ var generateCmd = &cobra.Command{ fmt.Printf("%v\n", string(b)) } - // generate config with each supplied target - for _, target := range targets { - params := generator.Params{ - Args: args, - PluginPaths: pluginPaths, - Target: target, - Verbose: verbose, - } - outputBytes, err := generator.Generate(&config, params) - if err != nil { - fmt.Printf("failed to generate config: %v\n", err) - os.Exit(1) - } + RunTargets(targets...) - outputMap := util.ConvertMapOutput(outputBytes) + }, +} - // if we have more than one target and output is set, create configs in directory - var ( - targetCount = len(targets) - templateCount = len(outputMap) - ) - if outputPath == "" { - // write only to stdout by default - if len(outputMap) == 1 { - for _, contents := range outputMap { - fmt.Printf("%s\n", string(contents)) - } - } else { - for path, contents := range outputMap { - fmt.Printf("-- file: %s, size: %d B\n%s\n", path, len(contents), string(contents)) - } +func RunTargets(config *configurator.Config, targets ...string) { + // generate config with each supplied target + for _, target := range targets { + params := generator.Params{ + Args: args, + PluginPaths: pluginPaths, + Target: target, + Verbose: verbose, + } + outputBytes, err := generator.Generate(&config, params) + if err != nil { + fmt.Printf("failed to generate config: %v\n", err) + os.Exit(1) + } + + outputMap := util.ConvertMapOutput(outputBytes) + + // if we have more than one target and output is set, create configs in directory + var ( + targetCount = len(targets) + templateCount = len(outputMap) + ) + if outputPath == "" { + // write only to stdout by default + if len(outputMap) == 1 { + for _, contents := range outputMap { + fmt.Printf("%s\n", string(contents)) } - } else if outputPath != "" && targetCount == 1 && templateCount == 1 { - // write just a single file using provided name - for _, contents := range outputBytes { - err := os.WriteFile(outputPath, contents, 0o644) - if err != nil { - fmt.Printf("failed to write config to file: %v", err) - os.Exit(1) - } - fmt.Printf("wrote file to '%s'\n", outputPath) + } else { + for path, contents := range outputMap { + fmt.Printf("-- file: %s, size: %d B\n%s\n", path, len(contents), string(contents)) } - } else if outputPath != "" && targetCount > 1 || templateCount > 1 { - // write multiple files in directory using template name - err := os.MkdirAll(filepath.Clean(outputPath), 0o755) + } + } else if outputPath != "" && targetCount == 1 && templateCount == 1 { + // write just a single file using provided name + for _, contents := range outputBytes { + err := os.WriteFile(outputPath, contents, 0o644) if err != nil { - fmt.Printf("failed to make output directory: %v", err) + fmt.Printf("failed to write config to file: %v", err) os.Exit(1) } - for path, contents := range outputBytes { - filename := filepath.Base(path) - cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) - err := os.WriteFile(cleanPath, contents, 0o644) - if err != nil { - fmt.Printf("failed to write config to file: %v", err) - os.Exit(1) - } - fmt.Printf("wrote file to '%s'\n", cleanPath) + fmt.Printf("wrote file to '%s'\n", outputPath) + } + } else if outputPath != "" && targetCount > 1 || templateCount > 1 { + // write multiple files in directory using template name + err := os.MkdirAll(filepath.Clean(outputPath), 0o755) + if err != nil { + fmt.Printf("failed to make output directory: %v", err) + os.Exit(1) + } + for path, contents := range outputBytes { + filename := filepath.Base(path) + cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) + err := os.WriteFile(cleanPath, contents, 0o644) + if err != nil { + fmt.Printf("failed to write config to file: %v", err) + os.Exit(1) } + fmt.Printf("wrote file to '%s'\n", cleanPath) } } - }, + // remove any targets that are the same as current to prevent infinite loop + nextTargets := util.CopyIf(config.Targets[targets].Targets, func(t T) bool { return t != target }) + + // ...then, run any other targets that the current target has + RunTargets(config, nextTargets...) + } } func init() { diff --git a/internal/util/util.go b/internal/util/util.go index 3a990f6..e35556e 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -61,3 +61,21 @@ func GitCommit() string { return strings.TrimRight(string(stdout), "\n") } + +// NOTE: would it be better to use slices.DeleteFunc instead +func RemoveIndex[T comparable](s []T, index int) []T { + ret := make([]T, 0) + ret = append(ret, s[:index]...) + return append(ret, s[index+1:]...) +} + +func CopyIf[T comparable](s []T, condition func(t T) bool) []T { + var f = make([]T, 0) + for _, e := range s { + if condition(e) { + f = append(f, e) + } + } + + return f +} From 962cf42e79d6828963d4dbf83f196d1410ad828c Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 27 Jun 2024 11:20:19 -0600 Subject: [PATCH 32/40] Updated warewulf plugin (still WIP) --- internal/generator/plugins/warewulf/warewulf.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/generator/plugins/warewulf/warewulf.go b/internal/generator/plugins/warewulf/warewulf.go index 7999afa..a0a610b 100644 --- a/internal/generator/plugins/warewulf/warewulf.go +++ b/internal/generator/plugins/warewulf/warewulf.go @@ -68,12 +68,17 @@ func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (g return nil, fmt.Errorf("no redfish endpoints found") } + // format output for template substitution + nodeEntries := "" + // load files and templates and copy to outputs files, err := generator.LoadFiles(target.FilePaths...) if err != nil { return nil, fmt.Errorf("failed to load files: %v", err) } - templates, err := generator.ApplyTemplates(generator.Mappings{}, target.Templates...) + templates, err := generator.ApplyTemplates(generator.Mappings{ + "node_entries": nodeEntries, + }, target.Templates...) if err != nil { return nil, fmt.Errorf("failed to load templates: %v", err) } From c81ef3b431bf484c144c87a261f985804c886ba1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 3 Jul 2024 10:00:49 -0600 Subject: [PATCH 33/40] Minor changes --- cmd/generate.go | 6 +++--- internal/generator/generator.go | 2 +- internal/util/util.go | 1 - templates/dnsmasq.jinja | 7 +++++++ 4 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 templates/dnsmasq.jinja diff --git a/cmd/generate.go b/cmd/generate.go index f921fc2..aaf4a1c 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -113,15 +113,15 @@ func RunTargets(config *configurator.Config, targets ...string) { // write multiple files in directory using template name err := os.MkdirAll(filepath.Clean(outputPath), 0o755) if err != nil { - fmt.Printf("failed to make output directory: %v", err) + fmt.Printf("failed to make output directory: %v\n", err) os.Exit(1) } for path, contents := range outputBytes { filename := filepath.Base(path) cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) - err := os.WriteFile(cleanPath, contents, 0o644) + err := os.WriteFile(cleanPath, contents, 0o755) if err != nil { - fmt.Printf("failed to write config to file: %v", err) + fmt.Printf("failed to write config to file: %v\n", err) os.Exit(1) } fmt.Printf("wrote file to '%s'\n", cleanPath) diff --git a/internal/generator/generator.go b/internal/generator/generator.go index 9b5a173..6687634 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -169,7 +169,7 @@ func LoadFiles(paths ...string) (Files, error) { for _, path := range paths { expandedPaths, err := filepath.Glob(path) if err != nil { - return nil, fmt.Errorf("failed to expand path: %v", err) + return nil, fmt.Errorf("failed to glob path: %v", err) } for _, expandedPath := range expandedPaths { info, err := os.Stat(expandedPath) diff --git a/internal/util/util.go b/internal/util/util.go index e35556e..8090ac4 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -76,6 +76,5 @@ func CopyIf[T comparable](s []T, condition func(t T) bool) []T { f = append(f, e) } } - return f } diff --git a/templates/dnsmasq.jinja b/templates/dnsmasq.jinja new file mode 100644 index 0000000..0b50fac --- /dev/null +++ b/templates/dnsmasq.jinja @@ -0,0 +1,7 @@ +# +# This file was auto-generated by the OpenCHAMI "configurator" tool using the "{{name}}" plugin. +# +# Source code: https://github.com/OpenCHAMI/configurator +# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# +{{ output }} From 2b064c2700476dc9909ca7c1b4af8970fc87adac Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 3 Jul 2024 10:01:39 -0600 Subject: [PATCH 34/40] Updated .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2455759..83abc11 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ **.yml **.so **.conf +**.ignore From 708980831bdaeeb172d9501b024888ba8c7d4282 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 3 Jul 2024 10:03:16 -0600 Subject: [PATCH 35/40] Updated README.md --- README.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9422771..56f8e41 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,12 @@ This will do the same thing as the `generate` subcommand, but remotely. The `configurator` uses generator plugins to define how config files are generated using a `Generator` interface. The interface is defined like so: ```go +type Files = map[string][]byte type Generator interface { GetName() string - GetGroups() []string - Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) + GetVersion() string + GetDescription() string + Generate(config *configurator.Config, opts ...util.Option) (Files, error) } ``` @@ -68,7 +70,17 @@ package main type MyGenerator struct {} func (g *MyGenerator) GetName() string { - return "my-generator" + // just an example...this can be done however you want + pluginInfo := LoadFromFile("path/to/plugin/info.json") + return pluginInfo["name"] +} + +func (g *MyGenerator) GetVersion() string { + return "v1.0.0" +} + +func (g *MyGenerator) GetDescription() string { + return "This is an example plugin." } func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { @@ -83,7 +95,7 @@ func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) // ... blah, blah, blah, format output, and so on... } - // apply the template and get substituted output as byte array + // apply the substitutions to Jinja template and return output as byte array return generator.ApplyTemplate(path, generator.Mappings{ "hosts": output, }) From 0c510db8f0ca8927c3bdc4eb0579148aae091efa Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 3 Jul 2024 10:13:15 -0600 Subject: [PATCH 36/40] Added initial set of example templates --- examples/templates/conman.jinja | 20 ++++++++ examples/templates/dhcpd.jinja | 48 +++++++++++++++++++ .../templates}/dnsmasq.jinja | 0 examples/templates/powerman.jinja | 12 +++++ 4 files changed, 80 insertions(+) create mode 100644 examples/templates/conman.jinja create mode 100644 examples/templates/dhcpd.jinja rename {templates => examples/templates}/dnsmasq.jinja (100%) create mode 100644 examples/templates/powerman.jinja diff --git a/examples/templates/conman.jinja b/examples/templates/conman.jinja new file mode 100644 index 0000000..6f21d7a --- /dev/null +++ b/examples/templates/conman.jinja @@ -0,0 +1,20 @@ +# +# This file was auto-generated by the OpenCHAMI "configurator" tool using the "{{name}}" plugin. +# +# Source code: https://github.com/OpenCHAMI/configurator +# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# +SERVER keepalive=ON +SERVER logdir="/var/log/conman" +SERVER logfile="/var/log/conman.log" +SERVER loopback=ON +SERVER pidfile="/var/run/conman.pid" +SERVER resetcmd="/usr/bin/powerman -0 \%N; sleep 5; /usr/bin/powerman -1 \%N" +SERVER tcpwrappers=ON +#SERVER timestamp=1h + +GLOBAL seropts="115200,8n1" +GLOBAL log="/var/log/conman/console.\%N" +GLOBAL logopts="sanitize,timestamp" + +{{ consoles }} diff --git a/examples/templates/dhcpd.jinja b/examples/templates/dhcpd.jinja new file mode 100644 index 0000000..377b244 --- /dev/null +++ b/examples/templates/dhcpd.jinja @@ -0,0 +1,48 @@ +# +# This file was auto-generated by the OpenCHAMI "configurator" tool using the "{{name}}" plugin. +# +# Source code: https://github.com/OpenCHAMI/configurator +# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# +allow booting; +allow bootp; +ddns-update-style interim; +authoritative; + +option space ipxe; + +# Tell iPXE to not wait for ProxyDHCP requests to speed up boot. +option ipxe.no-pxedhcp code 176 = unsigned integer 8; +option ipxe.no-pxedhcp 1; + +option architecture-type code 93 = unsigned integer 16; + +if exists user-class and option user-class = "iPXE" { + filename "http://%{IPADDR}/WW/ipxe/cfg/${mac}"; +} else { + if option architecture-type = 00:0B { + filename "/warewulf/ipxe/bin-arm64-efi/snp.efi"; + } elsif option architecture-type = 00:0A { + filename "/warewulf/ipxe/bin-arm32-efi/placeholder.efi"; + } elsif option architecture-type = 00:09 { + filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi"; + } elsif option architecture-type = 00:07 { + filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi"; + } elsif option architecture-type = 00:06 { + filename "/warewulf/ipxe/bin-i386-efi/snp.efi"; + } elsif option architecture-type = 00:00 { + filename "/warewulf/ipxe/bin-i386-pcbios/undionly.kpxe"; + } +} + +subnet %{NETWORK} netmask %{NETMASK} { + not authoritative; + # option interface-mtu 9000; + option subnet-mask %{NETMASK}; +} + +# Compute Nodes (WIP - see the dhcpd generator plugin) +{{ compute_nodes }} + +# Node entries will follow below +{{ node_entries }} \ No newline at end of file diff --git a/templates/dnsmasq.jinja b/examples/templates/dnsmasq.jinja similarity index 100% rename from templates/dnsmasq.jinja rename to examples/templates/dnsmasq.jinja diff --git a/examples/templates/powerman.jinja b/examples/templates/powerman.jinja new file mode 100644 index 0000000..11b1e8c --- /dev/null +++ b/examples/templates/powerman.jinja @@ -0,0 +1,12 @@ +# +# Ansible managed +# +include "/etc/powerman/ipmipower.dev" +include "/etc/powerman/ipmi.dev" + + +# list of devices +{{ devices }} + +# create nodes based on found nodes in hostfile +{{ nodes }} \ No newline at end of file From c59efb835f9a2f44a9ead1277d73c2d389a295b9 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 3 Jul 2024 11:40:17 -0600 Subject: [PATCH 37/40] Updated .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 83abc11..f9bc65d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ **.so **.conf **.ignore +**.tar.gz From e15018072f5716f5583b76089e530ba6cf2f9280 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 3 Jul 2024 11:40:33 -0600 Subject: [PATCH 38/40] Update README.md --- README.md | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 56f8e41..78a971a 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,12 @@ The tool can also run as a service to generate files for clients: Once the server is up and listening for HTTP requests, you can try making a request to it with `curl` or `configurator fetch`. Both commands below are essentially equivalent: ```bash -curl http://127.0.0.1:3334/generate?target=dnsmasq +curl http://127.0.0.1:3334/generate?target=dnsmasq -H "Authorization: Bearer $ACCESS_TOKEN" # ...or... -./configurator fetch --target dnsmasq --host http://127.0.0.1 --port 3334 +./configurator fetch --target dnsmasq --host http://127.0.0.1 --port 3334 ``` -This will do the same thing as the `generate` subcommand, but remotely. +This will do the same thing as the `generate` subcommand, but remotely. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set. The `ACCESS_TOKEN` environment variable passed to `curl` and it's corresponding CLI argument both expects a token as a JWT. ### Creating Generator Plugins @@ -118,31 +118,34 @@ Now your plugin should be available to use with the `configurator` main driver. Here is an example config file to start using configurator: ```yaml -server: # server settings when using as service +server: # Server-related parameters when using as service host: 127.0.0.1 port: 3334 - jwks: # set URL for JWKS to enable auth + jwks: # Set the JWKS uri to protect /generate route uri: "" retries: 5 -smd: # settings for SMD service +smd: . # SMD-related parameters host: http://127.0.0.1 port: 27779 -templates: # template mappings to generator plugins (by name) - dnsmasq: templates/dnsmasq.jinja - coredhcp: templates/coredhcp.jinja - syslog: templates/syslog.jinja - ansible: templates/ansible.jinja - powerman: templates/powerman.jinja - conman: templates/conman.jinja -groups: # (WIP) setting to allow creating configs by groups - warewulf: - - dnsmasq - - syslog - - ansible - - powerman - - conman -plugins: # path to plugin directories (may change to include files as well) +plugins: # path to plugin directories - "lib/" +targets: # targets to call with --target flag + dnsmasq: + templates: + - templates/dnsmasq.jinja + warewulf: + templates: # files using Jinja templating + - templates/warewulf/vnfs/dhcpd-template.jinja + - templates/warewulf/vnfs/dnsmasq-template.jinja + files: # files to be copied without templating + - templates/warewulf/defaults/provision.jinja + - templates/warewulf/defaults/node.jinja + - templates/warewulf/filesystem/examples/* + - templates/warewulf/vnfs/* + - templates/warewulf/bootstrap.jinja + - templates/warewulf/database.jinja + targets: # additional targets to run + - dnsmasq ``` The `server` section sets the properties for running the `configurator` tool as a service and is not required if you're only using the CLI. Also note that the `jwks-uri` parameter is only needs for protecting endpoints. If it is not set, then the API is entirely public. The `smd` section tells the `configurator` tool where to find SMD to pull state management data used by the internal client. The `templates` section is where the paths are mapped to each generator plugin by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `plugins` is a list of paths to load generator plugins. From 7836aef5c3231fb434fb8234e838c7b5cc07ce6e Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 3 Jul 2024 11:41:17 -0600 Subject: [PATCH 39/40] Added CONFIGURATOR_JWKS_URL var for key verification --- cmd/root.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index d376654..353523e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,4 +55,14 @@ func initConfig() { } else { config = configurator.NewConfig() } + + // + // set environment variables to override config values + // + + // set the JWKS url if we find the CONFIGURATOR_JWKS_URL environment variable + jwksUrl := os.Getenv("CONFIGURATOR_JWKS_URL") + if jwksUrl != "" { + config.Server.Jwks.Uri = jwksUrl + } } From bf937bf4d47dc5433b04388bafe9e07f252e35aa Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 3 Jul 2024 11:42:27 -0600 Subject: [PATCH 40/40] Changed server handlers to use public and protected routes correctly with auth --- cmd/serve.go | 3 +- internal/server/server.go | 113 ++++++++++++++++++++++---------------- 2 files changed, 69 insertions(+), 47 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index e150b34..7eeb459 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -51,6 +51,7 @@ var serveCmd = &cobra.Command{ // set up the routes and start the server server := server.Server{ + Config: &config, Server: &http.Server{ Addr: fmt.Sprintf("%s:%d", config.Server.Host, config.Server.Port), }, @@ -65,7 +66,7 @@ var serveCmd = &cobra.Command{ Verbose: verbose, }, } - err := server.Serve(&config) + err := server.Serve() if errors.Is(err, http.ErrServerClosed) { fmt.Printf("Server closed.") } else if err != nil { diff --git a/internal/server/server.go b/internal/server/server.go index bf7534d..5c16ca6 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -27,6 +27,7 @@ type Jwks struct { } type Server struct { *http.Server + Config *configurator.Config Jwks Jwks `yaml:"jwks"` GeneratorParams generator.Params TokenAuth *jwtauth.JWTAuth @@ -44,21 +45,21 @@ func New() *Server { } } -func (s *Server) Serve(config *configurator.Config) error { +func (s *Server) Serve() error { // create client just for the server to use to fetch data from SMD _ = &configurator.SmdClient{ - Host: config.SmdClient.Host, - Port: config.SmdClient.Port, + Host: s.Config.SmdClient.Host, + Port: s.Config.SmdClient.Port, } // set the server address with config values - s.Server.Addr = fmt.Sprintf("%s:%d", config.Server.Host, config.Server.Port) + s.Server.Addr = fmt.Sprintf("%s:%d", s.Config.Server.Host, s.Config.Server.Port) // fetch JWKS public key from authorization server - if config.Server.Jwks.Uri != "" && tokenAuth == nil { - for i := 0; i < config.Server.Jwks.Retries; i++ { + if s.Config.Server.Jwks.Uri != "" && tokenAuth == nil { + for i := 0; i < s.Config.Server.Jwks.Retries; i++ { var err error - tokenAuth, err = configurator.FetchPublicKeyFromURL(config.Server.Jwks.Uri) + tokenAuth, err = configurator.FetchPublicKeyFromURL(s.Config.Server.Jwks.Uri) if err != nil { logrus.Errorf("failed to fetch JWKS: %w", err) continue @@ -67,55 +68,75 @@ func (s *Server) Serve(config *configurator.Config) error { } } - var WriteError = func(w http.ResponseWriter, format string, a ...any) { - errmsg := fmt.Sprintf(format, a...) - fmt.Printf(errmsg) - w.Write([]byte(errmsg)) - } - // create new go-chi router with its routes router := chi.NewRouter() - router.Use(middleware.RedirectSlashes) + router.Use(middleware.RequestID) + router.Use(middleware.RealIP) + router.Use(middleware.Logger) + router.Use(middleware.Recoverer) + router.Use(middleware.StripSlashes) router.Use(middleware.Timeout(60 * time.Second)) - router.Group(func(r chi.Router) { - if config.Server.Jwks.Uri != "" { + if s.Config.Server.Jwks.Uri != "" { + router.Group(func(r chi.Router) { r.Use( jwtauth.Verifier(tokenAuth), jwtauth.Authenticator(tokenAuth), ) - } - r.HandleFunc("/generate", func(w http.ResponseWriter, r *http.Request) { - s.GeneratorParams.Target = r.URL.Query().Get("target") - outputs, err := generator.Generate(config, s.GeneratorParams) - if err != nil { - WriteError(w, "failed to generate config: %v", err) - return - } - // convert byte arrays to string - tmp := map[string]string{} - for path, output := range outputs { - tmp[path] = string(output) - } - - // marshal output to JSON then send - b, err := json.Marshal(tmp) - if err != nil { - WriteError(w, "failed to marshal output: %v", err) - return - } - _, err = w.Write(b) - if err != nil { - WriteError(w, "failed to write response: %v", err) - return - } + // protected routes if using auth + r.HandleFunc("/generate", s.Generate) + r.HandleFunc("/templates", s.ManageTemplates) }) - r.HandleFunc("/templates", func(w http.ResponseWriter, r *http.Request) { - // TODO: handle GET request - // TODO: handle POST request + } else { + // public routes without auth + router.HandleFunc("/generate", s.Generate) + router.HandleFunc("/templates", s.ManageTemplates) + } + + // always public routes go here (none at the moment) - }) - }) s.Handler = router return s.ListenAndServe() } + +func WriteError(w http.ResponseWriter, format string, a ...any) { + errmsg := fmt.Sprintf(format, a...) + fmt.Printf(errmsg) + w.Write([]byte(errmsg)) +} + +func (s *Server) Generate(w http.ResponseWriter, r *http.Request) { + s.GeneratorParams.Target = r.URL.Query().Get("target") + outputs, err := generator.Generate(s.Config, s.GeneratorParams) + if err != nil { + WriteError(w, "failed to generate config: %v", err) + return + } + + // convert byte arrays to string + tmp := map[string]string{} + for path, output := range outputs { + tmp[path] = string(output) + } + + // marshal output to JSON then send + b, err := json.Marshal(tmp) + if err != nil { + WriteError(w, "failed to marshal output: %v", err) + return + } + _, err = w.Write(b) + if err != nil { + WriteError(w, "failed to write response: %v", err) + return + } +} + +func (s *Server) ManageTemplates(w http.ResponseWriter, r *http.Request) { + // TODO: need to implement template managing API first in "internal/generator/templates" or something + _, err := w.Write([]byte("this is not implemented yet")) + if err != nil { + WriteError(w, "failed to write response: %v", err) + return + } +}