diff --git a/.gitignore b/.gitignore index 0bb9720..f9bc65d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ **configurator** **.yaml **.yml -**.so \ No newline at end of file +**.so +**.conf +**.ignore +**.tar.gz diff --git a/Makefile b/Makefile index 7b0fd12..84b094c 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ all: plugins exe # build the main executable to make configs main: exe +driver: exe exe: go build --tags=all -o configurator @@ -12,6 +13,15 @@ plugins: mkdir -p lib go build -buildmode=plugin -o lib/conman.so internal/generator/plugins/conman/conman.go go build -buildmode=plugin -o lib/coredhcp.so internal/generator/plugins/coredhcp/coredhcp.go + go build -buildmode=plugin -o lib/dhcpd.so internal/generator/plugins/dhcpd/dhcpd.go go build -buildmode=plugin -o lib/dnsmasq.so internal/generator/plugins/dnsmasq/dnsmasq.go + go build -buildmode=plugin -o lib/hostfile.so internal/generator/plugins/hostfile/hostfile.go go build -buildmode=plugin -o lib/powerman.so internal/generator/plugins/powerman/powerman.go go build -buildmode=plugin -o lib/syslog.so internal/generator/plugins/syslog/syslog.go + go build -buildmode=plugin -o lib/warewulf.so internal/generator/plugins/warewulf/warewulf.go + +# remove executable and all plugins +clean: + rm configurator + rm lib/* + diff --git a/README.md b/README.md index 3103256..78a971a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # OpenCHAMI Configurator -The `configurator` (portmanteau of config + generator) is a tool that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate commonly used config files. The tool is also capable of some templating using the Jinja 2 syntax with generator plugins. +The `configurator` (portmanteau of config + generator) is an extensible tool that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate commonly used config files based on Jinja 2 template files. The tool and generator plugins are written in Go and plugins can be written by following the ["Creating Generator Plugins"](#creating-generator-plugins) section of this README. ## Building and Usage @@ -32,29 +32,33 @@ These commands will build the default plugins and store them in the "lib" direct This will generate a new `dnsmasq` config file based on the Jinja 2 template specified in the config file for "dnsmasq". The `--target` flag specifies the type of config file to generate by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `configurator` tool requires a valid access token when making requests to an instance of SMD that has protected routes. -The tool can also run as a microservice: +The tool can also run as a service to generate files for clients: ```bash ./configurator serve --config config.yaml ``` -Once the server is up and listening for HTTP requests, you can try making a request to it with curl: +Once the server is up and listening for HTTP requests, you can try making a request to it with `curl` or `configurator fetch`. Both commands below are essentially equivalent: ```bash -curl http://127.0.0.1:3334/target?type=dhcp&template=dnsmasq +curl http://127.0.0.1:3334/generate?target=dnsmasq -H "Authorization: Bearer $ACCESS_TOKEN" +# ...or... +./configurator fetch --target dnsmasq --host http://127.0.0.1 --port 3334 ``` -This will do the same thing as the `generate` subcommand, but remotely. +This will do the same thing as the `generate` subcommand, but remotely. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set. The `ACCESS_TOKEN` environment variable passed to `curl` and it's corresponding CLI argument both expects a token as a JWT. ### Creating Generator Plugins The `configurator` uses generator plugins to define how config files are generated using a `Generator` interface. The interface is defined like so: ```go +type Files = map[string][]byte type Generator interface { GetName() string - GetGroups() []string - Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) + GetVersion() string + GetDescription() string + Generate(config *configurator.Config, opts ...util.Option) (Files, error) } ``` @@ -66,14 +70,20 @@ package main type MyGenerator struct {} func (g *MyGenerator) GetName() string { - return "my-generator" + // just an example...this can be done however you want + pluginInfo := LoadFromFile("path/to/plugin/info.json") + return pluginInfo["name"] } -func (g *MyGenerator) GetGroups() []string { - return []string{ "my-generator" } +func (g *MyGenerator) GetVersion() string { + return "v1.0.0" } -func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { +func (g *MyGenerator) GetDescription() string { + return "This is an example plugin." +} + +func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { // do config generation stuff here... var ( params = generator.GetParams(opts...) @@ -85,7 +95,7 @@ func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) // ... blah, blah, blah, format output, and so on... } - // apply the template and get substituted output as byte array + // apply the substitutions to Jinja template and return output as byte array return generator.ApplyTemplate(path, generator.Mappings{ "hosts": output, }) @@ -101,38 +111,41 @@ Finally, build the plugin and put it somewhere specified by `plugins` in your co go build -buildmode=plugin -o lib/mygenerator.so path/to/mygenerator.go ``` -Now your plugin should be available to use with the `configurator` main driver. +Now your plugin should be available to use with the `configurator` main driver. If you get an error about not loading the correct symbol type, make sure that you generator function definitions match the `Generator` interface exactly. ## Configuration Here is an example config file to start using configurator: ```yaml -server: +server: # Server-related parameters when using as service host: 127.0.0.1 port: 3334 - jwks: + jwks: # Set the JWKS uri to protect /generate route uri: "" retries: 5 -smd: +smd: . # SMD-related parameters host: http://127.0.0.1 port: 27779 -templates: - dnsmasq: templates/dnsmasq.jinja - coredhcp: templates/coredhcp.jinja - syslog: templates/syslog.jinja - ansible: templates/ansible.jinja - powerman: templates/powerman.jinja - conman: templates/conman.jinja -groups: - warewulf: - - dnsmasq - - syslog - - ansible - - powerman - - conman -plugins: +plugins: # path to plugin directories - "lib/" +targets: # targets to call with --target flag + dnsmasq: + templates: + - templates/dnsmasq.jinja + warewulf: + templates: # files using Jinja templating + - templates/warewulf/vnfs/dhcpd-template.jinja + - templates/warewulf/vnfs/dnsmasq-template.jinja + files: # files to be copied without templating + - templates/warewulf/defaults/provision.jinja + - templates/warewulf/defaults/node.jinja + - templates/warewulf/filesystem/examples/* + - templates/warewulf/vnfs/* + - templates/warewulf/bootstrap.jinja + - templates/warewulf/database.jinja + targets: # additional targets to run + - dnsmasq ``` The `server` section sets the properties for running the `configurator` tool as a service and is not required if you're only using the CLI. Also note that the `jwks-uri` parameter is only needs for protecting endpoints. If it is not set, then the API is entirely public. The `smd` section tells the `configurator` tool where to find SMD to pull state management data used by the internal client. The `templates` section is where the paths are mapped to each generator plugin by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `plugins` is a list of paths to load generator plugins. @@ -140,9 +153,10 @@ The `server` section sets the properties for running the `configurator` tool as ## Known Issues - Adds a new `OAuthClient` with every token request +- Plugins are being loaded each time a file is generated ## TODO -- Add group functionality -- Extend SMD client functionality -- Redo service API with authorization +- Add group functionality to create by files by groups +- Extend SMD client functionality (or make extensible?) +- Handle authentication with `OAuthClient`'s correctly diff --git a/cmd/fetch.go b/cmd/fetch.go new file mode 100644 index 0000000..110f03c --- /dev/null +++ b/cmd/fetch.go @@ -0,0 +1,54 @@ +//go:build client || all +// +build client all + +package cmd + +import ( + "fmt" + "net/http" + + "github.com/OpenCHAMI/configurator/internal/util" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var ( + remoteHost string + remotePort int +) + +var fetchCmd = &cobra.Command{ + Use: "fetch", + Short: "Fetch a config file from a remote instance of configurator", + Run: func(cmd *cobra.Command, args []string) { + // make sure a host is set + if remoteHost == "" { + logrus.Errorf("no '--host' argument set") + return + } + for _, target := range targets { + + // make a request for each target + url := fmt.Sprintf("%s:%d/generate?target=%s", remoteHost, remotePort, target) + res, body, err := util.MakeRequest(url, http.MethodGet, nil, nil) + if err != nil { + logrus.Errorf("failed to make request: %v", err) + return + } + if res != nil { + if res.StatusCode == http.StatusOK { + fmt.Printf("%s\n", string(body)) + } + } + } + }, +} + +func init() { + fetchCmd.Flags().StringVar(&remoteHost, "host", "", "set the remote configurator host") + fetchCmd.Flags().IntVar(&remotePort, "port", 3334, "set the remote configurator port") + fetchCmd.Flags().StringSliceVar(&targets, "target", nil, "set the target configs to make") + fetchCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") + + rootCmd.AddCommand(fetchCmd) +} diff --git a/cmd/generate.go b/cmd/generate.go index d73d645..aaf4a1c 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -6,66 +6,25 @@ package cmd import ( "encoding/json" "fmt" - "maps" "os" "path/filepath" configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/generator" - "github.com/sirupsen/logrus" + "github.com/OpenCHAMI/configurator/internal/util" "github.com/spf13/cobra" ) var ( tokenFetchRetries int pluginPaths []string + cacertPath string ) var generateCmd = &cobra.Command{ Use: "generate", Short: "Generate a config file from state management", Run: func(cmd *cobra.Command, args []string) { - // load generator plugins to generate configs or to print - var ( - generators = make(map[string]generator.Generator) - client = configurator.SmdClient{ - Host: config.SmdClient.Host, - Port: config.SmdClient.Port, - AccessToken: config.AccessToken, - } - ) - for _, path := range pluginPaths { - if verbose { - fmt.Printf("loading plugins from '%s'\n", path) - } - gens, err := generator.LoadPlugins(path) - if err != nil { - fmt.Printf("failed to load plugins: %v\n", err) - err = nil - continue - } - - // add loaded generator plugins to set - maps.Copy(generators, gens) - } - - // show config as JSON and generators if verbose - if verbose { - b, err := json.MarshalIndent(config, "", " ") - if err != nil { - fmt.Printf("failed to marshal config: %v\n", err) - } - fmt.Printf("%v\n", string(b)) - } - - // show available targets then exit - if len(args) == 0 && len(targets) == 0 { - for g := range generators { - fmt.Printf("\tplugin: %s, name:\n", g) - } - os.Exit(0) - } - // make sure that we have a token present before trying to make request if config.AccessToken == "" { // TODO: make request to check if request will need token @@ -82,126 +41,106 @@ var generateCmd = &cobra.Command{ } } - if targets == nil { - logrus.Errorf("no target supplied (--target type:template)") - } else { - // if we have more than one target and output is set, create configs in directory - targetCount := len(targets) - if outputPath != "" && targetCount > 1 { - err := os.MkdirAll(outputPath, 0o755) - if err != nil { - logrus.Errorf("failed to make output directory: %v", err) - return - } - } - - for _, target := range targets { - // split the target and type - // tmp := strings.Split(target, ":") - - // make sure each target has at least two args - // if len(tmp) < 2 { - // message := "target" - // if len(tmp) == 1 { - // message += fmt.Sprintf(" '%s'", tmp[0]) - // } - // message += " does not provide enough arguments (args: \"type:template\")" - // logrus.Errorf(message) - // continue - // } - // var ( - // _type = tmp[0] - // _template = tmp[1] - // ) - // g := generator.Generator{ - // Type: tmp[0], - // Template: tmp[1], - // } - - // check if another param is specified - // targetPath := "" - // if len(tmp) > 2 { - // targetPath = tmp[2] - // } - - // run the generator plugin from target passed - gen := generators[target] - if gen == nil { - fmt.Printf("invalid generator target (%s)\n", target) - continue - } - output, err := gen.Generate( - &config, - generator.WithTemplate(gen.GetName()), - generator.WithClient(client), - ) - if err != nil { - fmt.Printf("failed to generate config: %v\n", err) - continue - } - - // NOTE: we probably don't want to hardcode the types, but should do for now - // ext := "" - // contents := []byte{} - // if _type == "dhcp" { - // // fetch eths from SMD - // eths, err := client.FetchEthernetInterfaces() - // if err != nil { - // logrus.Errorf("failed to fetch DHCP metadata: %v\n", err) - // continue - // } - // if len(eths) <= 0 { - // continue - // } - // // generate a new config from that data - // contents, err = g.GenerateDHCP(&config, eths) - // if err != nil { - // logrus.Errorf("failed to generate DHCP config file: %v\n", err) - // continue - // } - // ext = "conf" - // } else if g.Type == "dns" { - // // TODO: fetch from SMD - // // TODO: generate config from pulled info - - // } else if g.Type == "syslog" { - - // } else if g.Type == "ansible" { - - // } else if g.Type == "warewulf" { - - // } - - // write config output if no specific targetPath is set - // if targetPath == "" { - if outputPath == "" { - // write only to stdout - fmt.Printf("%s\n", string(output)) - } else if outputPath != "" && targetCount == 1 { - // write just a single file using template name - err := os.WriteFile(outputPath, output, 0o644) - if err != nil { - logrus.Errorf("failed to write config to file: %v", err) - continue - } - } else if outputPath != "" && targetCount > 1 { - // write multiple files in directory using template name - err := os.WriteFile(fmt.Sprintf("%s/%s.%s", filepath.Clean(outputPath), target, ".conf"), output, 0o644) - if err != nil { - logrus.Errorf("failed to write config to file: %v", err) - continue - } - } - // } - } // for targets + // use cert path from cobra if empty + // TODO: this needs to be checked for the correct desired behavior + if config.CertPath == "" { + config.CertPath = cacertPath } + + // use config plugins if none supplied via CLI + if len(pluginPaths) <= 0 { + pluginPaths = append(pluginPaths, config.PluginDirs...) + } + + // show config as JSON and generators if verbose + if verbose { + b, err := json.MarshalIndent(config, "", " ") + if err != nil { + fmt.Printf("failed to marshal config: %v\n", err) + } + fmt.Printf("%v\n", string(b)) + } + + RunTargets(targets...) + }, } +func RunTargets(config *configurator.Config, targets ...string) { + // generate config with each supplied target + for _, target := range targets { + params := generator.Params{ + Args: args, + PluginPaths: pluginPaths, + Target: target, + Verbose: verbose, + } + outputBytes, err := generator.Generate(&config, params) + if err != nil { + fmt.Printf("failed to generate config: %v\n", err) + os.Exit(1) + } + + outputMap := util.ConvertMapOutput(outputBytes) + + // if we have more than one target and output is set, create configs in directory + var ( + targetCount = len(targets) + templateCount = len(outputMap) + ) + if outputPath == "" { + // write only to stdout by default + if len(outputMap) == 1 { + for _, contents := range outputMap { + fmt.Printf("%s\n", string(contents)) + } + } else { + for path, contents := range outputMap { + fmt.Printf("-- file: %s, size: %d B\n%s\n", path, len(contents), string(contents)) + } + } + } else if outputPath != "" && targetCount == 1 && templateCount == 1 { + // write just a single file using provided name + for _, contents := range outputBytes { + err := os.WriteFile(outputPath, contents, 0o644) + if err != nil { + fmt.Printf("failed to write config to file: %v", err) + os.Exit(1) + } + fmt.Printf("wrote file to '%s'\n", outputPath) + } + } else if outputPath != "" && targetCount > 1 || templateCount > 1 { + // write multiple files in directory using template name + err := os.MkdirAll(filepath.Clean(outputPath), 0o755) + if err != nil { + fmt.Printf("failed to make output directory: %v\n", err) + os.Exit(1) + } + for path, contents := range outputBytes { + filename := filepath.Base(path) + cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) + err := os.WriteFile(cleanPath, contents, 0o755) + if err != nil { + fmt.Printf("failed to write config to file: %v\n", err) + os.Exit(1) + } + fmt.Printf("wrote file to '%s'\n", cleanPath) + } + } + + // remove any targets that are the same as current to prevent infinite loop + nextTargets := util.CopyIf(config.Targets[targets].Targets, func(t T) bool { return t != target }) + + // ...then, run any other targets that the current target has + RunTargets(config, nextTargets...) + } +} + func init() { - generateCmd.Flags().StringSliceVar(&targets, "target", nil, "set the target configs to make") - generateCmd.Flags().StringSliceVar(&pluginPaths, "plugin", nil, "set the generator plugins directory path to shared libraries") + generateCmd.Flags().StringSliceVar(&targets, "target", []string{}, "set the target configs to make") + generateCmd.Flags().StringSliceVar(&pluginPaths, "plugins", []string{}, "set the generator plugins directory path") generateCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") + generateCmd.Flags().StringVar(&cacertPath, "ca-cert", "", "path to CA cert. (defaults to system CAs)") generateCmd.Flags().IntVar(&tokenFetchRetries, "fetch-retries", 5, "set the number of retries to fetch an access token") rootCmd.AddCommand(generateCmd) diff --git a/cmd/inspect.go b/cmd/inspect.go new file mode 100644 index 0000000..4ca4daf --- /dev/null +++ b/cmd/inspect.go @@ -0,0 +1,63 @@ +package cmd + +import ( + "fmt" + "maps" + "strings" + + "github.com/OpenCHAMI/configurator/internal/generator" + "github.com/spf13/cobra" +) + +var ( + pluginDirs []string + generators map[string]generator.Generator +) + +var inspectCmd = &cobra.Command{ + Use: "inspect", + Short: "Inspect generator plugin information", + Run: func(cmd *cobra.Command, args []string) { + // load specific plugins from positional args + generators = make(map[string]generator.Generator) + for _, path := range args { + gen, err := generator.LoadPlugin(path) + if err != nil { + fmt.Printf("failed to load plugin at path '%s': %v\n", path, err) + continue + } + generators[path] = gen + } + + // load plugins and print all plugin details + if len(pluginDirs) > 0 { + + } else { + for _, pluginDir := range config.PluginDirs { + gens, err := generator.LoadPlugins(pluginDir) + if err != nil { + fmt.Printf("failed to load plugin: %v\n", err) + continue + } + maps.Copy(generators, gens) + } + } + + // print all generator information + if len(generators) > 0 { + o := "" + for _, g := range generators { + o += fmt.Sprintf("- Name: %s\n", g.GetName()) + o += fmt.Sprintf(" Version: %s\n", g.GetVersion()) + o += fmt.Sprintf(" Description: %s\n", g.GetDescription()) + o += "\n" + } + o = strings.TrimRight(o, "\n") + fmt.Printf("%s", o) + } + }, +} + +func init() { + rootCmd.AddCommand(inspectCmd) +} diff --git a/cmd/root.go b/cmd/root.go index d376654..353523e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,4 +55,14 @@ func initConfig() { } else { config = configurator.NewConfig() } + + // + // set environment variables to override config values + // + + // set the JWKS url if we find the CONFIGURATOR_JWKS_URL environment variable + jwksUrl := os.Getenv("CONFIGURATOR_JWKS_URL") + if jwksUrl != "" { + config.Server.Jwks.Uri = jwksUrl + } } diff --git a/cmd/serve.go b/cmd/serve.go index d60ad9f..7eeb459 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -4,13 +4,14 @@ package cmd import ( + "encoding/json" "errors" "fmt" "net/http" "os" + "github.com/OpenCHAMI/configurator/internal/generator" "github.com/OpenCHAMI/configurator/internal/server" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -18,13 +19,58 @@ var serveCmd = &cobra.Command{ Use: "serve", Short: "Start configurator as a server and listen for requests", Run: func(cmd *cobra.Command, args []string) { + // make sure that we have a token present before trying to make request + if config.AccessToken == "" { + // TODO: make request to check if request will need token + + // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead + accessToken := os.Getenv("ACCESS_TOKEN") + if accessToken != "" { + config.AccessToken = accessToken + } else { + // TODO: try and fetch token first if it is needed + if verbose { + fmt.Printf("No token found. Attempting to generate config without one...\n") + } + } + } + + // use config plugins if none supplied via CLI + if len(pluginPaths) <= 0 { + pluginPaths = append(pluginPaths, config.PluginDirs...) + } + + // show config as JSON and generators if verbose + if verbose { + b, err := json.MarshalIndent(config, "", " ") + if err != nil { + fmt.Printf("failed to marshal config: %v\n", err) + } + fmt.Printf("%v\n", string(b)) + } + // set up the routes and start the server - server := server.New() - err := server.Start(&config) + server := server.Server{ + Config: &config, + Server: &http.Server{ + Addr: fmt.Sprintf("%s:%d", config.Server.Host, config.Server.Port), + }, + Jwks: server.Jwks{ + Uri: config.Server.Jwks.Uri, + Retries: config.Server.Jwks.Retries, + }, + GeneratorParams: generator.Params{ + Args: args, + PluginPaths: pluginPaths, + // Target: target, // NOTE: targets are set via HTTP requests (ex: curl http://configurator:3334/generate?target=dnsmasq) + Verbose: verbose, + }, + } + err := server.Serve() if errors.Is(err, http.ErrServerClosed) { fmt.Printf("Server closed.") } else if err != nil { - logrus.Errorf("failed to start server: %v", err) + fmt.Errorf("failed to start server: %v", err) os.Exit(1) } }, @@ -33,7 +79,8 @@ var serveCmd = &cobra.Command{ func init() { serveCmd.Flags().StringVar(&config.Server.Host, "host", config.Server.Host, "set the server host") serveCmd.Flags().IntVar(&config.Server.Port, "port", config.Server.Port, "set the server port") - serveCmd.Flags().StringVar(&config.Options.JwksUri, "jwks-uri", config.Options.JwksUri, "set the JWKS url to fetch public key") - serveCmd.Flags().IntVar(&config.Options.JwksRetries, "jwks-fetch-retries", config.Options.JwksRetries, "set the JWKS fetch retry count") + serveCmd.Flags().StringSliceVar(&pluginPaths, "plugins", nil, "set the generator plugins directory path") + serveCmd.Flags().StringVar(&config.Server.Jwks.Uri, "jwks-uri", config.Server.Jwks.Uri, "set the JWKS url to fetch public key") + serveCmd.Flags().IntVar(&config.Server.Jwks.Retries, "jwks-fetch-retries", config.Server.Jwks.Retries, "set the JWKS fetch retry count") rootCmd.AddCommand(serveCmd) } diff --git a/examples/templates/conman.jinja b/examples/templates/conman.jinja new file mode 100644 index 0000000..6f21d7a --- /dev/null +++ b/examples/templates/conman.jinja @@ -0,0 +1,20 @@ +# +# This file was auto-generated by the OpenCHAMI "configurator" tool using the "{{name}}" plugin. +# +# Source code: https://github.com/OpenCHAMI/configurator +# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# +SERVER keepalive=ON +SERVER logdir="/var/log/conman" +SERVER logfile="/var/log/conman.log" +SERVER loopback=ON +SERVER pidfile="/var/run/conman.pid" +SERVER resetcmd="/usr/bin/powerman -0 \%N; sleep 5; /usr/bin/powerman -1 \%N" +SERVER tcpwrappers=ON +#SERVER timestamp=1h + +GLOBAL seropts="115200,8n1" +GLOBAL log="/var/log/conman/console.\%N" +GLOBAL logopts="sanitize,timestamp" + +{{ consoles }} diff --git a/examples/templates/dhcpd.jinja b/examples/templates/dhcpd.jinja new file mode 100644 index 0000000..377b244 --- /dev/null +++ b/examples/templates/dhcpd.jinja @@ -0,0 +1,48 @@ +# +# This file was auto-generated by the OpenCHAMI "configurator" tool using the "{{name}}" plugin. +# +# Source code: https://github.com/OpenCHAMI/configurator +# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# +allow booting; +allow bootp; +ddns-update-style interim; +authoritative; + +option space ipxe; + +# Tell iPXE to not wait for ProxyDHCP requests to speed up boot. +option ipxe.no-pxedhcp code 176 = unsigned integer 8; +option ipxe.no-pxedhcp 1; + +option architecture-type code 93 = unsigned integer 16; + +if exists user-class and option user-class = "iPXE" { + filename "http://%{IPADDR}/WW/ipxe/cfg/${mac}"; +} else { + if option architecture-type = 00:0B { + filename "/warewulf/ipxe/bin-arm64-efi/snp.efi"; + } elsif option architecture-type = 00:0A { + filename "/warewulf/ipxe/bin-arm32-efi/placeholder.efi"; + } elsif option architecture-type = 00:09 { + filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi"; + } elsif option architecture-type = 00:07 { + filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi"; + } elsif option architecture-type = 00:06 { + filename "/warewulf/ipxe/bin-i386-efi/snp.efi"; + } elsif option architecture-type = 00:00 { + filename "/warewulf/ipxe/bin-i386-pcbios/undionly.kpxe"; + } +} + +subnet %{NETWORK} netmask %{NETMASK} { + not authoritative; + # option interface-mtu 9000; + option subnet-mask %{NETMASK}; +} + +# Compute Nodes (WIP - see the dhcpd generator plugin) +{{ compute_nodes }} + +# Node entries will follow below +{{ node_entries }} \ No newline at end of file diff --git a/examples/templates/dnsmasq.jinja b/examples/templates/dnsmasq.jinja new file mode 100644 index 0000000..0b50fac --- /dev/null +++ b/examples/templates/dnsmasq.jinja @@ -0,0 +1,7 @@ +# +# This file was auto-generated by the OpenCHAMI "configurator" tool using the "{{name}}" plugin. +# +# Source code: https://github.com/OpenCHAMI/configurator +# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# +{{ output }} diff --git a/examples/templates/powerman.jinja b/examples/templates/powerman.jinja new file mode 100644 index 0000000..11b1e8c --- /dev/null +++ b/examples/templates/powerman.jinja @@ -0,0 +1,12 @@ +# +# Ansible managed +# +include "/etc/powerman/ipmipower.dev" +include "/etc/powerman/ipmi.dev" + + +# list of devices +{{ devices }} + +# create nodes based on found nodes in hostfile +{{ nodes }} \ No newline at end of file diff --git a/internal/client.go b/internal/client.go index 68038d9..1226095 100644 --- a/internal/client.go +++ b/internal/client.go @@ -2,16 +2,21 @@ package configurator import ( "bytes" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "io" + "net" "net/http" + "os" + "time" "github.com/OpenCHAMI/configurator/internal/util" ) type SmdClient struct { - http.Client + http.Client `json:"-"` Host string `yaml:"host"` Port int `yaml:"port"` AccessToken string `yaml:"access-token"` @@ -19,8 +24,63 @@ type SmdClient struct { type Params = map[string]any type Option func(Params) +type ClientOption func(*SmdClient) -func WithVerbose() Option { +func NewSmdClient(opts ...ClientOption) SmdClient { + client := SmdClient{} + for _, opt := range opts { + opt(&client) + } + return client +} + +func WithHost(host string) ClientOption { + return func(c *SmdClient) { + c.Host = host + } +} + +func WithPort(port int) ClientOption { + return func(c *SmdClient) { + c.Port = port + } +} + +func WithAccessToken(token string) ClientOption { + return func(c *SmdClient) { + c.AccessToken = token + } +} + +func WithCertPool(certPool *x509.CertPool) ClientOption { + return func(c *SmdClient) { + c.Client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + } +} + +func WithSecureTLS(certPath string) ClientOption { + if certPath == "" { + return func(sc *SmdClient) {} + } + cacert, _ := os.ReadFile(certPath) + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + return WithCertPool(certPool) +} + +func WithVerbosity() Option { return func(p util.Params) { p["verbose"] = true } @@ -35,6 +95,11 @@ func NewParams() Params { // Fetch the ethernet interfaces from SMD service using its API. An access token may be required if the SMD // service SMD_JWKS_URL envirnoment variable is set. func (client *SmdClient) FetchEthernetInterfaces(opts ...util.Option) ([]EthernetInterface, error) { + var ( + params = util.GetParams(opts...) + verbose = util.Get[bool](params, "verbose") + eths = []EthernetInterface{} + ) // make request to SMD endpoint b, err := client.makeRequest("/Inventory/EthernetInterfaces") if err != nil { @@ -42,16 +107,14 @@ func (client *SmdClient) FetchEthernetInterfaces(opts ...util.Option) ([]Etherne } // unmarshal response body JSON and extract in object - eths := []EthernetInterface{} // []map[string]any{} err = json.Unmarshal(b, ðs) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) } // print what we got if verbose is set - params := util.GetParams(opts...) - if verbose, ok := params["verbose"].(bool); ok { - if verbose { + if verbose != nil { + if *verbose { fmt.Printf("Ethernet Interfaces: %v\n", string(b)) } } @@ -62,23 +125,40 @@ func (client *SmdClient) FetchEthernetInterfaces(opts ...util.Option) ([]Etherne // Fetch the components from SMD using its API. An access token may be required if the SMD // service SMD_JWKS_URL envirnoment variable is set. func (client *SmdClient) FetchComponents(opts ...util.Option) ([]Component, error) { + var ( + params = util.GetParams(opts...) + verbose = util.Get[bool](params, "verbose") + comps = []Component{} + ) // make request to SMD endpoint b, err := client.makeRequest("/State/Components") if err != nil { - return nil, fmt.Errorf("failed to read HTTP response: %v", err) + return nil, fmt.Errorf("failed to make HTTP request: %v", err) + } + + // make sure our response is actually JSON + if !json.Valid(b) { + return nil, fmt.Errorf("expected valid JSON response: %v", string(b)) } // unmarshal response body JSON and extract in object - comps := []Component{} + var tmp map[string]any + err = json.Unmarshal(b, &tmp) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + b, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) + if err != nil { + return nil, fmt.Errorf("failed to marshal JSON: %v", err) + } err = json.Unmarshal(b, &comps) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) } // print what we got if verbose is set - params := util.GetParams(opts...) - if verbose, ok := params["verbose"].(bool); ok { - if verbose { + if verbose != nil { + if *verbose { fmt.Printf("Components: %v\n", string(b)) } } @@ -86,6 +166,44 @@ func (client *SmdClient) FetchComponents(opts ...util.Option) ([]Component, erro return comps, nil } +func (client *SmdClient) FetchRedfishEndpoints(opts ...util.Option) ([]RedfishEndpoint, error) { + var ( + params = util.GetParams(opts...) + verbose = util.Get[bool](params, "verbose") + eps = []RedfishEndpoint{} + ) + + b, err := client.makeRequest("/Inventory/RedfishEndpoints") + if err != nil { + return nil, fmt.Errorf("failed to make HTTP resquest: %v", err) + } + if !json.Valid(b) { + return nil, fmt.Errorf("expected valid JSON response: %v", string(b)) + } + var tmp map[string]any + err = json.Unmarshal(b, &tmp) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + + b, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) + if err != nil { + return nil, fmt.Errorf("failed to marshal JSON: %v", err) + } + err = json.Unmarshal(b, &eps) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + + if verbose != nil { + if *verbose { + fmt.Printf("Redfish endpoints: %v\n", string(b)) + } + } + + return eps, nil +} + func (client *SmdClient) makeRequest(endpoint string) ([]byte, error) { if client == nil { return nil, fmt.Errorf("client is nil") diff --git a/internal/config.go b/internal/config.go index 7aabb0a..df58cc5 100644 --- a/internal/config.go +++ b/internal/config.go @@ -10,6 +10,12 @@ import ( type Options struct{} +type Target struct { + Templates []string `yaml:"templates,omitempty"` + FilePaths []string `yaml:"files,omitempty"` + RunTargets []string `yaml:"targets,omitempty"` +} + type Jwks struct { Uri string `yaml:"uri"` Retries int `yaml:"retries"` @@ -22,13 +28,14 @@ type Server struct { } type Config struct { - Version string `yaml:"version"` - Server Server `yaml:"server"` - SmdClient SmdClient `yaml:"smd"` - AccessToken string `yaml:"access-token"` - TemplatePaths map[string]string `yaml:"templates"` - Plugins []string `yaml:"plugins"` - Options Options `yaml:"options"` + Version string `yaml:"version"` + Server Server `yaml:"server"` + SmdClient SmdClient `yaml:"smd"` + AccessToken string `yaml:"access-token"` + Targets map[string]Target `yaml:"targets"` + PluginDirs []string `yaml:"plugins"` + CertPath string `yaml:"ca-cert"` + Options Options `yaml:"options"` } func NewConfig() Config { @@ -38,14 +45,22 @@ func NewConfig() Config { Host: "http://127.0.0.1", Port: 27779, }, - TemplatePaths: map[string]string{ - "dnsmasq": "templates/dnsmasq.jinja", - "syslog": "templates/syslog.jinja", - "ansible": "templates/ansible.jinja", - "powerman": "templates/powerman.jinja", - "conman": "templates/conman.jinja", + Targets: map[string]Target{ + "dnsmasq": Target{ + Templates: []string{}, + }, + "conman": Target{ + Templates: []string{}, + }, + "warewulf": Target{ + Templates: []string{ + "templates/warewulf/defaults/node.jinja", + "templates/warewulf/defaults/provision.jinja", + }, + }, }, - Plugins: []string{}, + + PluginDirs: []string{}, Server: Server{ Host: "127.0.0.1", Port: 3334, diff --git a/internal/config.yaml b/internal/config.yaml deleted file mode 100644 index 48308d3..0000000 --- a/internal/config.yaml +++ /dev/null @@ -1,56 +0,0 @@ -version: "0.0.1" -server: - host: "127.0.0.1" - port: 3333 - callback: "/oidc/callback" - -providers: - facebook: "http://facebook.com" - forgejo: "http://git.towk.local:3000" - gitlab: "https://gitlab.newmexicoconsortium.org" - github: "https://github.com" - -authentication: - clients: - - id: "7527e7b4-c96a-4df0-8fc5-00fde18bb65d" - secret: "gto_cc5uvpb5lsdczkwnbarvwmbpv5kcjwg7nhbc75zt65yrfh2ldenq" - name: "forgejo" - issuer: "http://git.towk.local:3000" - scope: - - "openid" - - "profile" - - "read" - - "email" - redirect-uris: - - "http://127.0.0.1:3333/oidc/callback" - - id: "7c0fab1153674a258a705976fcb9468350df3addd91de4ec622fc9ed24bfbcdd" - secret: "a9a8bc55b0cd99236756093adc00ab17855fa507ce106b8038e7f9390ef2ad99" - name: "gitlab" - issuer: "http://gitlab.newmexicoconsortium.org" - scope: - - "openid" - - "profile" - - "email" - redirect-uris: - - "http://127.0.0.1:3333/oidc/callback" - flows: - authorization-code: - state: "" - client-credentials: - -authorization: - urls: - #identities: http://127.0.0.1:4434/admin/identities - trusted-issuers: http://127.0.0.1:4445/admin/trust/grants/jwt-bearer/issuers - login: http://127.0.0.1:4433/self-service/login/api - clients: http://127.0.0.1:4445/admin/clients - authorize: http://127.0.0.1:4444/oauth2/auth - register: http://127.0.0.1:4444/oauth2/register - token: http://127.0.0.1:4444/oauth2/token - - -options: - decode-id-token: true - decode-access-token: true - run-once: true - open-browser: false diff --git a/internal/configurator.go b/internal/configurator.go index a20d5e3..68dae99 100644 --- a/internal/configurator.go +++ b/internal/configurator.go @@ -1,5 +1,7 @@ package configurator +import "encoding/json" + type IPAddr struct { IpAddress string `json:"IPAddress"` Network string `json:"Network"` @@ -16,6 +18,38 @@ type EthernetInterface struct { } type Component struct { + ID string `json:"ID"` + Type string `json:"Type"` + State string `json:"State,omitempty"` + Flag string `json:"Flag,omitempty"` + Enabled *bool `json:"Enabled,omitempty"` + SwStatus string `json:"SoftwareStatus,omitempty"` + Role string `json:"Role,omitempty"` + SubRole string `json:"SubRole,omitempty"` + NID json.Number `json:"NID,omitempty"` + Subtype string `json:"Subtype,omitempty"` + NetType string `json:"NetType,omitempty"` + Arch string `json:"Arch,omitempty"` + Class string `json:"Class,omitempty"` + ReservationDisabled bool `json:"ReservationDisabled,omitempty"` + Locked bool `json:"Locked,omitempty"` +} + +type RedfishEndpoint struct { + ID string `json:"ID"` + Type string `json:"Type"` + Name string `json:"Name,omitempty"` // user supplied descriptive name + Hostname string `json:"Hostname"` + Domain string `json:"Domain"` + FQDN string `json:"FQDN"` + Enabled bool `json:"Enabled"` + UUID string `json:"UUID,omitempty"` + User string `json:"User"` + Password string `json:"Password"` // Temporary until more secure method + UseSSDP bool `json:"UseSSDP,omitempty"` + MACRequired bool `json:"MACRequired,omitempty"` + MACAddr string `json:"MACAddr,omitempty"` + IPAddr string `json:"IPAddress,omitempty"` } type Node struct { diff --git a/internal/generator/generator.go b/internal/generator/generator.go index fabfe0c..6687634 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -3,20 +3,32 @@ package generator import ( "bytes" "fmt" + "maps" "os" + "path/filepath" "plugin" configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/util" "github.com/nikolalohinski/gonja/v2" "github.com/nikolalohinski/gonja/v2/exec" + "github.com/sirupsen/logrus" ) type Mappings = map[string]any +type Files = map[string][]byte type Generator interface { GetName() string - GetGroups() []string - Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) + GetVersion() string + GetDescription() string + Generate(config *configurator.Config, opts ...util.Option) (Files, error) +} + +type Params struct { + Args []string + PluginPaths []string + Target string + Verbose bool } func LoadPlugin(path string) (Generator, error) { @@ -25,14 +37,16 @@ func LoadPlugin(path string) (Generator, error) { return nil, fmt.Errorf("failed to load plugin: %v", err) } + // load the "Generator" symbol from plugin symbol, err := p.Lookup("Generator") if err != nil { - return nil, fmt.Errorf("failed to look up symbol: %v", err) + return nil, fmt.Errorf("failed to look up symbol at path '%s': %v", path, err) } + // assert that the plugin loaded has a valid generator gen, ok := symbol.(Generator) if !ok { - return nil, fmt.Errorf("failed to load the correct symbol type") + return nil, fmt.Errorf("failed to load the correct symbol type at path '%s'", path) } return gen, nil } @@ -45,53 +59,33 @@ func LoadPlugins(dirpath string, opts ...util.Option) (map[string]Generator, err ) items, _ := os.ReadDir(dirpath) - var LoadGenerator = func(path string) (Generator, error) { - // load each generator plugin - p, err := plugin.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to load plugin: %v", err) - } - - // lookup symbol in plugin - symbol, err := p.Lookup("Generator") - if err != nil { - return nil, fmt.Errorf("failed to look up symbol: %v", err) - } - - // assert that the loaded symbol is the correct type - gen, ok := symbol.(Generator) - if !ok { - return nil, fmt.Errorf("failed to load the correct symbol type") - } - return gen, nil - } for _, item := range items { if item.IsDir() { subitems, _ := os.ReadDir(item.Name()) for _, subitem := range subitems { if !subitem.IsDir() { - gen, err := LoadGenerator(subitem.Name()) + gen, err := LoadPlugin(subitem.Name()) if err != nil { fmt.Printf("failed to load generator in directory '%s': %v\n", item.Name(), err) continue } if verbose, ok := params["verbose"].(bool); ok { if verbose { - fmt.Printf("found plugin '%s'\n", item.Name()) + fmt.Printf("-- found plugin '%s'\n", item.Name()) } } gens[gen.GetName()] = gen } } } else { - gen, err := LoadGenerator(dirpath + item.Name()) + gen, err := LoadPlugin(dirpath + item.Name()) if err != nil { fmt.Printf("failed to load generator: %v\n", err) continue } if verbose, ok := params["verbose"].(bool); ok { if verbose { - fmt.Printf("found plugin '%s'\n", dirpath+item.Name()) + fmt.Printf("-- found plugin '%s'\n", dirpath+item.Name()) } } gens[gen.GetName()] = gen @@ -101,10 +95,10 @@ func LoadPlugins(dirpath string, opts ...util.Option) (map[string]Generator, err return gens, nil } -func WithTemplate(_template string) util.Option { +func WithTarget(target string) util.Option { return func(p util.Params) { if p != nil { - p["template"] = _template + p["target"] = target } } } @@ -123,17 +117,19 @@ func WithClient(client configurator.SmdClient) util.Option { } } -// Syntactic sugar generic function to get parameter from util.Params. -func Get[T any](params util.Params, key string) *T { - if v, ok := params[key].(T); ok { - return &v +func WithOption(key string, value any) util.Option { + return func(p util.Params) { + p[key] = value } - return nil } // Helper function to get client in generator plugins. func GetClient(params util.Params) *configurator.SmdClient { - return Get[configurator.SmdClient](params, "client") + return util.Get[configurator.SmdClient](params, "client") +} + +func GetTarget(config *configurator.Config, key string) configurator.Target { + return config.Targets[key] } func GetParams(opts ...util.Option) util.Params { @@ -144,24 +140,108 @@ func GetParams(opts ...util.Option) util.Params { return params } -func Generate(g Generator, config *configurator.Config, opts ...util.Option) { - g.Generate(config, opts...) -} +func ApplyTemplates(mappings map[string]any, paths ...string) (Files, error) { + var ( + data = exec.NewContext(mappings) + outputs = Files{} + ) -func ApplyTemplate(path string, mappings map[string]any) ([]byte, error) { - data := exec.NewContext(mappings) + for _, path := range paths { + // load jinja template from file + t, err := gonja.FromFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read template from file: %v", err) + } - // load jinja template from file - t, err := gonja.FromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %v", err) + // execute/render jinja template + b := bytes.Buffer{} + if err = t.Execute(&b, data); err != nil { + return nil, fmt.Errorf("failed to execute: %v", err) + } + outputs[path] = b.Bytes() } - // execute/render jinja template - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %v", err) + return outputs, nil +} + +func LoadFiles(paths ...string) (Files, error) { + var outputs = Files{} + for _, path := range paths { + expandedPaths, err := filepath.Glob(path) + if err != nil { + return nil, fmt.Errorf("failed to glob path: %v", err) + } + for _, expandedPath := range expandedPaths { + info, err := os.Stat(expandedPath) + if err != nil { + fmt.Println(err) + return nil, fmt.Errorf("failed to stat file or directory: %v", err) + } + // skip any directories found + if info.IsDir() { + continue + } + b, err := os.ReadFile(expandedPath) + if err != nil { + return nil, fmt.Errorf("failed to read file: %v", err) + } + + outputs[expandedPath] = b + } } - return b.Bytes(), nil + return outputs, nil +} + +func Generate(config *configurator.Config, params Params) (Files, error) { + // load generator plugins to generate configs or to print + var ( + generators = make(map[string]Generator) + client = configurator.NewSmdClient( + configurator.WithHost(config.SmdClient.Host), + configurator.WithPort(config.SmdClient.Port), + configurator.WithAccessToken(config.AccessToken), + configurator.WithSecureTLS(config.CertPath), + ) + ) + + // load all plugins from params + for _, path := range params.PluginPaths { + if params.Verbose { + fmt.Printf("loading plugins from '%s'\n", path) + } + gens, err := LoadPlugins(path) + if err != nil { + fmt.Printf("failed to load plugins: %v\n", err) + err = nil + continue + } + + // add loaded generator plugins to set + maps.Copy(generators, gens) + } + + // show available targets then exit + if len(params.Args) == 0 && params.Target == "" { + for g := range generators { + fmt.Printf("-- found generator plugin \"%s\"\n", g) + } + return nil, nil + } + + if params.Target == "" { + logrus.Errorf("no target supplied (--target name)") + } else { + // run the generator plugin from target passed + gen := generators[params.Target] + if gen == nil { + return nil, fmt.Errorf("invalid generator target (%s)", params.Target) + } + return gen.Generate( + config, + WithTarget(gen.GetName()), + WithClient(client), + ) + } + return nil, fmt.Errorf("an unknown error has occurred") } diff --git a/internal/generator/plugins/conman/conman.go b/internal/generator/plugins/conman/conman.go index 1477604..4fcef2c 100644 --- a/internal/generator/plugins/conman/conman.go +++ b/internal/generator/plugins/conman/conman.go @@ -1,14 +1,11 @@ package main import ( - "bytes" "fmt" configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/generator" "github.com/OpenCHAMI/configurator/internal/util" - "github.com/nikolalohinski/gonja/v2" - "github.com/nikolalohinski/gonja/v2/exec" ) type Conman struct{} @@ -17,30 +14,56 @@ func (g *Conman) GetName() string { return "conman" } -func (g *Conman) GetGroups() []string { - return []string{"conman"} +func (g *Conman) GetVersion() string { + return util.GitCommit() } -func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { - params := generator.GetParams(opts...) +func (g *Conman) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + +func (g *Conman) GetGroups() []string { + return []string{""} +} + +func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { var ( - template = params["template"].(string) - path = config.TemplatePaths[template] + params = generator.GetParams(opts...) + client = generator.GetClient(params) + targetKey = params["targets"].(string) // required param + target = config.Targets[targetKey] + eps []configurator.RedfishEndpoint = nil + err error = nil + // serverOpts = "" + // globalOpts = "" + consoles = "" ) - data := exec.NewContext(map[string]any{}) - t, err := gonja.FromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %v", err) - } - output := "# ========== GENERATED BY OCHAMI CONFIGURATOR ==========\n" - output += "# ======================================================" - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %v", err) + // fetch required data from SMD to create config + if client != nil { + eps, err = client.FetchRedfishEndpoints(opts...) + if err != nil { + return nil, fmt.Errorf("failed to fetch redfish endpoints with client: %v", err) + } } - return b.Bytes(), nil + // add any additional conman or server opts + // if extraOpts, ok := params["opts"].(map[string]any); ok { + + // } + + // format output to write to config file + consoles = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" + for _, ep := range eps { + consoles += fmt.Sprintf("CONSOLE name=%s dev=ipmi:%s-bmc ipmiopts=U:%s,P:%s,W:solpayloadsize\n", ep.Name, ep.Name, ep.User, ep.Password) + } + consoles += "# =====================================================================" + + // apply template substitutions and return output as byte array + return generator.ApplyTemplates(generator.Mappings{ + "server_opts": "", + "global_opts": "", + }, target.Templates...) } var Generator Conman diff --git a/internal/generator/plugins/coredhcp/coredhcp.go b/internal/generator/plugins/coredhcp/coredhcp.go index d1d0154..9b76227 100644 --- a/internal/generator/plugins/coredhcp/coredhcp.go +++ b/internal/generator/plugins/coredhcp/coredhcp.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/util" ) @@ -11,12 +13,16 @@ func (g *CoreDhcp) GetName() string { return "coredhcp" } -func (g *CoreDhcp) GetGroups() []string { - return []string{"coredhcp"} +func (g *CoreDhcp) GetVersion() string { + return util.GitCommit() } -func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { - return nil, nil +func (g *CoreDhcp) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s' to generate config files. This plugin is not complete and still a WIP.", g.GetName()) +} + +func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { + return nil, fmt.Errorf("plugin does not implement generation function") } var Generator CoreDhcp diff --git a/internal/generator/plugins/dhcpd/dhcpd.go b/internal/generator/plugins/dhcpd/dhcpd.go new file mode 100644 index 0000000..a56732e --- /dev/null +++ b/internal/generator/plugins/dhcpd/dhcpd.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + + configurator "github.com/OpenCHAMI/configurator/internal" + "github.com/OpenCHAMI/configurator/internal/generator" + "github.com/OpenCHAMI/configurator/internal/util" +) + +type Dhcpd struct{} + +func (g *Dhcpd) GetName() string { + return "dhcpd" +} + +func (g *Dhcpd) GetVersion() string { + return util.GitCommit() +} + +func (g *Dhcpd) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + +func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { + var ( + params = generator.GetParams(opts...) + client = generator.GetClient(params) + targetKey = params["target"].(string) + target = config.Targets[targetKey] + compute_nodes = "" + eths []configurator.EthernetInterface = nil + err error = nil + ) + + // + if client != nil { + eths, err = client.FetchEthernetInterfaces(opts...) + if err != nil { + return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) + } + } + + // check if we have the required params first + if eths == nil { + return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") + } + if len(eths) <= 0 { + return nil, fmt.Errorf("no ethernet interfaces found") + } + + // format output to write to config file + compute_nodes = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" + for _, eth := range eths { + if len(eth.IpAddresses) == 0 { + continue + } + compute_nodes += fmt.Sprintf("host %s { hardware ethernet %s; fixed-address %s} ", eth.ComponentId, eth.MacAddress, eth.IpAddresses[0]) + } + compute_nodes += "# =====================================================================" + + if verbose, ok := params["verbose"].(bool); ok { + if verbose { + fmt.Printf("") + } + } + return generator.ApplyTemplates(generator.Mappings{ + "compute_nodes": compute_nodes, + "node_entries": "", + }, target.Templates...) +} + +var Generator Dhcpd diff --git a/internal/generator/plugins/dnsmasq/dnsmasq.go b/internal/generator/plugins/dnsmasq/dnsmasq.go index c7eb39f..b1b71a7 100644 --- a/internal/generator/plugins/dnsmasq/dnsmasq.go +++ b/internal/generator/plugins/dnsmasq/dnsmasq.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strings" configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/generator" @@ -10,28 +11,19 @@ import ( type DnsMasq struct{} -func TestGenerateDnsMasq() { - var ( - g = DnsMasq{} - config = &configurator.Config{} - client = configurator.SmdClient{} - ) - g.Generate( - config, - generator.WithTemplate("dnsmasq"), - generator.WithClient(client), - ) -} - func (g *DnsMasq) GetName() string { return "dnsmasq" } -func (g *DnsMasq) GetGroups() []string { - return []string{"dnsmasq"} +func (g *DnsMasq) GetVersion() string { + return util.GitCommit() } -func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { +func (g *DnsMasq) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + +func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { // make sure we have a valid config first if config == nil { return nil, fmt.Errorf("invalid config (config is nil)") @@ -39,15 +31,16 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] // set all the defaults for variables var ( - params = generator.GetParams(opts...) - template = params["template"].(string) // required param - path = config.TemplatePaths[template] - eths []configurator.EthernetInterface = nil - err error = nil + params = generator.GetParams(opts...) + client = generator.GetClient(params) + targetKey = params["target"].(string) // required param + target = config.Targets[targetKey] + eths []configurator.EthernetInterface = nil + err error = nil ) // if we have a client, try making the request for the ethernet interfaces - if client, ok := params["client"].(configurator.SmdClient); ok { + if client != nil { eths, err = client.FetchEthernetInterfaces(opts...) if err != nil { return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) @@ -65,12 +58,12 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] // print message if verbose param found if verbose, ok := params["verbose"].(bool); ok { if verbose { - fmt.Printf("path: %s\neth count: %v\n", path, len(eths)) + fmt.Printf("template: \n%s\nethernet interfaces found: %v\n", strings.Join(target.Templates, "\n\t"), len(eths)) } } // format output to write to config file - output := "# ========== GENERATED BY OCHAMI CONFIGURATOR ==========\n" + output := "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" for _, eth := range eths { if eth.Type == "NodeBMC" { output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n" @@ -78,12 +71,13 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) ([] output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n" } } - output += "# ======================================================" + output += "# =====================================================================" // apply template substitutions and return output as byte array - return generator.ApplyTemplate(path, generator.Mappings{ - "hosts": output, - }) + return generator.ApplyTemplates(generator.Mappings{ + "name": g.GetName(), + "output": output, + }, target.Templates...) } var Generator DnsMasq diff --git a/internal/generator/plugins/example/example.go b/internal/generator/plugins/example/example.go new file mode 100644 index 0000000..044bcf7 --- /dev/null +++ b/internal/generator/plugins/example/example.go @@ -0,0 +1,32 @@ +package main + +import ( + "fmt" + + configurator "github.com/OpenCHAMI/configurator/internal" + "github.com/OpenCHAMI/configurator/internal/generator" + "github.com/OpenCHAMI/configurator/internal/util" +) + +type Example struct { + Message string +} + +func (g *Example) GetName() string { + return "example" +} + +func (g *Example) GetVersion() string { + return util.GitCommit() +} + +func (g *Example) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + +func (g *Example) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { + g.Message = ` + This is an example generator plugin. See the file in 'internal/generator/plugins/example/example.go' on + information about constructing plugins and plugin requirements.` + return generator.Files{"example": []byte(g.Message)}, nil +} diff --git a/internal/generator/plugins/hostfile/hostfile.go b/internal/generator/plugins/hostfile/hostfile.go new file mode 100644 index 0000000..9c66f3a --- /dev/null +++ b/internal/generator/plugins/hostfile/hostfile.go @@ -0,0 +1,28 @@ +package main + +import ( + "fmt" + + configurator "github.com/OpenCHAMI/configurator/internal" + "github.com/OpenCHAMI/configurator/internal/util" +) + +type Hostfile struct{} + +func (g *Hostfile) GetName() string { + return "hostfile" +} + +func (g *Hostfile) GetVersion() string { + return util.GitCommit() +} + +func (g *Hostfile) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + +func (g *Hostfile) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { + return nil, fmt.Errorf("plugin does not implement generation function") +} + +var Generator Hostfile diff --git a/internal/generator/plugins/hostfile/hostfile_test.go b/internal/generator/plugins/hostfile/hostfile_test.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/internal/generator/plugins/hostfile/hostfile_test.go @@ -0,0 +1 @@ +package main diff --git a/internal/generator/plugins/powerman/powerman.go b/internal/generator/plugins/powerman/powerman.go index 6c2b5eb..9f68dd7 100644 --- a/internal/generator/plugins/powerman/powerman.go +++ b/internal/generator/plugins/powerman/powerman.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/util" ) @@ -11,12 +13,16 @@ func (g *Powerman) GetName() string { return "powerman" } -func (g *Powerman) GetGroups() []string { - return []string{"powerman"} +func (g *Powerman) GetVersion() string { + return util.GitCommit() } -func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { - return nil, nil +func (g *Powerman) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + +func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { + return nil, fmt.Errorf("plugin does not implement generation function") } var Generator Powerman diff --git a/internal/generator/plugins/syslog/syslog.go b/internal/generator/plugins/syslog/syslog.go index db59ea6..f9caf40 100644 --- a/internal/generator/plugins/syslog/syslog.go +++ b/internal/generator/plugins/syslog/syslog.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + configurator "github.com/OpenCHAMI/configurator/internal" "github.com/OpenCHAMI/configurator/internal/util" ) @@ -11,12 +13,16 @@ func (g *Syslog) GetName() string { return "syslog" } -func (g *Syslog) GetGroups() []string { - return []string{"log"} +func (g *Syslog) GetVersion() string { + return util.GitCommit() } -func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) ([]byte, error) { - return nil, nil +func (g *Syslog) GetDescription() string { + return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) +} + +func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) (map[string][]byte, error) { + return nil, fmt.Errorf("plugin does not implement generation function") } var Generator Syslog diff --git a/internal/generator/plugins/warewulf/warewulf.go b/internal/generator/plugins/warewulf/warewulf.go index 9de624b..a0a610b 100644 --- a/internal/generator/plugins/warewulf/warewulf.go +++ b/internal/generator/plugins/warewulf/warewulf.go @@ -1,12 +1,13 @@ package main import ( - "bytes" "fmt" + "maps" + "strings" configurator "github.com/OpenCHAMI/configurator/internal" - "github.com/nikolalohinski/gonja/v2" - "github.com/nikolalohinski/gonja/v2/exec" + "github.com/OpenCHAMI/configurator/internal/generator" + "github.com/OpenCHAMI/configurator/internal/util" ) type Warewulf struct{} @@ -15,30 +16,87 @@ func (g *Warewulf) GetName() string { return "warewulf" } -func (g *Warewulf) GetGroups() []string { - return []string{"warewulf"} +func (g *Warewulf) GetVersion() string { + return util.GitCommit() } -func (g *Warewulf) Generate(config *configurator.Config, template string) ([]byte, error) { +func (g *Warewulf) GetDescription() string { + return "Configurator generator plugin for 'warewulf' config files." +} + +func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { var ( - path = config.TemplatePaths[template] + params = generator.GetParams(opts...) + client = generator.GetClient(params) + targetKey = params["target"].(string) + target = config.Targets[targetKey] + outputs = make(generator.Files, len(target.FilePaths)+len(target.Templates)) ) - t, err := gonja.FromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %v", err) + // check if our client is included and is valid + if client == nil { + return nil, fmt.Errorf("invalid client (client is nil)") } - output := "# ========== GENERATED BY OCHAMI CONFIGURATOR ==========\n" - output += "# ======================================================" - data := exec.NewContext(map[string]any{ - "hosts": output, - }) - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %v", err) + // if we have a client, try making the request for the ethernet interfaces + eths, err := client.FetchEthernetInterfaces(opts...) + if err != nil { + return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) } - return nil, nil + + // check if we have the required params first + if eths == nil { + return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") + } + if len(eths) <= 0 { + return nil, fmt.Errorf("no ethernet interfaces found") + } + + // print message if verbose param found + if verbose, ok := params["verbose"].(bool); ok { + if verbose { + fmt.Printf("template: \n%s\n ethernet interfaces found: %v\n", strings.Join(target.Templates, "\n\t"), len(eths)) + } + } + + // fetch redfish endpoints and handle errors + eps, err := client.FetchRedfishEndpoints(opts...) + if err != nil { + return nil, fmt.Errorf("failed to fetch redfish endpoints: %v", err) + } + if len(eps) <= 0 { + return nil, fmt.Errorf("no redfish endpoints found") + } + + // format output for template substitution + nodeEntries := "" + + // load files and templates and copy to outputs + files, err := generator.LoadFiles(target.FilePaths...) + if err != nil { + return nil, fmt.Errorf("failed to load files: %v", err) + } + templates, err := generator.ApplyTemplates(generator.Mappings{ + "node_entries": nodeEntries, + }, target.Templates...) + if err != nil { + return nil, fmt.Errorf("failed to load templates: %v", err) + } + + maps.Copy(outputs, files) + maps.Copy(outputs, templates) + + // print message if verbose param is found + if verbose, ok := params["verbose"].(bool); ok { + if verbose { + fmt.Printf("templates and files loaded: \n") + for path, _ := range outputs { + fmt.Printf("\t%s", path) + } + } + } + + return outputs, err } var Generator Warewulf diff --git a/internal/server/server.go b/internal/server/server.go index 5f57073..5c16ca6 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -4,11 +4,13 @@ package server import ( + "encoding/json" "fmt" "net/http" "time" configurator "github.com/OpenCHAMI/configurator/internal" + "github.com/OpenCHAMI/configurator/internal/generator" "github.com/OpenCHAMI/jwtauth/v5" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -19,9 +21,16 @@ var ( tokenAuth *jwtauth.JWTAuth = nil ) +type Jwks struct { + Uri string + Retries int +} type Server struct { *http.Server - JwksUri string `yaml:"jwks-uri"` + Config *configurator.Config + Jwks Jwks `yaml:"jwks"` + GeneratorParams generator.Params + TokenAuth *jwtauth.JWTAuth } func New() *Server { @@ -29,25 +38,28 @@ func New() *Server { Server: &http.Server{ Addr: "localhost:3334", }, - JwksUri: "", + Jwks: Jwks{ + Uri: "", + Retries: 5, + }, } } -func (s *Server) Start(config *configurator.Config) error { +func (s *Server) Serve() error { // create client just for the server to use to fetch data from SMD _ = &configurator.SmdClient{ - Host: config.SmdClient.Host, - Port: config.SmdClient.Port, + Host: s.Config.SmdClient.Host, + Port: s.Config.SmdClient.Port, } // set the server address with config values - s.Server.Addr = fmt.Sprintf("%s:%d", config.Server.Host, config.Server.Port) + s.Server.Addr = fmt.Sprintf("%s:%d", s.Config.Server.Host, s.Config.Server.Port) // fetch JWKS public key from authorization server - if config.Server.Jwks.Uri != "" && tokenAuth == nil { - for i := 0; i < config.Server.Jwks.Retries; i++ { + if s.Config.Server.Jwks.Uri != "" && tokenAuth == nil { + for i := 0; i < s.Config.Server.Jwks.Retries; i++ { var err error - tokenAuth, err = configurator.FetchPublicKeyFromURL(config.Server.Jwks.Uri) + tokenAuth, err = configurator.FetchPublicKeyFromURL(s.Config.Server.Jwks.Uri) if err != nil { logrus.Errorf("failed to fetch JWKS: %w", err) continue @@ -58,52 +70,73 @@ func (s *Server) Start(config *configurator.Config) error { // create new go-chi router with its routes router := chi.NewRouter() - router.Use(middleware.RedirectSlashes) + router.Use(middleware.RequestID) + router.Use(middleware.RealIP) + router.Use(middleware.Logger) + router.Use(middleware.Recoverer) + router.Use(middleware.StripSlashes) router.Use(middleware.Timeout(60 * time.Second)) - router.Group(func(r chi.Router) { - if config.Server.Jwks.Uri != "" { + if s.Config.Server.Jwks.Uri != "" { + router.Group(func(r chi.Router) { r.Use( jwtauth.Verifier(tokenAuth), jwtauth.Authenticator(tokenAuth), ) - } - r.HandleFunc("/target", func(w http.ResponseWriter, r *http.Request) { - // g := generator.Generator{ - // Type: r.URL.Query().Get("type"), - // Template: r.URL.Query().Get("template"), - // } - // NOTE: we probably don't want to hardcode the types, but should do for now - // if _type == "dhcp" { - // // fetch eths from SMD - // eths, err := client.FetchEthernetInterfaces() - // if err != nil { - // logrus.Errorf("failed to fetch DHCP metadata: %v\n", err) - // w.Write([]byte("An error has occurred")) - // return - // } - // if len(eths) <= 0 { - // logrus.Warnf("no ethernet interfaces found") - // w.Write([]byte("no ethernet interfaces found")) - // return - // } - // // generate a new config from that data - - // // b, err := g.GenerateDHCP(config, eths) - // if err != nil { - // logrus.Errorf("failed to generate DHCP: %v", err) - // w.Write([]byte("An error has occurred.")) - // return - // } - // w.Write(b) - // } + // protected routes if using auth + r.HandleFunc("/generate", s.Generate) + r.HandleFunc("/templates", s.ManageTemplates) }) - r.HandleFunc("/templates", func(w http.ResponseWriter, r *http.Request) { - // TODO: handle GET request - // TODO: handle POST request + } else { + // public routes without auth + router.HandleFunc("/generate", s.Generate) + router.HandleFunc("/templates", s.ManageTemplates) + } + + // always public routes go here (none at the moment) - }) - }) s.Handler = router return s.ListenAndServe() } + +func WriteError(w http.ResponseWriter, format string, a ...any) { + errmsg := fmt.Sprintf(format, a...) + fmt.Printf(errmsg) + w.Write([]byte(errmsg)) +} + +func (s *Server) Generate(w http.ResponseWriter, r *http.Request) { + s.GeneratorParams.Target = r.URL.Query().Get("target") + outputs, err := generator.Generate(s.Config, s.GeneratorParams) + if err != nil { + WriteError(w, "failed to generate config: %v", err) + return + } + + // convert byte arrays to string + tmp := map[string]string{} + for path, output := range outputs { + tmp[path] = string(output) + } + + // marshal output to JSON then send + b, err := json.Marshal(tmp) + if err != nil { + WriteError(w, "failed to marshal output: %v", err) + return + } + _, err = w.Write(b) + if err != nil { + WriteError(w, "failed to write response: %v", err) + return + } +} + +func (s *Server) ManageTemplates(w http.ResponseWriter, r *http.Request) { + // TODO: need to implement template managing API first in "internal/generator/templates" or something + _, err := w.Write([]byte("this is not implemented yet")) + if err != nil { + WriteError(w, "failed to write response: %v", err) + return + } +} diff --git a/internal/util/params.go b/internal/util/params.go index cc006fc..2417083 100644 --- a/internal/util/params.go +++ b/internal/util/params.go @@ -35,3 +35,20 @@ func AssertOptionsExist(params Params, opts ...string) []string { } return foundKeys } + +func WithDefault[T any](v T) Option { + return func(p Params) { + p["default"] = v + } +} + +// Syntactic sugar generic function to get parameter from util.Params. +func Get[T any](params Params, key string, opts ...Option) *T { + if v, ok := params[key].(T); ok { + return &v + } + if defaultValue, ok := params["default"].(T); ok { + return &defaultValue + } + return nil +} diff --git a/internal/util/util.go b/internal/util/util.go index cc37bec..8090ac4 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -7,6 +7,8 @@ import ( "io" "net/http" "os" + "os/exec" + "strings" ) func PathExists(path string) (bool, error) { @@ -26,7 +28,7 @@ func MakeRequest(url string, httpMethod string, body []byte, headers map[string] if err != nil { return nil, nil, fmt.Errorf("could not create new HTTP request: %v", err) } - req.Header.Add("User-Agent", "magellan") + req.Header.Add("User-Agent", "configurator") for k, v := range headers { req.Header.Add(k, v) } @@ -41,3 +43,38 @@ func MakeRequest(url string, httpMethod string, body []byte, headers map[string] } return res, b, err } + +func ConvertMapOutput(m map[string][]byte) map[string]string { + n := make(map[string]string, len(m)) + for k, v := range m { + n[k] = string(v) + } + return n +} + +func GitCommit() string { + c := exec.Command("git", "rev-parse", "HEAD") + stdout, err := c.Output() + if err != nil { + return "" + } + + return strings.TrimRight(string(stdout), "\n") +} + +// NOTE: would it be better to use slices.DeleteFunc instead +func RemoveIndex[T comparable](s []T, index int) []T { + ret := make([]T, 0) + ret = append(ret, s[:index]...) + return append(ret, s[index+1:]...) +} + +func CopyIf[T comparable](s []T, condition func(t T) bool) []T { + var f = make([]T, 0) + for _, e := range s { + if condition(e) { + f = append(f, e) + } + } + return f +}