From 1d862ebd8ceb99878cfee65b1eb20215220be9a4 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 8 Jul 2024 18:18:16 -0600 Subject: [PATCH] Added more API documentation and minor changes --- cmd/fetch.go | 3 + cmd/generate.go | 6 ++ internal/client.go | 11 +++- internal/generator/generator.go | 52 +++++++++++++--- internal/generator/plugins/conman/conman.go | 4 +- .../generator/plugins/coredhcp/coredhcp.go | 2 +- internal/generator/plugins/dhcpd/dhcpd.go | 4 +- internal/generator/plugins/dnsmasq/dnsmasq.go | 4 +- internal/generator/plugins/example/example.go | 4 +- .../generator/plugins/hostfile/hostfile.go | 2 +- .../generator/plugins/powerman/powerman.go | 2 +- internal/generator/plugins/syslog/syslog.go | 2 +- .../generator/plugins/warewulf/warewulf.go | 6 +- internal/server/server.go | 62 ++++++++++++------- internal/util/util.go | 1 + 15 files changed, 117 insertions(+), 48 deletions(-) diff --git a/cmd/fetch.go b/cmd/fetch.go index 4b9913b..738b8d1 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -21,6 +21,7 @@ var ( var fetchCmd = &cobra.Command{ Use: "fetch", Short: "Fetch a config file from a remote instance of configurator", + Long: "This command is simplified to make a HTTP request to the a configurator service.", Run: func(cmd *cobra.Command, args []string) { // make sure a host is set if remoteHost == "" { @@ -28,6 +29,7 @@ var fetchCmd = &cobra.Command{ return } + // add the "Authorization" header if an access token is supplied headers := map[string]string{} if accessToken != "" { headers["Authorization"] = "Bearer " + accessToken @@ -41,6 +43,7 @@ var fetchCmd = &cobra.Command{ logrus.Errorf("failed to make request: %v", err) return } + // handle getting other error codes other than a 200 if res != nil { if res.StatusCode == http.StatusOK { fmt.Printf("%s\n", string(body)) diff --git a/cmd/generate.go b/cmd/generate.go index ed2ea3a..cc55dd2 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -66,6 +66,12 @@ var generateCmd = &cobra.Command{ }, } +// Generate files by supplying a list of targets as string values. Currently, +// targets are defined statically in a config file. Targets are ran recursively +// if more targets are nested in a defined target, but will not run additional +// child targets if it is the same as the parent. +// +// NOTE: This may be changed in the future how this is done. func RunTargets(config *configurator.Config, args []string, targets ...string) { // generate config with each supplied target for _, target := range targets { diff --git a/internal/client.go b/internal/client.go index f351540..f5de764 100644 --- a/internal/client.go +++ b/internal/client.go @@ -16,6 +16,11 @@ import ( ) type ClientOption func(*SmdClient) + +// An struct that's meant to extend functionality of the base HTTP client by +// adding commonly made requests to SMD. The implemented functions are can be +// used in generator plugins to fetch data when it is needed to substitute +// values for the Jinja templates used. type SmdClient struct { http.Client `json:"-"` Host string `yaml:"host"` @@ -23,6 +28,8 @@ type SmdClient struct { AccessToken string `yaml:"access-token"` } +// Constructor function that allows supplying ClientOption arguments to set +// things like the host, port, access token, etc. func NewSmdClient(opts ...ClientOption) SmdClient { client := SmdClient{} for _, opt := range opts { @@ -67,7 +74,8 @@ func WithCertPool(certPool *x509.CertPool) ClientOption { } } -func WithSecureTLS(certPath string) ClientOption { +// FIXME: Need to check for errors when reading from a file +func WithCertPoolFile(certPath string) ClientOption { if certPath == "" { return func(sc *SmdClient) {} } @@ -83,6 +91,7 @@ func WithVerbosity() util.Option { } } +// Create a set of params with all default values. func NewParams() util.Params { return util.Params{ "verbose": false, diff --git a/internal/generator/generator.go b/internal/generator/generator.go index de68551..1ed6c9d 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -16,7 +16,8 @@ import ( ) type Mappings map[string]any -type Files map[string][]byte +type FileMap map[string][]byte +type FileList [][]byte // Generator interface used to define how files are created. Plugins can // be created entirely independent of the main driver program. @@ -24,7 +25,7 @@ type Generator interface { GetName() string GetVersion() string GetDescription() string - Generate(config *configurator.Config, opts ...util.Option) (Files, error) + Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) } // Params defined and used by the "generate" subcommand. @@ -35,7 +36,8 @@ type Params struct { Verbose bool } -func ConvertContentsToString(f Files) map[string]string { +// Converts the file outputs from map[string][]byte to map[string]string. +func ConvertContentsToString(f FileMap) map[string]string { n := make(map[string]string, len(f)) for k, v := range f { n[k] = string(v) @@ -44,8 +46,8 @@ func ConvertContentsToString(f Files) map[string]string { } // Loads files without applying any Jinja 2 templating. -func LoadFiles(paths ...string) (Files, error) { - var outputs = Files{} +func LoadFiles(paths ...string) (FileMap, error) { + var outputs = FileMap{} for _, path := range paths { expandedPaths, err := filepath.Glob(path) if err != nil { @@ -200,11 +202,41 @@ func GetParams(opts ...util.Option) util.Params { // Wrapper function to slightly abstract away some of the nuances with using gonja // into a single function call. This function is *mostly* for convenience and -// simplication. -func ApplyTemplates(mappings map[string]any, paths ...string) (Files, error) { +// simplication. If no paths are supplied, then no templates will be applied and +// there will be no output. +// +// The "FileList" returns a slice of byte arrays in the same order as the argument +// list supplied, but with the Jinja templating applied. +func ApplyTemplates(mappings Mappings, contents ...[]byte) (FileList, error) { var ( data = exec.NewContext(mappings) - outputs = Files{} + outputs = FileList{} + ) + + for _, b := range contents { + // load jinja template from file + t, err := gonja.FromBytes(b) + if err != nil { + return nil, fmt.Errorf("failed to read template from file: %v", err) + } + + // execute/render jinja template + b := bytes.Buffer{} + if err = t.Execute(&b, data); err != nil { + return nil, fmt.Errorf("failed to execute: %v", err) + } + outputs = append(outputs, b.Bytes()) + } + + return outputs, nil +} + +// Wrapper function similiar to "ApplyTemplates" but takes file paths as arguments. +// This function will load templates from a file instead of using file contents. +func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) { + var ( + data = exec.NewContext(mappings) + outputs = FileMap{} ) for _, path := range paths { @@ -234,7 +266,7 @@ func ApplyTemplates(mappings map[string]any, paths ...string) (Files, error) { // It is also call when running the configurator as a service with the "/generate" route. // // TODO: Separate loading plugins so we can load them once when running as a service. -func Generate(config *configurator.Config, params Params) (Files, error) { +func Generate(config *configurator.Config, params Params) (FileMap, error) { // load generator plugins to generate configs or to print var ( generators = make(map[string]Generator) @@ -242,7 +274,7 @@ func Generate(config *configurator.Config, params Params) (Files, error) { configurator.WithHost(config.SmdClient.Host), configurator.WithPort(config.SmdClient.Port), configurator.WithAccessToken(config.AccessToken), - configurator.WithSecureTLS(config.CertPath), + configurator.WithCertPoolFile(config.CertPath), ) ) diff --git a/internal/generator/plugins/conman/conman.go b/internal/generator/plugins/conman/conman.go index f5d629f..b324238 100644 --- a/internal/generator/plugins/conman/conman.go +++ b/internal/generator/plugins/conman/conman.go @@ -22,7 +22,7 @@ func (g *Conman) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { var ( params = generator.GetParams(opts...) client = generator.GetClient(params) @@ -56,7 +56,7 @@ func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (gen consoles += "# =====================================================================" // apply template substitutions and return output as byte array - return generator.ApplyTemplates(generator.Mappings{ + return generator.ApplyTemplateFromFiles(generator.Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), diff --git a/internal/generator/plugins/coredhcp/coredhcp.go b/internal/generator/plugins/coredhcp/coredhcp.go index 68d00c8..eac6359 100644 --- a/internal/generator/plugins/coredhcp/coredhcp.go +++ b/internal/generator/plugins/coredhcp/coredhcp.go @@ -22,7 +22,7 @@ func (g *CoreDhcp) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s' to generate config files. This plugin is not complete and still a WIP.", g.GetName()) } -func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/dhcpd/dhcpd.go b/internal/generator/plugins/dhcpd/dhcpd.go index 1990abc..49c32e8 100644 --- a/internal/generator/plugins/dhcpd/dhcpd.go +++ b/internal/generator/plugins/dhcpd/dhcpd.go @@ -22,7 +22,7 @@ func (g *Dhcpd) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { var ( params = generator.GetParams(opts...) client = generator.GetClient(params) @@ -64,7 +64,7 @@ func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (gene fmt.Printf("") } } - return generator.ApplyTemplates(generator.Mappings{ + return generator.ApplyTemplateFromFiles(generator.Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), diff --git a/internal/generator/plugins/dnsmasq/dnsmasq.go b/internal/generator/plugins/dnsmasq/dnsmasq.go index 3605e52..6483fd1 100644 --- a/internal/generator/plugins/dnsmasq/dnsmasq.go +++ b/internal/generator/plugins/dnsmasq/dnsmasq.go @@ -23,7 +23,7 @@ func (g *DnsMasq) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { // make sure we have a valid config first if config == nil { return nil, fmt.Errorf("invalid config (config is nil)") @@ -74,7 +74,7 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (ge output += "# =====================================================================" // apply template substitutions and return output as byte array - return generator.ApplyTemplates(generator.Mappings{ + return generator.ApplyTemplateFromFiles(generator.Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), diff --git a/internal/generator/plugins/example/example.go b/internal/generator/plugins/example/example.go index f82a72d..05b9fa0 100644 --- a/internal/generator/plugins/example/example.go +++ b/internal/generator/plugins/example/example.go @@ -24,11 +24,11 @@ func (g *Example) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Example) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *Example) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { g.Message = ` This is an example generator plugin. See the file in 'internal/generator/plugins/example/example.go' on information about constructing plugins and plugin requirements.` - return generator.Files{"example": []byte(g.Message)}, nil + return generator.FileMap{"example": []byte(g.Message)}, nil } var Generator Example diff --git a/internal/generator/plugins/hostfile/hostfile.go b/internal/generator/plugins/hostfile/hostfile.go index 4f40f0e..f77e008 100644 --- a/internal/generator/plugins/hostfile/hostfile.go +++ b/internal/generator/plugins/hostfile/hostfile.go @@ -22,7 +22,7 @@ func (g *Hostfile) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Hostfile) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *Hostfile) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/powerman/powerman.go b/internal/generator/plugins/powerman/powerman.go index 45f4000..40246f3 100644 --- a/internal/generator/plugins/powerman/powerman.go +++ b/internal/generator/plugins/powerman/powerman.go @@ -22,7 +22,7 @@ func (g *Powerman) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/syslog/syslog.go b/internal/generator/plugins/syslog/syslog.go index c3250cb..e2309dc 100644 --- a/internal/generator/plugins/syslog/syslog.go +++ b/internal/generator/plugins/syslog/syslog.go @@ -22,7 +22,7 @@ func (g *Syslog) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/internal/generator/plugins/warewulf/warewulf.go b/internal/generator/plugins/warewulf/warewulf.go index a0a610b..f9646b8 100644 --- a/internal/generator/plugins/warewulf/warewulf.go +++ b/internal/generator/plugins/warewulf/warewulf.go @@ -24,13 +24,13 @@ func (g *Warewulf) GetDescription() string { return "Configurator generator plugin for 'warewulf' config files." } -func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { var ( params = generator.GetParams(opts...) client = generator.GetClient(params) targetKey = params["target"].(string) target = config.Targets[targetKey] - outputs = make(generator.Files, len(target.FilePaths)+len(target.Templates)) + outputs = make(generator.FileMap, len(target.FilePaths)+len(target.Templates)) ) // check if our client is included and is valid @@ -76,7 +76,7 @@ func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (g if err != nil { return nil, fmt.Errorf("failed to load files: %v", err) } - templates, err := generator.ApplyTemplates(generator.Mappings{ + templates, err := generator.ApplyTemplateFromFiles(generator.Mappings{ "node_entries": nodeEntries, }, target.Templates...) if err != nil { diff --git a/internal/server/server.go b/internal/server/server.go index af1520d..425a594 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -33,19 +33,27 @@ type Server struct { TokenAuth *jwtauth.JWTAuth } +// Constructor to make a new server instance with an optional config. func New(config *configurator.Config) *Server { + // create default config if none supplied + if config == nil { + c := configurator.NewConfig() + config = &c + } + // return based on config values return &Server{ Config: config, Server: &http.Server{ - Addr: "localhost:3334", + Addr: fmt.Sprintf("%s:%d", config.Server.Host, config.Server.Port), }, Jwks: Jwks{ - Uri: "", - Retries: 5, + Uri: config.Server.Jwks.Uri, + Retries: config.Server.Jwks.Retries, }, } } +// Main function to start up configurator as a service. func (s *Server) Serve() error { // create client just for the server to use to fetch data from SMD _ = &configurator.SmdClient{ @@ -94,50 +102,60 @@ func (s *Server) Serve() error { router.HandleFunc("/templates", s.ManageTemplates) } - // always public routes go here (none at the moment) + // always available public routes go here (none at the moment) s.Handler = router return s.ListenAndServe() } -func WriteError(w http.ResponseWriter, format string, a ...any) { - errmsg := fmt.Sprintf(format, a...) - fmt.Printf(errmsg) - w.Write([]byte(errmsg)) -} - +// This is the corresponding service function to generate templated files, that +// works similarly to the CLI variant. This function takes similiar arguments as +// query parameters that are included in the HTTP request URL. func (s *Server) Generate(w http.ResponseWriter, r *http.Request) { + // get all of the expect query URL params and validate s.GeneratorParams.Target = r.URL.Query().Get("target") - outputs, err := generator.Generate(s.Config, s.GeneratorParams) - if err != nil { - WriteError(w, "failed to generate config: %v", err) + if s.GeneratorParams.Target == "" { + writeError(w, "no targets supplied") return } - // convert byte arrays to string - tmp := map[string]string{} - for path, output := range outputs { - tmp[path] = string(output) + // generate a new config file from supplied params + outputs, err := generator.Generate(s.Config, s.GeneratorParams) + if err != nil { + writeError(w, "failed to generate config: %v", err) + return } - // marshal output to JSON then send + // marshal output to JSON then send response to client + tmp := generator.ConvertContentsToString(outputs) b, err := json.Marshal(tmp) if err != nil { - WriteError(w, "failed to marshal output: %v", err) + writeError(w, "failed to marshal output: %v", err) return } _, err = w.Write(b) if err != nil { - WriteError(w, "failed to write response: %v", err) + writeError(w, "failed to write response: %v", err) return } } +// Incomplete WIP function for managing templates remotely. There is currently no +// internal API to do this yet. +// +// TODO: need to implement template managing API first in "internal/generator/templates" or something func (s *Server) ManageTemplates(w http.ResponseWriter, r *http.Request) { - // TODO: need to implement template managing API first in "internal/generator/templates" or something _, err := w.Write([]byte("this is not implemented yet")) if err != nil { - WriteError(w, "failed to write response: %v", err) + writeError(w, "failed to write response: %v", err) return } } + +// Wrapper function to simplify writting error message responses. This function +// is only intended to be used with the service and nothing else. +func writeError(w http.ResponseWriter, format string, a ...any) { + errmsg := fmt.Sprintf(format, a...) + fmt.Printf(errmsg) + w.Write([]byte(errmsg)) +} diff --git a/internal/util/util.go b/internal/util/util.go index 58ddb47..fd0daa2 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -23,6 +23,7 @@ func PathExists(path string) (bool, error) { return false, err } +// Wrapper function to simplify checking if a path is a directory. func IsDirectory(path string) (bool, error) { // This returns an *os.FileInfo type fileInfo, err := os.Stat(path)