Moved internal/ to pkg/ to make building external plugins possible

This commit is contained in:
David Allen 2024-07-10 12:11:28 -06:00
parent 075b1a1f7f
commit 7361ec739f
No known key found for this signature in database
GPG key ID: 717C593FF60A2ACC
19 changed files with 4 additions and 3 deletions

320
pkg/generator/generator.go Normal file
View file

@ -0,0 +1,320 @@
package generator
import (
"bytes"
"fmt"
"maps"
"os"
"path/filepath"
"plugin"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/util"
"github.com/nikolalohinski/gonja/v2"
"github.com/nikolalohinski/gonja/v2/exec"
"github.com/sirupsen/logrus"
)
type Mappings map[string]any
type FileMap map[string][]byte
type FileList [][]byte
// Generator interface used to define how files are created. Plugins can
// be created entirely independent of the main driver program.
type Generator interface {
GetName() string
GetVersion() string
GetDescription() string
Generate(config *configurator.Config, opts ...util.Option) (FileMap, error)
}
// Params defined and used by the "generate" subcommand.
type Params struct {
Args []string
PluginPaths []string
Target string
Verbose bool
}
// Converts the file outputs from map[string][]byte to map[string]string.
func ConvertContentsToString(f FileMap) map[string]string {
n := make(map[string]string, len(f))
for k, v := range f {
n[k] = string(v)
}
return n
}
// Loads files without applying any Jinja 2 templating.
func LoadFiles(paths ...string) (FileMap, error) {
var outputs = FileMap{}
for _, path := range paths {
expandedPaths, err := filepath.Glob(path)
if err != nil {
return nil, fmt.Errorf("failed to glob path: %v", err)
}
for _, expandedPath := range expandedPaths {
info, err := os.Stat(expandedPath)
if err != nil {
fmt.Println(err)
return nil, fmt.Errorf("failed to stat file or directory: %v", err)
}
// skip any directories found
if info.IsDir() {
continue
}
b, err := os.ReadFile(expandedPath)
if err != nil {
return nil, fmt.Errorf("failed to read file: %v", err)
}
outputs[expandedPath] = b
}
}
return outputs, nil
}
// Loads a single generator plugin given a single file path.
func LoadPlugin(path string) (Generator, error) {
// skip loading plugin if path is a directory with no error
if isDir, err := util.IsDirectory(path); err == nil && isDir {
return nil, nil
} else if err != nil {
return nil, fmt.Errorf("failed to test if path is directory: %v", err)
}
// try and open the plugin
p, err := plugin.Open(path)
if err != nil {
return nil, fmt.Errorf("failed to open plugin: %v", err)
}
// load the "Generator" symbol from plugin
symbol, err := p.Lookup("Generator")
if err != nil {
return nil, fmt.Errorf("failed to look up symbol at path '%s': %v", path, err)
}
// assert that the plugin loaded has a valid generator
gen, ok := symbol.(Generator)
if !ok {
return nil, fmt.Errorf("failed to load the correct symbol type at path '%s'", path)
}
return gen, nil
}
// Loads all generator plugins in a given directory.
//
// Returns a map of generators. Each generator can be accessed by the name
// returned by the generator.GetName() implemented.
func LoadPlugins(dirpath string, opts ...util.Option) (map[string]Generator, error) {
// check if verbose option is supplied
var (
gens = make(map[string]Generator)
params = util.GetParams(opts...)
)
items, _ := os.ReadDir(dirpath)
for _, item := range items {
if item.IsDir() {
subitems, _ := os.ReadDir(item.Name())
for _, subitem := range subitems {
if !subitem.IsDir() {
gen, err := LoadPlugin(subitem.Name())
if err != nil {
fmt.Printf("failed to load generator in directory '%s': %v\n", item.Name(), err)
continue
}
if verbose, ok := params["verbose"].(bool); ok {
if verbose {
fmt.Printf("-- found plugin '%s'\n", item.Name())
}
}
gens[gen.GetName()] = gen
}
}
} else {
gen, err := LoadPlugin(dirpath + item.Name())
if err != nil {
fmt.Printf("failed to load plugin: %v\n", err)
continue
}
if verbose, ok := params["verbose"].(bool); ok {
if verbose {
fmt.Printf("-- found plugin '%s'\n", dirpath+item.Name())
}
}
gens[gen.GetName()] = gen
}
}
return gens, nil
}
// Option to specify "target" in parameter map. This is used to set which generator
// to use to generate a config file.
func WithTarget(target string) util.Option {
return func(p util.Params) {
if p != nil {
p["target"] = target
}
}
}
// Option to specify "type" in parameter map. This is not currently used.
func WithType(_type string) util.Option {
return func(p util.Params) {
if p != nil {
p["type"] = _type
}
}
}
// Option to a specific client to include in implementing plugin generator.Generate().
//
// NOTE: This may be changed to pass some kind of client interface as an argument in
// the future instead.
func WithClient(client configurator.SmdClient) util.Option {
return func(p util.Params) {
p["client"] = client
}
}
// Helper function to get client in generator.Generate() plugin implementations.
func GetClient(params util.Params) *configurator.SmdClient {
return util.Get[configurator.SmdClient](params, "client")
}
// Helper function to get the target in generator.Generate() plugin implementations.
func GetTarget(config *configurator.Config, key string) configurator.Target {
return config.Targets[key]
}
// Helper function to load all options set with With*() into parameter map.
func GetParams(opts ...util.Option) util.Params {
params := util.Params{}
for _, opt := range opts {
opt(params)
}
return params
}
// Wrapper function to slightly abstract away some of the nuances with using gonja
// into a single function call. This function is *mostly* for convenience and
// simplication. If no paths are supplied, then no templates will be applied and
// there will be no output.
//
// The "FileList" returns a slice of byte arrays in the same order as the argument
// list supplied, but with the Jinja templating applied.
func ApplyTemplates(mappings Mappings, contents ...[]byte) (FileList, error) {
var (
data = exec.NewContext(mappings)
outputs = FileList{}
)
for _, b := range contents {
// load jinja template from file
t, err := gonja.FromBytes(b)
if err != nil {
return nil, fmt.Errorf("failed to read template from file: %v", err)
}
// execute/render jinja template
b := bytes.Buffer{}
if err = t.Execute(&b, data); err != nil {
return nil, fmt.Errorf("failed to execute: %v", err)
}
outputs = append(outputs, b.Bytes())
}
return outputs, nil
}
// Wrapper function similiar to "ApplyTemplates" but takes file paths as arguments.
// This function will load templates from a file instead of using file contents.
func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) {
var (
data = exec.NewContext(mappings)
outputs = FileMap{}
)
for _, path := range paths {
// load jinja template from file
t, err := gonja.FromFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read template from file: %v", err)
}
// execute/render jinja template
b := bytes.Buffer{}
if err = t.Execute(&b, data); err != nil {
return nil, fmt.Errorf("failed to execute: %v", err)
}
outputs[path] = b.Bytes()
}
return outputs, nil
}
// Main function to generate a collection of files as a map with the path as the key and
// the contents of the file as the value. This function currently expects a list of plugin
// paths to load all plugins within a directory. Then, each plugin's generator.Generate()
// function is called for each target specified.
//
// This function is the corresponding implementation for the "generate" CLI subcommand.
// It is also call when running the configurator as a service with the "/generate" route.
//
// TODO: Separate loading plugins so we can load them once when running as a service.
func Generate(config *configurator.Config, params Params) (FileMap, error) {
// load generator plugins to generate configs or to print
var (
generators = make(map[string]Generator)
client = configurator.NewSmdClient(
configurator.WithHost(config.SmdClient.Host),
configurator.WithPort(config.SmdClient.Port),
configurator.WithAccessToken(config.AccessToken),
configurator.WithCertPoolFile(config.CertPath),
)
)
// load all plugins from params
for _, path := range params.PluginPaths {
if params.Verbose {
fmt.Printf("loading plugins from '%s'\n", path)
}
gens, err := LoadPlugins(path)
if err != nil {
fmt.Printf("failed to load plugins: %v\n", err)
err = nil
continue
}
// add loaded generator plugins to set
maps.Copy(generators, gens)
}
// show available targets then exit
if len(params.Args) == 0 && params.Target == "" {
for g := range generators {
fmt.Printf("-- found generator plugin \"%s\"\n", g)
}
return nil, nil
}
if params.Target == "" {
logrus.Errorf("no target supplied (--target name)")
} else {
// run the generator plugin from target passed
gen := generators[params.Target]
if gen == nil {
return nil, fmt.Errorf("invalid generator target (%s)", params.Target)
}
return gen.Generate(
config,
WithTarget(gen.GetName()),
WithClient(client),
)
}
return nil, fmt.Errorf("an unknown error has occurred")
}

View file

@ -0,0 +1,68 @@
package main
import (
"fmt"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/generator"
"github.com/OpenCHAMI/configurator/internal/util"
)
type Conman struct{}
func (g *Conman) GetName() string {
return "conman"
}
func (g *Conman) GetVersion() string {
return util.GitCommit()
}
func (g *Conman) GetDescription() string {
return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
}
func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
var (
params = generator.GetParams(opts...)
client = generator.GetClient(params)
targetKey = params["targets"].(string) // required param
target = config.Targets[targetKey]
eps []configurator.RedfishEndpoint = nil
err error = nil
// serverOpts = ""
// globalOpts = ""
consoles = ""
)
// fetch required data from SMD to create config
if client != nil {
eps, err = client.FetchRedfishEndpoints(opts...)
if err != nil {
return nil, fmt.Errorf("failed to fetch redfish endpoints with client: %v", err)
}
}
// add any additional conman or server opts
// if extraOpts, ok := params["opts"].(map[string]any); ok {
// }
// format output to write to config file
consoles = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n"
for _, ep := range eps {
consoles += fmt.Sprintf("CONSOLE name=%s dev=ipmi:%s-bmc ipmiopts=U:%s,P:%s,W:solpayloadsize\n", ep.Name, ep.Name, ep.User, ep.Password)
}
consoles += "# ====================================================================="
// apply template substitutions and return output as byte array
return generator.ApplyTemplateFromFiles(generator.Mappings{
"plugin_name": g.GetName(),
"plugin_version": g.GetVersion(),
"plugin_description": g.GetDescription(),
"server_opts": "",
"global_opts": "",
}, target.Templates...)
}
var Generator Conman

View file

@ -0,0 +1,29 @@
package main
import (
"fmt"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/generator"
"github.com/OpenCHAMI/configurator/internal/util"
)
type CoreDhcp struct{}
func (g *CoreDhcp) GetName() string {
return "coredhcp"
}
func (g *CoreDhcp) GetVersion() string {
return util.GitCommit()
}
func (g *CoreDhcp) GetDescription() string {
return fmt.Sprintf("Configurator generator plugin for '%s' to generate config files. This plugin is not complete and still a WIP.", g.GetName())
}
func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
return nil, fmt.Errorf("plugin does not implement generation function")
}
var Generator CoreDhcp

View file

@ -0,0 +1,76 @@
package main
import (
"fmt"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/generator"
"github.com/OpenCHAMI/configurator/internal/util"
)
type Dhcpd struct{}
func (g *Dhcpd) GetName() string {
return "dhcpd"
}
func (g *Dhcpd) GetVersion() string {
return util.GitCommit()
}
func (g *Dhcpd) GetDescription() string {
return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
}
func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
var (
params = generator.GetParams(opts...)
client = generator.GetClient(params)
targetKey = params["target"].(string)
target = config.Targets[targetKey]
compute_nodes = ""
eths []configurator.EthernetInterface = nil
err error = nil
)
//
if client != nil {
eths, err = client.FetchEthernetInterfaces(opts...)
if err != nil {
return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err)
}
}
// check if we have the required params first
if eths == nil {
return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)")
}
if len(eths) <= 0 {
return nil, fmt.Errorf("no ethernet interfaces found")
}
// format output to write to config file
compute_nodes = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n"
for _, eth := range eths {
if len(eth.IpAddresses) == 0 {
continue
}
compute_nodes += fmt.Sprintf("host %s { hardware ethernet %s; fixed-address %s} ", eth.ComponentId, eth.MacAddress, eth.IpAddresses[0])
}
compute_nodes += "# ====================================================================="
if verbose, ok := params["verbose"].(bool); ok {
if verbose {
fmt.Printf("")
}
}
return generator.ApplyTemplateFromFiles(generator.Mappings{
"plugin_name": g.GetName(),
"plugin_version": g.GetVersion(),
"plugin_description": g.GetDescription(),
"compute_nodes": compute_nodes,
"node_entries": "",
}, target.Templates...)
}
var Generator Dhcpd

View file

@ -0,0 +1,85 @@
package main
import (
"fmt"
"strings"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/generator"
"github.com/OpenCHAMI/configurator/internal/util"
)
type DnsMasq struct{}
func (g *DnsMasq) GetName() string {
return "dnsmasq"
}
func (g *DnsMasq) GetVersion() string {
return util.GitCommit()
}
func (g *DnsMasq) GetDescription() string {
return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
}
func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
// make sure we have a valid config first
if config == nil {
return nil, fmt.Errorf("invalid config (config is nil)")
}
// set all the defaults for variables
var (
params = generator.GetParams(opts...)
client = generator.GetClient(params)
targetKey = params["target"].(string) // required param
target = config.Targets[targetKey]
eths []configurator.EthernetInterface = nil
err error = nil
)
// if we have a client, try making the request for the ethernet interfaces
if client != nil {
eths, err = client.FetchEthernetInterfaces(opts...)
if err != nil {
return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err)
}
}
// check if we have the required params first
if eths == nil {
return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)")
}
if len(eths) <= 0 {
return nil, fmt.Errorf("no ethernet interfaces found")
}
// print message if verbose param found
if verbose, ok := params["verbose"].(bool); ok {
if verbose {
fmt.Printf("template: \n%s\nethernet interfaces found: %v\n", strings.Join(target.Templates, "\n\t"), len(eths))
}
}
// format output to write to config file
output := "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n"
for _, eth := range eths {
if eth.Type == "NodeBMC" {
output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n"
} else {
output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n"
}
}
output += "# ====================================================================="
// apply template substitutions and return output as byte array
return generator.ApplyTemplateFromFiles(generator.Mappings{
"plugin_name": g.GetName(),
"plugin_version": g.GetVersion(),
"plugin_description": g.GetDescription(),
"dhcp-hosts": output,
}, target.Templates...)
}
var Generator DnsMasq

View file

@ -0,0 +1,34 @@
package main
import (
"fmt"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/generator"
"github.com/OpenCHAMI/configurator/internal/util"
)
type Example struct {
Message string
}
func (g *Example) GetName() string {
return "example"
}
func (g *Example) GetVersion() string {
return util.GitCommit()
}
func (g *Example) GetDescription() string {
return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
}
func (g *Example) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
g.Message = `
This is an example generator plugin. See the file in 'internal/generator/plugins/example/example.go' on
information about constructing plugins and plugin requirements.`
return generator.FileMap{"example": []byte(g.Message)}, nil
}
var Generator Example

View file

@ -0,0 +1,29 @@
package main
import (
"fmt"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/generator"
"github.com/OpenCHAMI/configurator/internal/util"
)
type Hostfile struct{}
func (g *Hostfile) GetName() string {
return "hostfile"
}
func (g *Hostfile) GetVersion() string {
return util.GitCommit()
}
func (g *Hostfile) GetDescription() string {
return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
}
func (g *Hostfile) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
return nil, fmt.Errorf("plugin does not implement generation function")
}
var Generator Hostfile

View file

@ -0,0 +1,29 @@
package main
import (
"fmt"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/generator"
"github.com/OpenCHAMI/configurator/internal/util"
)
type Powerman struct{}
func (g *Powerman) GetName() string {
return "powerman"
}
func (g *Powerman) GetVersion() string {
return util.GitCommit()
}
func (g *Powerman) GetDescription() string {
return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
}
func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
return nil, fmt.Errorf("plugin does not implement generation function")
}
var Generator Powerman

View file

@ -0,0 +1,29 @@
package main
import (
"fmt"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/generator"
"github.com/OpenCHAMI/configurator/internal/util"
)
type Syslog struct{}
func (g *Syslog) GetName() string {
return "syslog"
}
func (g *Syslog) GetVersion() string {
return util.GitCommit()
}
func (g *Syslog) GetDescription() string {
return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
}
func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
return nil, fmt.Errorf("plugin does not implement generation function")
}
var Generator Syslog

View file

@ -0,0 +1,102 @@
package main
import (
"fmt"
"maps"
"strings"
configurator "github.com/OpenCHAMI/configurator/internal"
"github.com/OpenCHAMI/configurator/internal/generator"
"github.com/OpenCHAMI/configurator/internal/util"
)
type Warewulf struct{}
func (g *Warewulf) GetName() string {
return "warewulf"
}
func (g *Warewulf) GetVersion() string {
return util.GitCommit()
}
func (g *Warewulf) GetDescription() string {
return "Configurator generator plugin for 'warewulf' config files."
}
func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
var (
params = generator.GetParams(opts...)
client = generator.GetClient(params)
targetKey = params["target"].(string)
target = config.Targets[targetKey]
outputs = make(generator.FileMap, len(target.FilePaths)+len(target.Templates))
)
// check if our client is included and is valid
if client == nil {
return nil, fmt.Errorf("invalid client (client is nil)")
}
// if we have a client, try making the request for the ethernet interfaces
eths, err := client.FetchEthernetInterfaces(opts...)
if err != nil {
return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err)
}
// check if we have the required params first
if eths == nil {
return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)")
}
if len(eths) <= 0 {
return nil, fmt.Errorf("no ethernet interfaces found")
}
// print message if verbose param found
if verbose, ok := params["verbose"].(bool); ok {
if verbose {
fmt.Printf("template: \n%s\n ethernet interfaces found: %v\n", strings.Join(target.Templates, "\n\t"), len(eths))
}
}
// fetch redfish endpoints and handle errors
eps, err := client.FetchRedfishEndpoints(opts...)
if err != nil {
return nil, fmt.Errorf("failed to fetch redfish endpoints: %v", err)
}
if len(eps) <= 0 {
return nil, fmt.Errorf("no redfish endpoints found")
}
// format output for template substitution
nodeEntries := ""
// load files and templates and copy to outputs
files, err := generator.LoadFiles(target.FilePaths...)
if err != nil {
return nil, fmt.Errorf("failed to load files: %v", err)
}
templates, err := generator.ApplyTemplateFromFiles(generator.Mappings{
"node_entries": nodeEntries,
}, target.Templates...)
if err != nil {
return nil, fmt.Errorf("failed to load templates: %v", err)
}
maps.Copy(outputs, files)
maps.Copy(outputs, templates)
// print message if verbose param is found
if verbose, ok := params["verbose"].(bool); ok {
if verbose {
fmt.Printf("templates and files loaded: \n")
for path, _ := range outputs {
fmt.Printf("\t%s", path)
}
}
}
return outputs, err
}
var Generator Warewulf