From d7100cefe777f23d9e1ad2c4a43caad5ac8fdb54 Mon Sep 17 00:00:00 2001 From: towk Date: Sat, 10 May 2025 20:54:57 -0600 Subject: [PATCH 01/89] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ca94345..5d405b7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# OpenCHAMI Configurator +# Configurator: Next Generation -The `configurator` is an extensible tool that is capable of dynamically generating files on the fly. The tool includes a built-in generator that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate files based on Jinja 2 template files. The tool and generator plugins are written in Go and plugins can be written by following the ["Creating Generator Plugins"](#creating-generator-plugins) section of this README. +The `configurator` is general-purpose FTP-like service with rendering capabilities using Jinja 2 templating. It is designed with managing configuration files across a fleet of systems in mind and provides flexible mechanisms for populating template mappings. ## Building and Usage From bfd83f35a3c01519704fd1e2d1400c57011c6f3e Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 3 Aug 2025 20:25:18 -0600 Subject: [PATCH 02/89] refactor: initial commit for major rewrite --- CHANGELOG.md | 4 + cmd/config.go | 32 --- cmd/download.go | 7 + cmd/fetch.go | 77 ------ cmd/generate.go | 212 --------------- cmd/inspect.go | 75 ------ cmd/list.go | 0 cmd/profiles.go | 0 cmd/render.go | 0 cmd/root.go | 65 +---- cmd/serve.go | 67 ----- cmd/upload.go | 14 + examples/plugin/test.go | 4 +- examples/templates/conman.jinja | 2 +- examples/templates/dhcpd.jinja | 2 +- examples/templates/dnsmasq.jinja | 2 +- examples/templates/powerman.jinja | 2 +- examples/templates/test.j2 | 2 +- go.mod | 4 +- main.go | 2 +- pkg/{ => auth}/auth.go | 0 pkg/client/client.go | 66 ----- pkg/client/smd.go | 174 ------------- pkg/config/config.go | 100 ------- pkg/configurator.go | 66 ----- pkg/generator/conman.go | 8 +- pkg/generator/coredhcp.go | 4 +- pkg/generator/dhcpd.go | 8 +- pkg/generator/dnsmasq.go | 8 +- pkg/generator/example.go | 4 +- pkg/generator/generator.go | 8 +- pkg/generator/hostfile.go | 4 +- pkg/generator/params.go | 6 +- pkg/generator/powerman.go | 4 +- pkg/generator/syslog.go | 4 +- pkg/generator/templates.go | 2 +- pkg/generator/warewulf.go | 6 +- pkg/plugin.go | 14 + pkg/server/server.go | 327 ----------------------- pkg/service/profile.go | 184 +++++++++++++ pkg/service/routes.go | 38 +++ pkg/service/service.go | 122 +++++++++ res/archlinux/PKGBUILD | 2 +- tests/generate_local.hurl | 22 -- tests/generate_test.go | 418 ------------------------------ 45 files changed, 439 insertions(+), 1733 deletions(-) delete mode 100644 cmd/config.go create mode 100644 cmd/download.go delete mode 100644 cmd/fetch.go delete mode 100644 cmd/generate.go delete mode 100644 cmd/inspect.go create mode 100644 cmd/list.go create mode 100644 cmd/profiles.go create mode 100644 cmd/render.go delete mode 100644 cmd/serve.go create mode 100644 cmd/upload.go rename pkg/{ => auth}/auth.go (100%) delete mode 100644 pkg/client/client.go delete mode 100644 pkg/client/smd.go delete mode 100644 pkg/config/config.go delete mode 100644 pkg/configurator.go create mode 100644 pkg/plugin.go delete mode 100644 pkg/server/server.go create mode 100644 pkg/service/profile.go create mode 100644 pkg/service/routes.go create mode 100644 pkg/service/service.go delete mode 100644 tests/generate_local.hurl delete mode 100644 tests/generate_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a6c68b3..51332f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +[0.2.0] + +- Major rewrite of entire code base + [0.1.0] - Initial prerelease of configurator \ No newline at end of file diff --git a/cmd/config.go b/cmd/config.go deleted file mode 100644 index f56c34c..0000000 --- a/cmd/config.go +++ /dev/null @@ -1,32 +0,0 @@ -package cmd - -import ( - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" -) - -var configCmd = &cobra.Command{ - Use: "config", - Short: "Create a new default config file", - Run: func(cmd *cobra.Command, args []string) { - // create a new config at all args (paths) - // - // TODO: change this to only take a single arg since more - // than one arg is *maybe* a mistake - for _, path := range args { - // check and make sure something doesn't exist first - if exists, err := util.PathExists(path); exists || err != nil { - log.Error().Err(err).Msg("file or directory exists") - continue - } - config.SaveDefault(path) - } - }, -} - -func init() { - rootCmd.AddCommand(configCmd) -} diff --git a/cmd/download.go b/cmd/download.go new file mode 100644 index 0000000..0160d4f --- /dev/null +++ b/cmd/download.go @@ -0,0 +1,7 @@ +package cmd + +import "github.com/spf13/cobra" + +var downloadCmd = cobra.Command{ + Use: "download", +} diff --git a/cmd/fetch.go b/cmd/fetch.go deleted file mode 100644 index 0b105a7..0000000 --- a/cmd/fetch.go +++ /dev/null @@ -1,77 +0,0 @@ -//go:build client || all -// +build client all - -package cmd - -import ( - "fmt" - "net/http" - "os" - - "github.com/OpenCHAMI/configurator/pkg/util" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" -) - -var fetchCmd = &cobra.Command{ - Use: "fetch", - Short: "Fetch a config file from a remote instance of configurator", - Long: "This command is simplified to make a HTTP request to the a configurator service.", - Run: func(cmd *cobra.Command, args []string) { - // make sure a host is set - if remoteHost == "" { - log.Error().Msg("no '--host' argument set") - return - } - - // check if we actually have any targets to run - if len(targets) <= 0 { - log.Error().Msg("must specify a target") - os.Exit(1) - } - - // check to see if an access token is available from env - if conf.AccessToken == "" { - // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead - accessToken := os.Getenv("ACCESS_TOKEN") - if accessToken != "" { - conf.AccessToken = accessToken - } else { - // TODO: try and fetch token first if it is needed - if verbose { - log.Warn().Msg("No token found. Attempting to generate config without one...") - } - } - } - - // add the "Authorization" header if an access token is supplied - headers := map[string]string{} - if accessToken != "" { - headers["Authorization"] = "Bearer " + accessToken - } - - for _, target := range targets { - // make a request for each target - url := fmt.Sprintf("%s/generate?target=%s", remoteHost, target) - res, body, err := util.MakeRequest(url, http.MethodGet, nil, headers) - if err != nil { - log.Error().Err(err).Msg("failed to fetch files") - return - } - // handle getting other error codes other than a 200 - if res != nil { - // NOTE: the server responses are already marshaled to JSON - fmt.Print(string(body)) - } - } - }, -} - -func init() { - fetchCmd.Flags().StringVar(&remoteHost, "host", "", "set the remote configurator host and port") - fetchCmd.Flags().StringSliceVar(&targets, "target", nil, "set the target configs to make") - fetchCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") - fetchCmd.Flags().StringVar(&accessToken, "access-token", "o", "set the output path for config targets") - - rootCmd.AddCommand(fetchCmd) -} diff --git a/cmd/generate.go b/cmd/generate.go deleted file mode 100644 index 578c9f0..0000000 --- a/cmd/generate.go +++ /dev/null @@ -1,212 +0,0 @@ -//go:build client || all -// +build client all - -package cmd - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/OpenCHAMI/configurator/pkg/client" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/generator" - "github.com/OpenCHAMI/configurator/pkg/util" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" -) - -var ( - tokenFetchRetries int - templatePaths []string - pluginPath string - useCompression bool -) - -var generateCmd = &cobra.Command{ - Use: "generate", - Short: "Generate a config file from state management", - Run: func(cmd *cobra.Command, args []string) { - // make sure that we have a token present before trying to make request - if conf.AccessToken == "" { - // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead - accessToken := os.Getenv("ACCESS_TOKEN") - if accessToken != "" { - conf.AccessToken = accessToken - } else { - // TODO: try and fetch token first if it is needed - if verbose { - log.Warn().Msg("No token found. Attempting to generate conf without one...\n") - } - } - } - - // use cert path from cobra if empty - if conf.CertPath == "" { - conf.CertPath = cacertPath - } - - // show conf as JSON and generators if verbose - if verbose { - b, err := json.MarshalIndent(conf, "", " ") - if err != nil { - log.Error().Err(err).Msg("failed to marshal config") - } - // print the config file as JSON - fmt.Printf("%v\n", string(b)) - } - - // run all of the target recursively until completion if provided - if len(targets) > 0 { - RunTargets(&conf, args, targets...) - } else { - if pluginPath == "" { - log.Error().Msg("no plugin path specified") - return - } - - // load the templates to use - templates := map[string]generator.Template{} - for _, path := range templatePaths { - template := generator.Template{} - template.LoadFromFile(path) - if !template.IsEmpty() { - templates[path] = template - } - } - - params := generator.Params{ - Templates: templates, - } - - // set the client options - // params.ClientOpts = append(params.ClientOpts, client.WithHost(remoteHost)) - if conf.AccessToken != "" { - params.ClientOpts = append(params.ClientOpts, client.WithAccessToken(conf.AccessToken)) - } - if conf.CertPath != "" { - params.ClientOpts = append(params.ClientOpts, client.WithCertPoolFile(conf.CertPath)) - } - - // run generator.Generate() with just plugin path and templates provided - outputBytes, err := generator.Generate(&conf, pluginPath, params) - if err != nil { - log.Error().Err(err).Msg("failed to generate files") - } - - // if we have more than one target and output is set, create configs in directory - outputMap := generator.ConvertContentsToString(outputBytes) - writeOutput(outputBytes, len(targets), len(outputMap)) - } - }, -} - -// Generate files by supplying a list of targets as string values. Currently, -// targets are defined statically in a config file. Targets are ran recursively -// if more targets are nested in a defined target, but will not run additional -// child targets if it is the same as the parent. -// -// NOTE: This may be changed in the future how this is done. -func RunTargets(conf *config.Config, args []string, targets ...string) { - // generate config with each supplied target - for _, target := range targets { - outputBytes, err := generator.GenerateWithTarget(conf, target) - if err != nil { - log.Error().Err(err).Str("target", target).Msg("failed to generate config") - os.Exit(1) - } - - // if we have more than one target and output is set, create configs in directory - outputMap := generator.ConvertContentsToString(outputBytes) - writeOutput(outputBytes, len(targets), len(outputMap)) - - // remove any targets that are the same as current to prevent infinite loop - nextTargets := util.CopyIf(conf.Targets[target].RunTargets, func(nextTarget string) bool { - return nextTarget != target - }) - - // ...then, run any other targets that the current target has - RunTargets(conf, args, nextTargets...) - } -} - -func writeOutput(outputBytes generator.FileMap, targetCount int, templateCount int) { - outputMap := generator.ConvertContentsToString(outputBytes) - if outputPath == "" { - // write only to stdout by default - if len(outputMap) == 1 { - for _, contents := range outputMap { - fmt.Printf("%s\n", string(contents)) - } - } else { - for path, contents := range outputMap { - fmt.Printf("-- file: %s, size: %d B\n%s\n", path, len(contents), string(contents)) - } - } - } else if outputPath != "" && targetCount == 1 && templateCount == 1 { - // write just a single file using provided name - for _, contents := range outputBytes { - err := os.WriteFile(outputPath, contents, 0o644) - if err != nil { - log.Error().Err(err).Str("path", outputPath).Msg("failed to write config file") - os.Exit(1) - } - log.Info().Msgf("wrote file to '%s'\n", outputPath) - } - } else if outputPath != "" && targetCount > 1 && useCompression { - // write multiple files to archive, compress, then save to output path - out, err := os.Create(fmt.Sprintf("%s.tar.gz", outputPath)) - if err != nil { - log.Error().Err(err).Str("path", outputPath).Msg("failed to write archive") - os.Exit(1) - } - files := make([]string, len(outputBytes)) - i := 0 - for path := range outputBytes { - files[i] = path - i++ - } - err = util.CreateArchive(files, out) - if err != nil { - log.Error().Err(err).Str("path", outputPath).Msg("failed to create archive") - os.Exit(1) - } - - } else if outputPath != "" && targetCount > 1 || templateCount > 1 { - // write multiple files in directory using template name - err := os.MkdirAll(filepath.Clean(outputPath), 0o755) - if err != nil { - log.Error().Err(err).Str("path", filepath.Clean(outputPath)).Msg("failed to make output directory") - os.Exit(1) - } - for path, contents := range outputBytes { - filename := filepath.Base(path) - cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) - err := os.WriteFile(cleanPath, contents, 0o755) - if err != nil { - log.Error().Err(err).Str("path", path).Msg("failed to write config to file") - os.Exit(1) - } - log.Info().Msgf("wrote file to '%s'\n", cleanPath) - } - } -} - -func init() { - generateCmd.Flags().StringSliceVar(&targets, "target", []string{}, "set the targets to run pre-defined conf") - generateCmd.Flags().StringSliceVar(&templatePaths, "template", []string{}, "set the paths for the Jinja 2 templates to use") - generateCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugin path") - generateCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for conf targets") - generateCmd.Flags().IntVar(&tokenFetchRetries, "fetch-retries", 5, "set the number of retries to fetch an access token") - generateCmd.Flags().StringVar(&remoteHost, "host", "http://localhost", "set the remote host") - generateCmd.Flags().BoolVar(&useCompression, "compress", false, "set whether to archive and compress multiple file outputs") - - // requires either 'target' by itself or 'plugin' and 'templates' together - // generateCmd.MarkFlagsOneRequired("target", "plugin") - generateCmd.MarkFlagsMutuallyExclusive("target", "plugin") - generateCmd.MarkFlagsMutuallyExclusive("target", "template") - generateCmd.MarkFlagsRequiredTogether("plugin", "template") - - rootCmd.AddCommand(generateCmd) -} diff --git a/cmd/inspect.go b/cmd/inspect.go deleted file mode 100644 index 735fa7b..0000000 --- a/cmd/inspect.go +++ /dev/null @@ -1,75 +0,0 @@ -package cmd - -import ( - "fmt" - "io/fs" - "path/filepath" - "strings" - - "github.com/OpenCHAMI/configurator/pkg/generator" - "github.com/OpenCHAMI/configurator/pkg/util" - "github.com/rodaine/table" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" -) - -var ( - byTarget bool -) - -var inspectCmd = &cobra.Command{ - Use: "inspect", - Short: "Inspect generator plugin information", - Long: "The 'inspect' sub-command takes a list of directories and prints all found plugin information.", - Run: func(cmd *cobra.Command, args []string) { - // set up table formatter - table.DefaultHeaderFormatter = func(format string, vals ...interface{}) string { - return strings.ToUpper(fmt.Sprintf(format, vals...)) - } - - // remove duplicate clean paths from CLI - paths := make([]string, len(args)) - for _, path := range args { - paths = append(paths, filepath.Clean(path)) - } - paths = util.RemoveDuplicates(paths) - - // load specific plugins from positional args - var generators = make(map[string]generator.Generator) - for _, path := range paths { - err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - gen, err := generator.LoadPlugin(path) - if err != nil { - return err - } - generators[gen.GetName()] = gen - return nil - }) - - if err != nil { - log.Error().Err(err).Msg("failed to walk directory") - continue - } - } - - // print all generator plugin information found - tbl := table.New("Name", "Version", "Description") - for _, g := range generators { - tbl.AddRow(g.GetName(), g.GetVersion(), g.GetDescription()) - } - if len(generators) > 0 { - tbl.Print() - } - }, -} - -func init() { - inspectCmd.Flags().BoolVar(&byTarget, "by-target", false, "set whether to ") - rootCmd.AddCommand(inspectCmd) -} diff --git a/cmd/list.go b/cmd/list.go new file mode 100644 index 0000000..e69de29 diff --git a/cmd/profiles.go b/cmd/profiles.go new file mode 100644 index 0000000..e69de29 diff --git a/cmd/render.go b/cmd/render.go new file mode 100644 index 0000000..e69de29 diff --git a/cmd/root.go b/cmd/root.go index 583ee6c..2e70c6a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,35 +4,20 @@ import ( "fmt" "os" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" - "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) -var ( - conf config.Config - configPath string - cacertPath string - verbose bool - targets []string - outputPath string - accessToken string - remoteHost string -) - -var rootCmd = &cobra.Command{ - Use: "configurator", - Short: "Dynamically generate files defined by generators", +var rootCmd = cobra.Command{ + Use: "configurator", Run: func(cmd *cobra.Command, args []string) { - if len(args) == 0 { - cmd.Help() - os.Exit(0) - } + }, } func Execute() { + // run initialization code first + initEnv() + if err := rootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -40,39 +25,13 @@ func Execute() { } func init() { - cobra.OnInitialize(InitConfig) - rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config path") - rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set to enable verbose output") - rootCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") + // initialize the config a single time } -func InitConfig() { - // empty from not being set - if configPath != "" { - exists, err := util.PathExists(configPath) - if err != nil { - log.Error().Err(err).Str("path", configPath).Msg("failed to load config") - os.Exit(1) - } else if exists { - conf = config.Load(configPath) - } else { - // show error and exit since a path was specified - log.Error().Str("path", configPath).Msg("config file not found") - os.Exit(1) - } - } else { - // set to the default value and create a new one - configPath = "./config.yaml" - conf = config.New() - } +func initConfigFromFile(path string) { + +} + +func initEnv() { - // - // set environment variables to override config values - // - - // set the JWKS url if we find the CONFIGURATOR_JWKS_URL environment variable - jwksUrl := os.Getenv("CONFIGURATOR_JWKS_URL") - if jwksUrl != "" { - conf.Server.Jwks.Uri = jwksUrl - } } diff --git a/cmd/serve.go b/cmd/serve.go deleted file mode 100644 index 48e3629..0000000 --- a/cmd/serve.go +++ /dev/null @@ -1,67 +0,0 @@ -//go:build server || all -// +build server all - -package cmd - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "os" - - "github.com/OpenCHAMI/configurator/pkg/server" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" -) - -var serveCmd = &cobra.Command{ - Use: "serve", - Short: "Start configurator as a server and listen for requests", - Run: func(cmd *cobra.Command, args []string) { - // make sure that we have a token present before trying to make request - if conf.AccessToken == "" { - // check if ACCESS_TOKEN env var is set if no access token is provided and use that instead - accessToken := os.Getenv("ACCESS_TOKEN") - if accessToken != "" { - conf.AccessToken = accessToken - } else { - if verbose { - log.Warn().Msg("No token found. Continuing without one...\n") - } - } - } - - // show config as JSON and generators if verbose - if verbose { - b, err := json.MarshalIndent(conf, "", "\t") - if err != nil { - log.Error().Err(err).Msg("failed to marshal config") - os.Exit(1) - } - fmt.Printf("%v\n", string(b)) - } - - // start listening with the server - var ( - s *server.Server = server.New(&conf) - err error = s.Serve() - ) - if errors.Is(err, http.ErrServerClosed) { - if verbose { - log.Info().Msg("server closed") - } - } else if err != nil { - log.Error().Err(err).Msg("failed to start server") - os.Exit(1) - } - }, -} - -func init() { - serveCmd.Flags().StringVar(&conf.Server.Host, "host", conf.Server.Host, "set the server host and port") - // serveCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugins directory path") - serveCmd.Flags().StringVar(&conf.Server.Jwks.Uri, "jwks-uri", conf.Server.Jwks.Uri, "set the JWKS url to fetch public key") - serveCmd.Flags().IntVar(&conf.Server.Jwks.Retries, "jwks-fetch-retries", conf.Server.Jwks.Retries, "set the JWKS fetch retry count") - rootCmd.AddCommand(serveCmd) -} diff --git a/cmd/upload.go b/cmd/upload.go new file mode 100644 index 0000000..9fc1f12 --- /dev/null +++ b/cmd/upload.go @@ -0,0 +1,14 @@ +package cmd + +import "github.com/spf13/cobra" + +var uploadCmd = &cobra.Command{ + Use: "upload", + Run: func(cmd *cobra.Command, args []string) { + + }, +} + +func init() { + rootCmd.AddCommand(uploadCmd) +} diff --git a/examples/plugin/test.go b/examples/plugin/test.go index 58aa310..6289e1c 100644 --- a/examples/plugin/test.go +++ b/examples/plugin/test.go @@ -1,8 +1,8 @@ package main import ( - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/generator" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/generator" ) type TestGenerator struct{} diff --git a/examples/templates/conman.jinja b/examples/templates/conman.jinja index 249994c..69e779c 100644 --- a/examples/templates/conman.jinja +++ b/examples/templates/conman.jinja @@ -5,7 +5,7 @@ # Description: {{ plugin_description }} # # Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins # SERVER keepalive=ON SERVER logdir="/var/log/conman" diff --git a/examples/templates/dhcpd.jinja b/examples/templates/dhcpd.jinja index 5613f73..264e590 100644 --- a/examples/templates/dhcpd.jinja +++ b/examples/templates/dhcpd.jinja @@ -5,7 +5,7 @@ # Description: {{ plugin_description }} # # Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins # allow booting; allow bootp; diff --git a/examples/templates/dnsmasq.jinja b/examples/templates/dnsmasq.jinja index d187a9a..1cc4dab 100644 --- a/examples/templates/dnsmasq.jinja +++ b/examples/templates/dnsmasq.jinja @@ -5,6 +5,6 @@ # Description: {{ plugin_description }} # # Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins # {{ dhcp_hosts }} diff --git a/examples/templates/powerman.jinja b/examples/templates/powerman.jinja index 5808124..44c13e4 100644 --- a/examples/templates/powerman.jinja +++ b/examples/templates/powerman.jinja @@ -5,7 +5,7 @@ # Description: {{ plugin_description }} # # Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins # include "/etc/powerman/ipmipower.dev" include "/etc/powerman/ipmi.dev" diff --git a/examples/templates/test.j2 b/examples/templates/test.j2 index df2bb4e..17bd6b6 100644 --- a/examples/templates/test.j2 +++ b/examples/templates/test.j2 @@ -5,7 +5,7 @@ # Description: {{ plugin_description }} # # Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins # # TODO: test variables diff --git a/go.mod b/go.mod index e9ff5f2..dcb6e87 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/OpenCHAMI/configurator +module git.towk2.me/towk/configurator go 1.21.5 @@ -11,7 +11,6 @@ require ( github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700 github.com/rodaine/table v1.2.0 github.com/rs/zerolog v1.33.0 - github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 gopkg.in/yaml.v2 v2.4.0 @@ -35,6 +34,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/segmentio/asm v1.2.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/sys v0.22.0 // indirect diff --git a/main.go b/main.go index a3e5a86..0828d29 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ package main -import "github.com/OpenCHAMI/configurator/cmd" +import "git.towk2.me/towk/configurator/cmd" func main() { cmd.Execute() diff --git a/pkg/auth.go b/pkg/auth/auth.go similarity index 100% rename from pkg/auth.go rename to pkg/auth/auth.go diff --git a/pkg/client/client.go b/pkg/client/client.go deleted file mode 100644 index a4b7454..0000000 --- a/pkg/client/client.go +++ /dev/null @@ -1,66 +0,0 @@ -package client - -import ( - "crypto/tls" - "crypto/x509" - "net" - "net/http" - "os" - "time" -) - -type Option func(*Params) -type Params struct { - Host string `yaml:"host"` - AccessToken string `yaml:"access-token"` - Transport *http.Transport -} - -func ToParams(opts ...Option) *Params { - params := &Params{} - for _, opt := range opts { - opt(params) - } - return params -} - -func WithHost(host string) Option { - return func(c *Params) { - c.Host = host - } -} - -func WithAccessToken(token string) Option { - return func(c *Params) { - c.AccessToken = token - } -} - -func WithCertPool(certPool *x509.CertPool) Option { - return func(c *Params) { - c.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: certPool, - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - Dial: (&net.Dialer{ - Timeout: 120 * time.Second, - KeepAlive: 120 * time.Second, - }).Dial, - TLSHandshakeTimeout: 120 * time.Second, - ResponseHeaderTimeout: 120 * time.Second, - } - } -} - -// FIXME: Need to check for errors when reading from a file -func WithCertPoolFile(certPath string) Option { - if certPath == "" { - return func(sc *Params) {} - } - cacert, _ := os.ReadFile(certPath) - certPool := x509.NewCertPool() - certPool.AppendCertsFromPEM(cacert) - return WithCertPool(certPool) -} diff --git a/pkg/client/smd.go b/pkg/client/smd.go deleted file mode 100644 index eb80ce8..0000000 --- a/pkg/client/smd.go +++ /dev/null @@ -1,174 +0,0 @@ -package client - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/rs/zerolog/log" -) - -// An struct that's meant to extend functionality of the base HTTP client by -// adding commonly made requests to SMD. The implemented functions are can be -// used in generator plugins to fetch data when it is needed to substitute -// values for the Jinja templates used. -type SmdClient struct { - http.Client `json:"-" yaml:"-"` - Host string `yaml:"host"` - Port int `yaml:"port"` - AccessToken string `yaml:"access-token"` -} - -// Constructor function that allows supplying Option arguments to set -// things like the host, port, access token, etc. -func NewSmdClient(opts ...Option) SmdClient { - var ( - params = ToParams(opts...) - client = SmdClient{ - Host: params.Host, - AccessToken: params.AccessToken, - } - ) - - return client -} - -// Fetch the ethernet interfaces from SMD service using its API. An access token may be required if the SMD -// service SMD_JWKS_URL envirnoment variable is set. -func (client *SmdClient) FetchEthernetInterfaces(verbose bool) ([]configurator.EthernetInterface, error) { - var ( - eths = []configurator.EthernetInterface{} - bytes []byte - err error - ) - // make request to SMD endpoint - bytes, err = client.makeRequest("/Inventory/EthernetInterfaces") - if err != nil { - return nil, fmt.Errorf("failed to read HTTP response: %v", err) - } - - // unmarshal response body JSON and extract in object - err = json.Unmarshal(bytes, ðs) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal response: %v", err) - } - - // print what we got if verbose is set - if verbose { - log.Info().Str("ethernet_interfaces", string(bytes)).Msg("found interfaces") - } - - return eths, nil -} - -// Fetch the components from SMD using its API. An access token may be required if the SMD -// service SMD_JWKS_URL envirnoment variable is set. -func (client *SmdClient) FetchComponents(verbose bool) ([]configurator.Component, error) { - var ( - comps = []configurator.Component{} - bytes []byte - err error - ) - // make request to SMD endpoint - bytes, err = client.makeRequest("/State/Components") - if err != nil { - return nil, fmt.Errorf("failed to make HTTP request: %v", err) - } - - // make sure our response is actually JSON - if !json.Valid(bytes) { - return nil, fmt.Errorf("expected valid JSON response: %v", string(bytes)) - } - - // unmarshal response body JSON and extract in object - var tmp map[string]any - err = json.Unmarshal(bytes, &tmp) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal response: %v", err) - } - bytes, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - err = json.Unmarshal(bytes, &comps) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal response: %v", err) - } - - // print what we got if verbose is set - if verbose { - log.Info().Str("components", string(bytes)).Msg("found components") - } - - return comps, nil -} - -// TODO: improve implementation of this function -func (client *SmdClient) FetchRedfishEndpoints(verbose bool) ([]configurator.RedfishEndpoint, error) { - var ( - eps = []configurator.RedfishEndpoint{} - tmp map[string]any - ) - - // make initial request to get JSON with 'RedfishEndpoints' as property - b, err := client.makeRequest("/Inventory/RedfishEndpoints") - if err != nil { - return nil, fmt.Errorf("failed to make HTTP resquest: %v", err) - } - // make sure response is in JSON - if !json.Valid(b) { - return nil, fmt.Errorf("expected valid JSON response: %v", string(b)) - } - err = json.Unmarshal(b, &tmp) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal response: %v", err) - } - - // marshal RedfishEndpoint JSON back to configurator.RedfishEndpoint - b, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - err = json.Unmarshal(b, &eps) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal response: %v", err) - } - - // show the final result - if verbose { - log.Info().Str("redfish_endpoints", string(b)).Msg("found redfish endpoints") - } - - return eps, nil -} - -func (client *SmdClient) makeRequest(endpoint string) ([]byte, error) { - if client == nil { - return nil, fmt.Errorf("client is nil") - } - - // fetch DHCP related information from SMD's endpoint: - url := fmt.Sprintf("%s/hsm/v2%s", client.Host, endpoint) - req, err := http.NewRequest(http.MethodGet, url, bytes.NewBuffer([]byte{})) - if err != nil { - return nil, fmt.Errorf("failed to create new HTTP request: %v", err) - } - - // include access token in authorzation header if found - // NOTE: This shouldn't be needed for this endpoint since it's public - if client.AccessToken != "" { - req.Header.Add("Authorization", "Bearer "+client.AccessToken) - } - - // make the request to SMD - res, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to make request: %v", err) - } - - // read the contents of the response body - return io.ReadAll(res.Body) -} diff --git a/pkg/config/config.go b/pkg/config/config.go deleted file mode 100644 index c20797b..0000000 --- a/pkg/config/config.go +++ /dev/null @@ -1,100 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - - "github.com/rs/zerolog/log" - - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/client" - "gopkg.in/yaml.v2" -) - -type Jwks struct { - Uri string `yaml:"uri"` - Retries int `yaml:"retries,omitempty"` -} - -type Server struct { - Host string `yaml:"host"` - Port int `yaml:"port"` - Jwks Jwks `yaml:"jwks,omitempty"` -} - -type Config struct { - Version string `yaml:"version,omitempty"` - Server Server `yaml:"server,omitempty"` - SmdClient client.SmdClient `yaml:"smd,omitempty"` - AccessToken string `yaml:"access-token,omitempty"` - Targets map[string]configurator.Target `yaml:"targets,omitempty"` - PluginDirs []string `yaml:"plugins,omitempty"` - CertPath string `yaml:"cacert,omitempty"` -} - -// Creates a new config with default parameters. -func New() Config { - return Config{ - Version: "", - SmdClient: client.SmdClient{Host: "http://127.0.0.1:27779"}, - Targets: map[string]configurator.Target{}, - PluginDirs: []string{}, - Server: Server{ - Host: "127.0.0.1:3334", - Jwks: Jwks{ - Uri: "", - Retries: 5, - }, - }, - } -} - -func Load(path string) Config { - var c Config = New() - file, err := os.ReadFile(path) - if err != nil { - log.Error().Err(err).Msg("failed to read config file") - return c - } - err = yaml.Unmarshal(file, &c) - if err != nil { - log.Error().Err(err).Msg("failed to unmarshal config") - return c - } - return c -} - -func (config *Config) Save(path string) { - path = filepath.Clean(path) - if path == "" || path == "." { - path = "config.yaml" - } - data, err := yaml.Marshal(config) - if err != nil { - log.Error().Err(err).Msg("failed to marshal config") - return - } - err = os.WriteFile(path, data, 0o644) - if err != nil { - log.Error().Err(err).Msg("failed to write default config file") - return - } -} - -func SaveDefault(path string) { - path = filepath.Clean(path) - if path == "" || path == "." { - path = "config.yaml" - } - var c = New() - data, err := yaml.Marshal(c) - if err != nil { - log.Error().Err(err).Msg("failed to marshal config") - return - } - err = os.WriteFile(path, data, 0o644) - if err != nil { - log.Error().Err(err).Msg("failed to write default config file") - return - } -} diff --git a/pkg/configurator.go b/pkg/configurator.go deleted file mode 100644 index 7a19a36..0000000 --- a/pkg/configurator.go +++ /dev/null @@ -1,66 +0,0 @@ -package configurator - -import "encoding/json" - -type Target struct { - Plugin string `yaml:"plugin,omitempty"` // Set the plugin or it's path - TemplatePaths []string `yaml:"templates,omitempty"` // Set the template paths - FilePaths []string `yaml:"files,omitempty"` // Set the file paths - RunTargets []string `yaml:"targets,omitempty"` // Set additional targets to run -} - -type IPAddr struct { - IpAddress string `json:"IPAddress"` - Network string `json:"Network"` -} - -type EthernetInterface struct { - Id string - Description string - MacAddress string - LastUpdate string - ComponentId string - Type string - IpAddresses []IPAddr -} - -type Component struct { - ID string `json:"ID"` - Type string `json:"Type"` - State string `json:"State,omitempty"` - Flag string `json:"Flag,omitempty"` - Enabled *bool `json:"Enabled,omitempty"` - SwStatus string `json:"SoftwareStatus,omitempty"` - Role string `json:"Role,omitempty"` - SubRole string `json:"SubRole,omitempty"` - NID json.Number `json:"NID,omitempty"` - Subtype string `json:"Subtype,omitempty"` - NetType string `json:"NetType,omitempty"` - Arch string `json:"Arch,omitempty"` - Class string `json:"Class,omitempty"` - ReservationDisabled bool `json:"ReservationDisabled,omitempty"` - Locked bool `json:"Locked,omitempty"` -} - -type RedfishEndpoint struct { - ID string `json:"ID"` - Type string `json:"Type"` - Name string `json:"Name,omitempty"` // user supplied descriptive name - Hostname string `json:"Hostname"` - Domain string `json:"Domain"` - FQDN string `json:"FQDN"` - Enabled bool `json:"Enabled"` - UUID string `json:"UUID,omitempty"` - User string `json:"User"` - Password string `json:"Password"` // Temporary until more secure method - UseSSDP bool `json:"UseSSDP,omitempty"` - MACRequired bool `json:"MACRequired,omitempty"` - MACAddr string `json:"MACAddr,omitempty"` - IPAddr string `json:"IPAddress,omitempty"` -} - -type Node struct { -} - -type BMC struct { -} diff --git a/pkg/generator/conman.go b/pkg/generator/conman.go index ccadaa6..be8e882 100644 --- a/pkg/generator/conman.go +++ b/pkg/generator/conman.go @@ -3,10 +3,10 @@ package generator import ( "fmt" - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/client" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + configurator "git.towk2.me/towk/configurator/pkg" + "git.towk2.me/towk/configurator/pkg/client" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" ) type Conman struct{} diff --git a/pkg/generator/coredhcp.go b/pkg/generator/coredhcp.go index 0a256e9..ccc79cd 100644 --- a/pkg/generator/coredhcp.go +++ b/pkg/generator/coredhcp.go @@ -3,8 +3,8 @@ package generator import ( "fmt" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" ) type CoreDhcp struct{} diff --git a/pkg/generator/dhcpd.go b/pkg/generator/dhcpd.go index cc32a48..75a41bb 100644 --- a/pkg/generator/dhcpd.go +++ b/pkg/generator/dhcpd.go @@ -3,10 +3,10 @@ package generator import ( "fmt" - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/client" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + configurator "git.towk2.me/towk/configurator/pkg" + "git.towk2.me/towk/configurator/pkg/client" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" ) type DHCPd struct{} diff --git a/pkg/generator/dnsmasq.go b/pkg/generator/dnsmasq.go index 8db68ba..16ea6c9 100644 --- a/pkg/generator/dnsmasq.go +++ b/pkg/generator/dnsmasq.go @@ -3,10 +3,10 @@ package generator import ( "fmt" - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/client" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + configurator "git.towk2.me/towk/configurator/pkg" + "git.towk2.me/towk/configurator/pkg/client" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" ) type DNSMasq struct{} diff --git a/pkg/generator/example.go b/pkg/generator/example.go index f18abe4..12091ba 100644 --- a/pkg/generator/example.go +++ b/pkg/generator/example.go @@ -3,8 +3,8 @@ package generator import ( "fmt" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" ) type Example struct { diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 3397e36..3667130 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -7,10 +7,10 @@ import ( "path/filepath" "plugin" - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/client" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + configurator "git.towk2.me/towk/configurator/pkg" + "git.towk2.me/towk/configurator/pkg/client" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" "github.com/rs/zerolog/log" ) diff --git a/pkg/generator/hostfile.go b/pkg/generator/hostfile.go index 7ce26c8..a9f92e5 100644 --- a/pkg/generator/hostfile.go +++ b/pkg/generator/hostfile.go @@ -3,8 +3,8 @@ package generator import ( "fmt" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" ) type Hostfile struct{} diff --git a/pkg/generator/params.go b/pkg/generator/params.go index e54420a..e9037ec 100644 --- a/pkg/generator/params.go +++ b/pkg/generator/params.go @@ -1,9 +1,9 @@ package generator import ( - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/client" - "github.com/OpenCHAMI/configurator/pkg/config" + configurator "git.towk2.me/towk/configurator/pkg" + "git.towk2.me/towk/configurator/pkg/client" + "git.towk2.me/towk/configurator/pkg/config" ) type ( diff --git a/pkg/generator/powerman.go b/pkg/generator/powerman.go index 08745e5..d8e376f 100644 --- a/pkg/generator/powerman.go +++ b/pkg/generator/powerman.go @@ -3,8 +3,8 @@ package generator import ( "fmt" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" ) type Powerman struct{} diff --git a/pkg/generator/syslog.go b/pkg/generator/syslog.go index 67b28cf..450dcbc 100644 --- a/pkg/generator/syslog.go +++ b/pkg/generator/syslog.go @@ -3,8 +3,8 @@ package generator import ( "fmt" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" ) type Syslog struct{} diff --git a/pkg/generator/templates.go b/pkg/generator/templates.go index 321076c..3028949 100644 --- a/pkg/generator/templates.go +++ b/pkg/generator/templates.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/OpenCHAMI/configurator/pkg/util" + "git.towk2.me/towk/configurator/pkg/util" "github.com/nikolalohinski/gonja/v2" "github.com/nikolalohinski/gonja/v2/exec" "github.com/rs/zerolog/log" diff --git a/pkg/generator/warewulf.go b/pkg/generator/warewulf.go index bdfbda1..5a7c803 100644 --- a/pkg/generator/warewulf.go +++ b/pkg/generator/warewulf.go @@ -5,9 +5,9 @@ import ( "maps" "strings" - "github.com/OpenCHAMI/configurator/pkg/client" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/util" + "git.towk2.me/towk/configurator/pkg/client" + "git.towk2.me/towk/configurator/pkg/config" + "git.towk2.me/towk/configurator/pkg/util" "github.com/rs/zerolog/log" ) diff --git a/pkg/plugin.go b/pkg/plugin.go new file mode 100644 index 0000000..602cfce --- /dev/null +++ b/pkg/plugin.go @@ -0,0 +1,14 @@ +package configurator + +type Plugin interface { + // plugin data + Name() string + Version() string + Description() string + Metadata() map[string]string + + // run the plugin + Init() error + Run() error + Cleanup() error +} diff --git a/pkg/server/server.go b/pkg/server/server.go deleted file mode 100644 index d10ccd7..0000000 --- a/pkg/server/server.go +++ /dev/null @@ -1,327 +0,0 @@ -//go:build server || all -// +build server all - -package server - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "time" - - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/client" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/generator" - "github.com/OpenCHAMI/jwtauth/v5" - "github.com/go-chi/chi/v5" - "github.com/go-chi/chi/v5/middleware" - "github.com/rs/zerolog" - - openchami_authenticator "github.com/openchami/chi-middleware/auth" - openchami_logger "github.com/openchami/chi-middleware/log" - "github.com/rs/zerolog/log" -) - -var ( - tokenAuth *jwtauth.JWTAuth = nil -) - -type Jwks struct { - Uri string - Retries int -} -type Server struct { - *http.Server - Config *config.Config - Jwks Jwks `yaml:"jwks"` - GeneratorParams generator.Params - TokenAuth *jwtauth.JWTAuth - Targets map[string]Target -} - -type Target struct { - Name string `json:"name"` - PluginPath string `json:"plugin"` - Templates []generator.Template `json:"templates"` -} - -// Constructor to make a new server instance with an optional config. -func New(conf *config.Config) *Server { - // create default config if none supplied - if conf == nil { - c := config.New() - conf = &c - } - newServer := &Server{ - Config: conf, - Server: &http.Server{Addr: conf.Server.Host}, - Jwks: Jwks{ - Uri: conf.Server.Jwks.Uri, - Retries: conf.Server.Jwks.Retries, - }, - } - // load templates for server from config - newServer.loadTargets() - log.Debug().Any("targets", newServer.Targets).Msg("new server targets") - return newServer -} - -// Main function to start up configurator as a service. -func (s *Server) Serve() error { - // Setup logger - zerolog.TimeFieldFormat = zerolog.TimeFormatUnix - logger := log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) - - // set the server address with config values - s.Server.Addr = s.Config.Server.Host - - // fetch JWKS public key from authorization server - if s.Config.Server.Jwks.Uri != "" && tokenAuth == nil { - for i := 0; i < s.Config.Server.Jwks.Retries; i++ { - var err error - tokenAuth, err = configurator.FetchPublicKeyFromURL(s.Config.Server.Jwks.Uri) - if err != nil { - log.Error().Err(err).Msgf("failed to fetch JWKS") - continue - } - break - } - } - - // create client with opts to use to fetch data from SMD - opts := []client.Option{ - client.WithHost(s.Config.SmdClient.Host), - client.WithAccessToken(s.Config.AccessToken), - client.WithCertPoolFile(s.Config.CertPath), - } - - // create new go-chi router with its routes - router := chi.NewRouter() - router.Use(middleware.RequestID) - router.Use(middleware.RealIP) - router.Use(middleware.Logger) - router.Use(middleware.Recoverer) - router.Use(middleware.StripSlashes) - router.Use(middleware.Timeout(60 * time.Second)) - router.Use(openchami_logger.OpenCHAMILogger(logger)) - if s.Config.Server.Jwks.Uri != "" { - router.Group(func(r chi.Router) { - r.Use( - jwtauth.Verifier(tokenAuth), - openchami_authenticator.AuthenticatorWithRequiredClaims(tokenAuth, []string{"sub", "iss", "aud"}), - ) - - // protected routes if using auth - r.HandleFunc("/generate", s.Generate(opts...)) - r.Post("/targets", s.createTarget) - }) - } else { - // public routes without auth - router.HandleFunc("/generate", s.Generate(opts...)) - router.Post("/targets", s.createTarget) - } - - // always available public routes go here (none at the moment) - router.HandleFunc("/configurator/status", s.GetStatus) - - s.Handler = router - return s.ListenAndServe() -} - -// TODO: implement a way to shut the server down -func (s *Server) Close() { - -} - -// This is the corresponding service function to generate templated files, that -// works similarly to the CLI variant. This function takes similiar arguments as -// query parameters that are included in the HTTP request URL. -func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - // get all of the expect query URL params and validate - var ( - targetParam string = r.URL.Query().Get("target") - target *Target = s.getTarget(targetParam) - outputs generator.FileMap - err error - ) - s.GeneratorParams = parseGeneratorParams(r, target, opts...) - if targetParam == "" { - err = writeErrorResponse(w, "must specify a target") - log.Error().Err(err).Msg("failed to parse generator params") - return - } - - // try to generate with target supplied by client first - if target != nil { - log.Debug().Any("target", target).Msg("target for Generate()") - outputs, err = generator.Generate(s.Config, target.PluginPath, s.GeneratorParams) - log.Debug().Any("outputs map", outputs).Msgf("after generate") - if err != nil { - log.Error().Err(err).Msg("failed to generate file") - return - } - } else { - // try and generate a new config file from supplied params - log.Debug().Str("target", targetParam).Msg("target for GenerateWithTarget()") - outputs, err = generator.GenerateWithTarget(s.Config, targetParam) - if err != nil { - writeErrorResponse(w, "failed to generate file") - log.Error().Err(err).Msgf("failed to generate file with target '%s'", target) - return - } - } - - // marshal output to JSON then send response to client - tmp := generator.ConvertContentsToString(outputs) - b, err := json.Marshal(tmp) - if err != nil { - writeErrorResponse(w, "failed to marshal output: %v", err) - log.Error().Err(err).Msg("failed to marshal output") - return - } - _, err = w.Write(b) - if err != nil { - writeErrorResponse(w, "failed to write response: %v", err) - log.Error().Err(err).Msg("failed to write response") - return - } - } -} - -func (s *Server) loadTargets() { - // make sure the map is initialized first - if s.Targets == nil { - s.Targets = make(map[string]Target) - } - // add default generator targets - for name, _ := range generator.DefaultGenerators { - serverTarget := Target{ - Name: name, - PluginPath: name, - } - s.Targets[name] = serverTarget - } - // add targets from config to server (overwrites default targets) - for name, target := range s.Config.Targets { - serverTarget := Target{ - Name: name, - } - // only overwrite plugin path if it's set - if target.Plugin != "" { - serverTarget.PluginPath = target.Plugin - } else { - serverTarget.PluginPath = name - } - // add templates using template paths from config - for _, templatePath := range target.TemplatePaths { - template := generator.Template{} - template.LoadFromFile(templatePath) - serverTarget.Templates = append(serverTarget.Templates, template) - } - s.Targets[name] = serverTarget - } -} - -func (s *Server) GetStatus(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - data := map[string]any{ - "code": 200, - "message": "Configurator is healthy", - } - err := json.NewEncoder(w).Encode(data) - if err != nil { - fmt.Printf("failed to encode JSON: %v\n", err) - return - } -} - -// Create a new target with name, generator, templates, and files. -// -// Example: -// -// curl -X POST /target?name=test&plugin=dnsmasq -// -// TODO: need to implement template managing API first in "internal/generator/templates" or something -func (s *Server) createTarget(w http.ResponseWriter, r *http.Request) { - var ( - target = Target{} - bytes []byte - err error - ) - if r == nil { - err = writeErrorResponse(w, "request is invalid") - log.Error().Err(err).Msg("request == nil") - return - } - - bytes, err = io.ReadAll(r.Body) - if err != nil { - writeErrorResponse(w, "failed to read response body: %v", err) - log.Error().Err(err).Msg("failed to read response body") - return - } - defer r.Body.Close() - - err = json.Unmarshal(bytes, &target) - if err != nil { - writeErrorResponse(w, "failed to unmarshal target: %v", err) - log.Error().Err(err).Msg("failed to unmarshal target") - return - } - - // make sure a plugin and at least one template is supplied - if target.Name == "" { - err = writeErrorResponse(w, "target name is required") - log.Error().Err(err).Msg("set target as a URL query parameter") - return - } - if target.PluginPath == "" { - err = writeErrorResponse(w, "generator name is required") - log.Error().Err(err).Msg("must supply a generator name") - return - } - if len(target.Templates) <= 0 { - writeErrorResponse(w, "requires at least one template") - log.Error().Err(err).Msg("must supply at least one template") - return - } - - s.Targets[target.Name] = target - -} - -func (s *Server) getTarget(name string) *Target { - t, ok := s.Targets[name] - if ok { - return &t - } - return nil -} - -// Wrapper function to simplify writting error message responses. This function -// is only intended to be used with the service and nothing else. -func writeErrorResponse(w http.ResponseWriter, format string, a ...any) error { - errmsg := fmt.Sprintf(format, a...) - bytes, _ := json.Marshal(map[string]any{ - "level": "error", - "time": time.Now().Unix(), - "message": errmsg, - }) - http.Error(w, string(bytes), http.StatusInternalServerError) - return fmt.Errorf(errmsg) -} - -func parseGeneratorParams(r *http.Request, target *Target, opts ...client.Option) generator.Params { - var params = generator.Params{ - ClientOpts: opts, - Templates: make(map[string]generator.Template, len(target.Templates)), - } - for i, template := range target.Templates { - params.Templates[fmt.Sprintf("%s_%d", target.Name, i)] = template - } - return params -} diff --git a/pkg/service/profile.go b/pkg/service/profile.go new file mode 100644 index 0000000..cbfe5c0 --- /dev/null +++ b/pkg/service/profile.go @@ -0,0 +1,184 @@ +package service + +import ( + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + + "github.com/go-chi/chi/v5" +) + +type Profile struct { + ID string `json:"id"` // profile ID + Description string `json:"description"` // profile description + Tags []string `json:"tags"` // tags used for ... + Paths []string `json:"paths"` // paths to download + Plugins []string `json:"plugins"` // plugins to run + Data map[string]any `json:"data"` // include render data +} + +func (s *Service) GetProfiles() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + path = s.RootPath + PROFILES_RELPATH + profiles []*Profile + contents []byte + err error + ) + + // walk profiles directory to load all profiles + err = filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + // skip directories + if info.IsDir() { + return nil + } + + // read file contents + var profile *Profile + profile, err = LoadProfile(path) + if err != nil { + return err + } + + profiles = append(profiles, profile) + + fmt.Println(path, info.Size()) + return nil + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + + // marshal and send all the profiles + contents, err = json.Marshal(profiles) + if err != nil { + http.Error(w, fmt.Sprintf("failed to marshal profiles: %v", err), http.StatusInternalServerError) + } + + _, err = w.Write(contents) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +// func (s *Service) CreateProfiles() http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) { +// var ( +// path = chi.URLParam(r, "path") +// err error +// ) + +// } +// } + +func (s *Service) GetProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + id = chi.URLParam(r, "id") + path = s.BuildProfilePath(id) + profile *Profile + contents []byte + err error + ) + + profile, err = LoadProfile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + contents, err = json.Marshal(profile) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + _, err = w.Write(contents) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +func (s *Service) CreateProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + type input struct { + path string `json:"path"` + profile *Profile `json:"profile"` + } + var ( + body []byte + in input + err error + ) + + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // use the request info to build profile + err = json.Unmarshal(body, &in) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // create a new profile on disk + os.WriteFile() + + w.WriteHeader(http.StatusOK) + } +} + +func (s *Service) CreateProfileVar() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} + +func (s *Service) DeleteProfileVar() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} + +func (s *Service) GetProfileData() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} + +func (s *Service) GetProfileVar() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} + +func (s *Service) CreateProfilePath() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} + +func (s *Service) DeleteProfilePath() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} + +func (s *Service) GetProfilePath() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} + +func (s *Service) GetPlugins() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} + +func (s *Service) CreatePlugins() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} + +func (s *Service) DeletePlugins() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) {} +} diff --git a/pkg/service/routes.go b/pkg/service/routes.go new file mode 100644 index 0000000..f0e55ab --- /dev/null +++ b/pkg/service/routes.go @@ -0,0 +1,38 @@ +package service + +import ( + "encoding/json" + "fmt" + "net/http" +) + +func (s *Service) Download() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + } +} + +func (s *Service) Upload() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + } +} + +func (s *Service) List() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + } +} + +func (s *Service) GetStatus(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + data := map[string]any{ + "code": 200, + "message": "Configurator is healthy", + } + err := json.NewEncoder(w).Encode(data) + if err != nil { + fmt.Printf("failed to encode JSON: %v\n", err) + return + } +} diff --git a/pkg/service/service.go b/pkg/service/service.go new file mode 100644 index 0000000..6c3467d --- /dev/null +++ b/pkg/service/service.go @@ -0,0 +1,122 @@ +package service + +import ( + "encoding/json" + "fmt" + "os" + "time" + + configurator "git.towk2.me/towk/configurator/pkg" + "github.com/go-chi/chi/middleware" + "github.com/go-chi/chi/v5" +) + +const ( + PLUGINS_RELPATH = "/plugins" + TEMPLATES_RELPATH = "/templates" + PROFILES_RELPATH = "/profiles" +) + +type Service struct { + RootPath string `yaml:"root,omitempty"` + Environment map[string]string + + // max counts + PluginsMaxCount int + ProfilesMaxCount int +} + +// New creates the directories at specified path +func New() *Service { + return &Service{ + RootPath: ".", + Environment: map[string]string{ + "CONFIGURATOR_HOST_URI": "", + "ACCESS_TOKEN": "", + }, + PluginsMaxCount: 64, + ProfilesMaxCount: 256, + } +} + +// Serve() starts the configurator service and waits for requests. +func (s *Service) Serve() error { + router := chi.NewRouter() + router.Use(middleware.RequestID) + router.Use(middleware.RealIP) + router.Use(middleware.Logger) + router.Use(middleware.Recoverer) + router.Use(middleware.StripSlashes) + router.Use(middleware.Timeout(60 * time.Second)) + + if s.requireAuth() { + + } else { + // general + router.Get("/download", s.Download()) + router.Post("/upload", s.Upload()) + router.Get("/list", s.List()) + + // profiles + router.Get("/profiles", s.GetProfiles()) + // router.Post("/profiles", s.CreateProfiles()) + router.Get("/profile/{id}", s.GetProfile()) + router.Post("/profile/{id}", s.CreateProfile()) + router.Post("/profile/{id}/data/{varname}", s.CreateProfileVar()) + router.Delete("/profile/{id}/data/{varname}", s.DeleteProfileVar()) + router.Get("/profile/{id}/data", s.GetProfileData()) + router.Get("/profile/{id}/data/{varname}", s.GetProfileVar()) + router.Post("/profile/{id}/paths/{path}", s.CreateProfilePath()) + router.Delete("/profile/{id}/paths/{path}", s.DeleteProfilePath()) + router.Get("/profile/{id}/paths/{path}", s.GetProfilePath()) + + // plugins + router.Get("/plugins", s.GetPlugins()) + router.Post("/plugins", s.CreatePlugins()) + router.Delete("/plugins/{id}", s.DeletePlugins()) + } + + // always available public routes go here + router.HandleFunc("/status", s.GetStatus) + return nil +} + +func (s *Service) requireAuth() bool { + return false +} + +func (s *Service) FetchJwks(uri string) { + +} + +func LoadProfile(path string) (*Profile, error) { + return LoadFromJSONFile[Profile](path) +} + +func LoadPlugin(path string) (*configurator.Plugin, error) { + return LoadFromJSONFile[configurator.Plugin](path) +} + +func LoadFromJSONFile[T any](path string) (*T, error) { + var ( + res *T + contents []byte + err error + ) + + contents, err = os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read plugin file: %v", err) + } + + err = json.Unmarshal(contents, &res) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal plugin: %v", err) + } + + return res, err +} + +func (s *Service) BuildProfilePath(id string) string { + return s.RootPath + PLUGINS_RELPATH + "/" + id +} diff --git a/res/archlinux/PKGBUILD b/res/archlinux/PKGBUILD index 56159b3..5796121 100644 --- a/res/archlinux/PKGBUILD +++ b/res/archlinux/PKGBUILD @@ -9,7 +9,7 @@ license=('MIT') groups=("openchami") provides=('configurator') conflicts=('configurator') -https://github.com/OpenCHAMI/configurator/releases/download/v0.1.0-alpha/configurator +https://git.towk2.me/towk/configurator/releases/download/v0.1.0-alpha/configurator source_x86_64=( "${url}/releases/download/v0.1.0-alpha/${pkgname}.tar.gz" ) diff --git a/tests/generate_local.hurl b/tests/generate_local.hurl deleted file mode 100644 index d198e46..0000000 --- a/tests/generate_local.hurl +++ /dev/null @@ -1,22 +0,0 @@ -## -## Run these tests after starting server with `configurator serve...` -## - -# Generate a `example` config with default plugin and template -GET http://127.0.0.1:3334/generate?target=example -HTTP 200 - -# Create a new target using the API -POST http://127.0.0.1:3334/targets -{ - "name": "test", - "plugin": "example", - "templates": [{ - "contents": "This is an example template used with the example plugin." - }] -} -HTTP 200 - -# Test the new target just add from POST above -GET http://127.0.0.1:3334/generate?target=example -HTTP 200 \ No newline at end of file diff --git a/tests/generate_test.go b/tests/generate_test.go deleted file mode 100644 index 21ff978..0000000 --- a/tests/generate_test.go +++ /dev/null @@ -1,418 +0,0 @@ -package tests - -import ( - "encoding/json" - "fmt" - "log" - "net/http" - "os" - "os/exec" - "path/filepath" - "testing" - - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/generator" - "github.com/OpenCHAMI/configurator/pkg/server" - "github.com/OpenCHAMI/configurator/pkg/util" -) - -var ( - workDir string - replaceDir string - err error -) - -// A valid test generator that implements the `Generator` interface. -type TestGenerator struct{} - -func (g *TestGenerator) GetName() string { return "test" } -func (g *TestGenerator) GetVersion() string { return "v1.0.0" } -func (g *TestGenerator) GetDescription() string { - return "This is a plugin created for running tests." -} -func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) { - // Jinja 2 template file - files := map[string]generator.Template{ - "test1": generator.Template{ - Contents: []byte(` -Name: {{plugin_name}} -Version: {{plugin_version}} -Description: {{plugin_description}} - -This is the first test template file. - `), - }, - "test2": generator.Template{ - Contents: []byte(` -This is another testing Jinja 2 template file using {{plugin_name}}. - `), - }, - } - - // apply Jinja templates to file - fileMap, err := generator.ApplyTemplates(generator.Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - }, files) - if err != nil { - return nil, fmt.Errorf("failed to apply templates: %v", err) - } - - // make sure we have a valid config we can access - if config == nil { - return nil, fmt.Errorf("invalid config (config is nil)") - } - - // TODO: make sure we can get a target - - // make sure we have the same number of files in file list - var ( - fileInputCount = len(files) - fileOutputCount = len(fileMap) - ) - if fileInputCount != fileOutputCount { - return nil, fmt.Errorf("file output count (%d) is not the same as the input (%d)", fileOutputCount, fileInputCount) - } - - return fileMap, nil -} - -func init() { - workDir, err = os.Getwd() - if err != nil { - log.Fatalf("failed to get working directory: %v", err) - } - replaceDir = fmt.Sprintf("%s", filepath.Dir(workDir)) -} - -// Test building and loading plugins -func TestPlugin(t *testing.T) { - var ( - testPluginDir = t.TempDir() - testPluginPath = fmt.Sprintf("%s/test-plugin.so", testPluginDir) - testPluginSourcePath = fmt.Sprintf("%s/test-plugin.go", testPluginDir) - testPluginSource = []byte( - `package main - -import ( - "github.com/OpenCHAMI/configurator/pkg/config" - "github.com/OpenCHAMI/configurator/pkg/generator" -) - -type TestGenerator struct{} - -func (g *TestGenerator) GetName() string { return "test" } -func (g *TestGenerator) GetVersion() string { return "v1.0.0" } -func (g *TestGenerator) GetDescription() string { - return "This is a plugin creating for running tests." -} -func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) { - return generator.FileMap{"test": []byte("test")}, nil -} - -var Generator TestGenerator`) - ) - - // get directory to replace remote pkg with local - // _, filename, _, _ := runtime.Caller(0) - // replaceDir := fmt.Sprintf("%s", filepath.Dir(workDir)) - - // show all paths to make sure we're using the correct ones - fmt.Printf("(TestPlugin) working directory: %v\n", workDir) - fmt.Printf("(TestPlugin) plugin directory: %v\n", testPluginDir) - fmt.Printf("(TestPlugin) plugin path: %v\n", testPluginPath) - fmt.Printf("(TestPlugin) plugin source path: %v\n", testPluginSourcePath) - - // make temporary directory to test plugin - err = os.MkdirAll(testPluginDir, 0o777) - if err != nil { - t.Fatalf("failed to make temporary directory: %v", err) - } - - // dump the plugin source code to a file - err = os.WriteFile(testPluginSourcePath, testPluginSource, os.ModePerm) - if err != nil { - t.Fatalf("failed to write test plugin file: %v", err) - } - - // make sure the source file was actually written - fileInfo, err := os.Stat(testPluginSourcePath) - if err != nil { - t.Fatalf("failed to stat path: %v", err) - } - if fileInfo.IsDir() { - t.Fatalf("expected file but found directory") - } - - // change to testing directory to run command - err = os.Chdir(testPluginDir) - if err != nil { - t.Fatalf("failed to 'cd' to temporary directory: %v", err) - } - - // initialize the plugin directory as a Go project - cmd := exec.Command("bash", "-c", "go mod init github.com/OpenCHAMI/configurator-test-plugin") - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to execute command: %v\n%s", err, string(output)) - } - - // use the local `pkg` instead of the release one - cmd = exec.Command("bash", "-c", fmt.Sprintf("go mod edit -replace=github.com/OpenCHAMI/configurator=%s", replaceDir)) - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to execute command: %v\n%s", err, string(output)) - } - - // run `go mod tidy` for dependencies - cmd = exec.Command("bash", "-c", "go mod tidy") - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to execute command: %v\n%s", err, string(output)) - } - - // execute command to build the plugin - cmd = exec.Command("bash", "-c", "go build -buildmode=plugin -o=test-plugin.so test-plugin.go") - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to execute command: %v\n%s", err, string(output)) - } - - // stat the file to confirm that the plugin was built - fileInfo, err = os.Stat(testPluginPath) - if err != nil { - t.Fatalf("failed to stat plugin file: %v", err) - } - if fileInfo.IsDir() { - t.Fatalf("directory file but a file was expected") - } - if fileInfo.Size() <= 0 { - t.Fatal("found an empty file or file with size of 0 bytes") - } - - // test loading plugins both individually and in a dir - gen, err := generator.LoadPlugin("test-plugin.so") - if err != nil { - t.Fatalf("failed to load the test plugin: %v", err) - } - - // test that we have all expected methods with type assertions - if _, ok := gen.(interface { - GetName() string - GetVersion() string - GetDescription() string - Generate(*config.Config, generator.Params) (generator.FileMap, error) - }); !ok { - t.Error("plugin does not implement all of the generator interface") - } - - // test loading plugins from a directory (should just load a single one) - gens, err := generator.LoadPlugins(testPluginDir) - if err != nil { - t.Fatalf("failed to load plugins in '%s': %v", testPluginDir, err) - } - - // test all of the plugins loaded from a directory (should expect same result as above) - for _, gen := range gens { - if _, ok := gen.(interface { - GetName() string - GetVersion() string - GetDescription() string - Generate(*config.Config, generator.Params) (generator.FileMap, error) - }); !ok { - t.Error("plugin does not implement all of the generator interface") - } - } - -} - -// Test that expects to fail with a specific error using a partially -// implemented generator. The purpose of this test is to make sure we're -// seeing the correct error that we would expect in these situations. -// The errors should be something like: -// - no symbol: "failed to look up symbol at path" -// - invalid symbol: "failed to load the correct symbol type at path" -func TestPluginWithInvalidOrNoSymbol(t *testing.T) { - var ( - testPluginDir = t.TempDir() - testPluginPath = fmt.Sprintf("%s/invalid-plugin.so", testPluginDir) - testPluginSourcePath = fmt.Sprintf("%s/invalid-plugin.go", testPluginDir) - testPluginSource = []byte(` -package main - -// An invalid generator that does not or partially implements -// the "Generator" interface. -type InvalidGenerator struct{} -func (g *InvalidGenerator) GetName() string { return "invalid" } -var Generator InvalidGenerator - `) - ) - - // show all paths to make sure we're using the correct ones - fmt.Printf("(TestPluginWithInvalidOrNoSymbol) working directory: %v\n", workDir) - fmt.Printf("(TestPluginWithInvalidOrNoSymbol) plugin directory: %v\n", testPluginDir) - fmt.Printf("(TestPluginWithInvalidOrNoSymbol) plugin path: %v\n", testPluginPath) - fmt.Printf("(TestPluginWithInvalidOrNoSymbol) plugin source path: %v\n", testPluginSourcePath) - - // get directory to replace remote pkg with local - // _, filename, _, _ := runtime.Caller(0) - - // make temporary directory to test plugin - err = os.MkdirAll(testPluginDir, os.ModeDir) - if err != nil { - t.Fatalf("failed to make temporary directory: %v", err) - } - - // dump the plugin source code to a file - err = os.WriteFile(testPluginSourcePath, testPluginSource, os.ModePerm) - if err != nil { - t.Fatalf("failed to write test plugin file: %v", err) - } - - // make sure the source file was actually written - fileInfo, err := os.Stat(testPluginSourcePath) - if err != nil { - t.Fatalf("failed to stat path: %v", err) - } - if fileInfo.IsDir() { - t.Fatalf("expected file but found directory") - } - - // change to testing directory to run command - err = os.Chdir(testPluginDir) - if err != nil { - t.Fatalf("failed to 'cd' to temporary directory: %v", err) - } - - // initialize the plugin directory as a Go project - cmd := exec.Command("bash", "-c", "go mod init github.com/OpenCHAMI/configurator-test-plugin") - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to execute command: %v\n%s", err, string(output)) - } - - // use the local `pkg` instead of the release one - cmd = exec.Command("bash", "-c", fmt.Sprintf("go mod edit -replace=github.com/OpenCHAMI/configurator=%s", replaceDir)) - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to execute command: %v\n%s", err, string(output)) - } - - // run `go mod tidy` for dependencies - cmd = exec.Command("bash", "-c", "go mod tidy") - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to execute command: %v\n%s", err, string(output)) - } - - // execute command to build the plugin - cmd = exec.Command("bash", "-c", "go build -buildmode=plugin -o=invalid-plugin.so invalid-plugin.go") - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to execute command: %v\n%s", err, string(output)) - } - - // stat the file to confirm that the plugin was built - fileInfo, err = os.Stat(testPluginPath) - if err != nil { - t.Fatalf("failed to stat plugin file: %v", err) - } - if fileInfo.IsDir() { - t.Fatalf("directory file but a file was expected") - } - if fileInfo.Size() <= 0 { - t.Fatal("found an empty file or file with size of 0 bytes") - } - - // try and load plugin, but expect specific error - _, err = generator.LoadPlugin(testPluginSourcePath) - if err == nil { - t.Fatalf("expected an error, but returned nil") - } -} - -// Test that expects to successfully "generate" a file using the built-in -// example plugin with no fetching. -// -// NOTE: Normally we would dynamically load a generator from a plugin, but -// we're not doing it here since that's not what is being tested. -func TestGenerateExample(t *testing.T) { - var ( - conf = config.New() - gen = TestGenerator{} - ) - - // make sure our generator returns expected strings - t.Run("properties", func(t *testing.T) { - if gen.GetName() != "test" { - t.Error("test generator return unexpected name") - } - if gen.GetVersion() != "v1.0.0" { - t.Error("test generator return unexpected version") - } - if gen.GetDescription() != "This is a plugin created for running tests." { - t.Error("test generator return unexpected description") - } - }) - - // try to generate a file with templating applied - fileMap, err := gen.Generate(&conf, generator.Params{}) - if err != nil { - t.Fatalf("failed to generate file: %v", err) - } - - // test for 2 expected files to be generated in the output (hint: check the - // TestGenerator.Generate implementation) - if len(fileMap) != 2 { - t.Error("expected 2 files in generated output") - } -} - -// Test that expects to successfully "generate" a file using the built-in -// example plugin but by making a HTTP request to a service instance instead. -// -// NOTE: This test uses the default server settings to run. Also, no need to -// try and load the plugin from a lib here either. -func TestGenerateExampleWithServer(t *testing.T) { - var ( - conf = config.New() - gen = TestGenerator{} - headers = make(map[string]string, 0) - ) - - // NOTE: Currently, the server needs a config to know where to get load plugins, - // and how to handle targets/templates. This will be simplified in the future to - // decoupled the server from required a config altogether. - conf.Targets["test"] = configurator.Target{ - TemplatePaths: []string{}, - FilePaths: []string{}, - } - - // create new server, add test generator, and start in background - server := server.New(&conf) - generator.DefaultGenerators["test"] = &gen - go server.Serve() - - // make request to server to generate a file - res, b, err := util.MakeRequest("http://127.0.0.1:3334/generate?target=test", http.MethodGet, nil, headers) - if err != nil { - t.Fatalf("failed to make request: %v", err) - } - if res.StatusCode != http.StatusOK { - t.Fatalf("expect status code 200 from response but received %d instead", res.StatusCode) - } - - // test for specific output from request - // - // NOTE: we don't actually use the config in this plugin implementation, - // but we do check that a valid config was passed. - fileMap, err := gen.Generate(&conf, generator.Params{}) - if err != nil { - t.Fatalf("failed to generate file: %v", err) - } - for path, contents := range fileMap { - tmp := make(map[string]string, 1) - err := json.Unmarshal(b, &tmp) - if err != nil { - t.Errorf("failed to unmarshal response: %v", err) - continue - } - if string(contents) != string(tmp[path]) { - t.Fatalf("response does not match expected output...\nexpected:%s\noutput:%s", string(contents), string(tmp[path])) - } - } -} From ba684bd1494870276c224a4145ca43df64f8dd8a Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 4 Aug 2025 15:58:02 -0600 Subject: [PATCH 03/89] refactor: more implementation to refactor --- pkg/service/profile.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pkg/service/profile.go b/pkg/service/profile.go index cbfe5c0..e0fb473 100644 --- a/pkg/service/profile.go +++ b/pkg/service/profile.go @@ -118,9 +118,9 @@ func (s *Service) CreateProfile() http.HandlerFunc { profile *Profile `json:"profile"` } var ( - body []byte - in input - err error + body, contents []byte + in input + err error ) body, err = io.ReadAll(r.Body) @@ -136,8 +136,17 @@ func (s *Service) CreateProfile() http.HandlerFunc { return } + // serialize just the profile part + contents, err = json.Marshal(in.profile) + if err != nil { + + } + // create a new profile on disk - os.WriteFile() + err = os.WriteFile(in.path, contents, os.ModePerm) + if err != nil { + + } w.WriteHeader(http.StatusOK) } From 50e6b5309182671f844a85e2d4f3f5641286576c Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 4 Aug 2025 22:32:41 -0600 Subject: [PATCH 04/89] refactor: more implementation to refactor and deleted files --- examples/plugin/test.go | 23 ---- go.mod | 21 +-- go.sum | 86 ++---------- pkg/generator/conman.go | 56 -------- pkg/generator/coredhcp.go | 26 ---- pkg/generator/dhcpd.go | 64 --------- pkg/generator/dnsmasq.go | 71 ---------- pkg/generator/example.go | 31 ----- pkg/generator/generator.go | 264 ------------------------------------- pkg/generator/hostfile.go | 26 ---- pkg/generator/params.go | 43 ------ pkg/generator/powerman.go | 26 ---- pkg/generator/syslog.go | 26 ---- pkg/generator/templates.go | 98 -------------- pkg/generator/warewulf.go | 78 ----------- pkg/service/profile.go | 81 +++++++++--- pkg/service/service.go | 9 +- 17 files changed, 84 insertions(+), 945 deletions(-) delete mode 100644 examples/plugin/test.go delete mode 100644 pkg/generator/conman.go delete mode 100644 pkg/generator/coredhcp.go delete mode 100644 pkg/generator/dhcpd.go delete mode 100644 pkg/generator/dnsmasq.go delete mode 100644 pkg/generator/example.go delete mode 100644 pkg/generator/generator.go delete mode 100644 pkg/generator/hostfile.go delete mode 100644 pkg/generator/params.go delete mode 100644 pkg/generator/powerman.go delete mode 100644 pkg/generator/syslog.go delete mode 100644 pkg/generator/templates.go delete mode 100644 pkg/generator/warewulf.go diff --git a/examples/plugin/test.go b/examples/plugin/test.go deleted file mode 100644 index 6289e1c..0000000 --- a/examples/plugin/test.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/generator" -) - -type TestGenerator struct{} - -func (g *TestGenerator) GetName() string { return "test" } -func (g *TestGenerator) GetVersion() string { return "v1.0.0" } -func (g *TestGenerator) GetDescription() string { - return "This is a plugin creating for running tests." -} -func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) { - return generator.ApplyTemplates(generator.Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - }, params.Templates) -} - -var Generator TestGenerator diff --git a/go.mod b/go.mod index dcb6e87..97ef719 100644 --- a/go.mod +++ b/go.mod @@ -4,39 +4,28 @@ go 1.21.5 require ( github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 + github.com/go-chi/chi v1.5.5 github.com/go-chi/chi/v5 v5.1.0 github.com/lestrrat-go/jwx/v2 v2.1.1 - github.com/nikolalohinski/gonja/v2 v2.2.0 - github.com/openchami/chi-middleware/auth v0.0.0-20240812224658-b16b83c70700 - github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700 - github.com/rodaine/table v1.2.0 - github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.0 + github.com/tidwall/sjson v1.2.5 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 - gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/kr/text v0.1.0 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/segmentio/asm v1.2.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tidwall/gjson v1.14.2 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect ) diff --git a/go.sum b/go.sum index 8452dee..b8090b3 100644 --- a/go.sum +++ b/go.sum @@ -1,39 +1,19 @@ -github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= -github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE= +github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= @@ -46,81 +26,35 @@ github.com/lestrrat-go/jwx/v2 v2.1.1 h1:Y2ltVl8J6izLYFs54BVcpXLv5msSW4o8eXwnzZLI github.com/lestrrat-go/jwx/v2 v2.1.1/go.mod h1:4LvZg7oxu6Q5VJwn7Mk/UwooNRnTHUpXBj2C4j3HNx0= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/nikolalohinski/gonja/v2 v2.2.0 h1:tAs3BDHNjvPj48F2BL5t7iVhN32HhgeldAl3EmdsLh8= -github.com/nikolalohinski/gonja/v2 v2.2.0/go.mod h1:l9DuWJvT/BddBr2SsmEimESD6msSqRw7u5HzI2Um+sc= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= -github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= -github.com/openchami/chi-middleware/auth v0.0.0-20240812224658-b16b83c70700 h1:XADGipD2FZ9swuFUqeL7h63j3voiq9qA7P0aKsqgZKg= -github.com/openchami/chi-middleware/auth v0.0.0-20240812224658-b16b83c70700/go.mod h1:kswb9kU5cZAFRAvf1dAUJRWbQyjDEb0qkxW4ncDdEXg= -github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700 h1:Gzt5f6RK39CHvY3SJudzBb/RK4tVh/S3CpJ0eQlbNdg= -github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700/go.mod h1:UuXvr2loD4MtvZeKr57W0WpBs+gm0KM1kdtcXrE8M6s= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rodaine/table v1.2.0 h1:38HEnwK4mKSHQJIkavVj+bst1TEY7j9zhLMWu4QJrMA= -github.com/rodaine/table v1.2.0/go.mod h1:wejb/q/Yd4T/SVmBSRMr7GCq3KlcZp3gyNYdLSBhkaE= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= -github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/generator/conman.go b/pkg/generator/conman.go deleted file mode 100644 index be8e882..0000000 --- a/pkg/generator/conman.go +++ /dev/null @@ -1,56 +0,0 @@ -package generator - -import ( - "fmt" - - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Conman struct{} - -func (g *Conman) GetName() string { - return "conman" -} - -func (g *Conman) GetVersion() string { - return util.GitCommit() -} - -func (g *Conman) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Conman) Generate(config *config.Config, params Params) (FileMap, error) { - var ( - smdClient = client.NewSmdClient(params.ClientOpts...) - eps = []configurator.RedfishEndpoint{} - err error = nil - consoles = "" - ) - - // fetch required data from SMD to create config - eps, err = smdClient.FetchRedfishEndpoints(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch redfish endpoints with client: %v", err) - } - - // format output to write to config file - consoles = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" - for _, ep := range eps { - consoles += fmt.Sprintf("CONSOLE name=%s dev=ipmi:%s-bmc ipmiopts=U:%s,P:%s,W:solpayloadsize\n", ep.Name, ep.Name, ep.User, ep.Password) - } - consoles += "# =====================================================================" - - // apply template substitutions and return output as byte array - return ApplyTemplates(Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - "server_opts": "", - "global_opts": "", - "consoles": consoles, - }, params.Templates) -} diff --git a/pkg/generator/coredhcp.go b/pkg/generator/coredhcp.go deleted file mode 100644 index ccc79cd..0000000 --- a/pkg/generator/coredhcp.go +++ /dev/null @@ -1,26 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type CoreDhcp struct{} - -func (g *CoreDhcp) GetName() string { - return "coredhcp" -} - -func (g *CoreDhcp) GetVersion() string { - return util.GitCommit() -} - -func (g *CoreDhcp) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s' to generate config files. (WIP)", g.GetName()) -} - -func (g *CoreDhcp) Generate(config *config.Config, params Params) (FileMap, error) { - return nil, fmt.Errorf("plugin does not implement generation function") -} diff --git a/pkg/generator/dhcpd.go b/pkg/generator/dhcpd.go deleted file mode 100644 index 75a41bb..0000000 --- a/pkg/generator/dhcpd.go +++ /dev/null @@ -1,64 +0,0 @@ -package generator - -import ( - "fmt" - - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type DHCPd struct{} - -func (g *DHCPd) GetName() string { - return "dhcpd" -} - -func (g *DHCPd) GetVersion() string { - return util.GitCommit() -} - -func (g *DHCPd) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *DHCPd) Generate(config *config.Config, params Params) (FileMap, error) { - var ( - smdClient = client.NewSmdClient(params.ClientOpts...) - eths = []configurator.EthernetInterface{} - computeNodes = "" - err error = nil - ) - - // - eths, err = smdClient.FetchEthernetInterfaces(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %w", err) - } - - // check if we have the required params first - if eths == nil { - return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") - } - if len(eths) <= 0 { - return nil, fmt.Errorf("no ethernet interfaces found") - } - - // format output to write to config file - computeNodes = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" - for _, eth := range eths { - if len(eth.IpAddresses) == 0 { - continue - } - computeNodes += fmt.Sprintf("host %s { hardware ethernet %s; fixed-address %s} ", eth.ComponentId, eth.MacAddress, eth.IpAddresses[0]) - } - computeNodes += "# =====================================================================" - return ApplyTemplates(Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - "compute_nodes": computeNodes, - "node_entries": "", - }, params.Templates) -} diff --git a/pkg/generator/dnsmasq.go b/pkg/generator/dnsmasq.go deleted file mode 100644 index 16ea6c9..0000000 --- a/pkg/generator/dnsmasq.go +++ /dev/null @@ -1,71 +0,0 @@ -package generator - -import ( - "fmt" - - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type DNSMasq struct{} - -func (g *DNSMasq) GetName() string { - return "dnsmasq" -} - -func (g *DNSMasq) GetVersion() string { - return util.GitCommit() -} - -func (g *DNSMasq) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *DNSMasq) Generate(config *config.Config, params Params) (FileMap, error) { - // make sure we have a valid config first - if config == nil { - return nil, fmt.Errorf("invalid config (config is nil)") - } - - // set all the defaults for variables - var ( - smdClient = client.NewSmdClient(params.ClientOpts...) - eths = []configurator.EthernetInterface{} - err error = nil - ) - - // if we have a client, try making the request for the ethernet interfaces - eths, err = smdClient.FetchEthernetInterfaces(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) - } - - // check if we have the required params first - if eths == nil { - return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") - } - if len(eths) <= 0 { - return nil, fmt.Errorf("no ethernet interfaces found") - } - - // format output to write to config file - output := "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" - for _, eth := range eths { - if eth.Type == "NodeBMC" { - output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n" - } else { - output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n" - } - } - output += "# =====================================================================" - - // apply template substitutions and return output as byte array - return ApplyTemplates(Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - "dhcp_hosts": output, - }, params.Templates) -} diff --git a/pkg/generator/example.go b/pkg/generator/example.go deleted file mode 100644 index 12091ba..0000000 --- a/pkg/generator/example.go +++ /dev/null @@ -1,31 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Example struct { - Message string -} - -func (g *Example) GetName() string { - return "example" -} - -func (g *Example) GetVersion() string { - return util.GitCommit() -} - -func (g *Example) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Example) Generate(config *config.Config, params Params) (FileMap, error) { - g.Message = ` - This is an example generator plugin. See the file in 'internal/generator/plugins/example/example.go' on - information about constructing plugins and plugin requirements.` - return FileMap{"example": []byte(g.Message)}, nil -} diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go deleted file mode 100644 index 3667130..0000000 --- a/pkg/generator/generator.go +++ /dev/null @@ -1,264 +0,0 @@ -package generator - -import ( - "fmt" - "io/fs" - "os" - "path/filepath" - "plugin" - - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" - "github.com/rs/zerolog/log" -) - -type ( - Mappings map[string]any - FileMap map[string][]byte - FileList [][]byte - - // Generator interface used to define how files are created. Plugins can - // be created entirely independent of the main driver program. - Generator interface { - GetName() string - GetVersion() string - GetDescription() string - Generate(config *config.Config, params Params) (FileMap, error) - } -) - -var DefaultGenerators = createDefaultGenerators() - -func createDefaultGenerators() map[string]Generator { - var ( - generatorMap = map[string]Generator{} - generators = []Generator{ - &Conman{}, &DHCPd{}, &DNSMasq{}, &Warewulf{}, &Example{}, &CoreDhcp{}, - } - ) - for _, g := range generators { - generatorMap[g.GetName()] = g - } - return generatorMap -} - -// Converts the file outputs from map[string][]byte to map[string]string. -func ConvertContentsToString(f FileMap) map[string]string { - n := make(map[string]string, len(f)) - for k, v := range f { - n[k] = string(v) - } - return n -} - -// Loads files without applying any Jinja 2 templating. -func LoadFiles(paths ...string) (FileMap, error) { - var outputs = FileMap{} - for _, path := range paths { - expandedPaths, err := filepath.Glob(path) - if err != nil { - return nil, fmt.Errorf("failed to glob path: %w", err) - } - for _, expandedPath := range expandedPaths { - info, err := os.Stat(expandedPath) - if err != nil { - fmt.Println(err) - return nil, fmt.Errorf("failed to stat file or directory: %w", err) - } - // skip any directories found - if info.IsDir() { - continue - } - b, err := os.ReadFile(expandedPath) - if err != nil { - return nil, fmt.Errorf("failed to read file: %w", err) - } - - outputs[expandedPath] = b - } - } - - return outputs, nil -} - -// Loads a single generator plugin given a single file path. -func LoadPlugin(path string) (Generator, error) { - // skip loading plugin if path is a directory with no error - if isDir, err := util.IsDirectory(path); err == nil && isDir { - return nil, nil - } else if err != nil { - return nil, fmt.Errorf("failed to test if plugin path is directory: %w", err) - } - - // try and open the plugin - p, err := plugin.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open plugin: %w", err) - } - - // load the "Generator" symbol from plugin - symbol, err := p.Lookup("Generator") - if err != nil { - return nil, fmt.Errorf("failed to look up symbol at path '%s': %w", path, err) - } - - // assert that the plugin loaded has a valid generator - gen, ok := symbol.(Generator) - if !ok { - return nil, fmt.Errorf("failed to load the correct symbol type at path '%s'", path) - } - return gen, nil -} - -// Loads all generator plugins in a given directory. -// -// Returns a map of generators. Each generator can be accessed by the name -// returned by the generator.GetName() implemented. -func LoadPlugins(dirpath string, opts ...Option) (map[string]Generator, error) { - // check if verbose option is supplied - var ( - generators = make(map[string]Generator) - params = ToParams(opts...) - ) - - // - err := filepath.Walk(dirpath, func(path string, info fs.FileInfo, err error) error { - // skip trying to load generator plugin if directory or error - if info.IsDir() || err != nil { - return nil - } - - // only try loading if file has .so extension - if filepath.Ext(path) != ".so" { - return nil - } - - // load the generator plugin from current path - gen, err := LoadPlugin(path) - if err != nil { - return fmt.Errorf("failed to load generator in directory '%s': %w", path, err) - } - - // show the plugins found if verbose flag is set - if params.Verbose { - log.Info().Str("plugin_name", gen.GetName()).Msg("found plugin") - } - - // map each generator plugin by name for lookup - generators[gen.GetName()] = gen - return nil - }) - - if err != nil { - return nil, fmt.Errorf("failed to walk directory: %w", err) - } - - return generators, nil -} - -// Generate() is the main function to generate a collection of files and returns them as a map. -// This function only expects a path to a plugin and paths to a collection of templates to -// be used. This function will only load the plugin on-demand and fetch resources as needed. -// -// This function requires that a target and plugin path be set at minimum. -func Generate(config *config.Config, plugin string, params Params) (FileMap, error) { - var ( - gen Generator - ok bool - err error - ) - - // check if generator is built-in first before loading external plugin - log.Debug().Any("generators", DefaultGenerators).Msg("available generators") - gen, ok = DefaultGenerators[plugin] - if !ok { - // only load the plugin needed for this target if we don't find default - log.Error().Str("plugin", plugin).Msg("could not find target in default generators") - gen, err = LoadPlugin(plugin) - if err != nil { - return nil, fmt.Errorf("failed to load plugin from file: %v", err) - } - } - - return gen.Generate(config, params) -} - -// Main function to generate a collection of files as a map with the path as the key and -// the contents of the file as the value. This function currently expects a list of plugin -// paths to load all plugins within a directory. Then, each plugin's generator.GenerateWithTarget() -// function is called for each target specified. -// -// This function is the corresponding implementation for the "generate" CLI subcommand. -// It is also call when running the configurator as a service with the "/generate" route. -// -// TODO: Separate loading plugins so we can load them once when running as a service. -func GenerateWithTarget(config *config.Config, target string) (FileMap, error) { - // load generator plugins to generate configs or to print - var ( - opts []client.Option - targetInfo configurator.Target - generator Generator - params Params - err error - ok bool - ) - - // check if a target is supplied - if target == "" { - return nil, fmt.Errorf("must specify a target") - } - - // load target information from config - targetInfo, ok = config.Targets[target] - if !ok { - log.Warn().Str("target", target).Msg("target not found in config") - } - - // if no plugin supplied in config target, then using the target supplied - if targetInfo.Plugin == "" { - targetInfo.Plugin = target - } - - // check if generator is built-in first before loading - generator, ok = DefaultGenerators[target] - if !ok { - // only load the plugin needed for this target if we don't find default - log.Warn().Str("target", target).Msg("could not find target in default generators") - generator, err = LoadPlugin(targetInfo.Plugin) - if err != nil { - return nil, fmt.Errorf("failed to load plugin: %v", err) - } - } - - // check if there's at least one template available - if len(targetInfo.TemplatePaths) <= 0 { - return nil, fmt.Errorf("expects at least one template to be available") - } - - // prepare params to pass into generator - params.Templates = map[string]Template{} - for _, templatePath := range targetInfo.TemplatePaths { - template := Template{} - template.LoadFromFile(templatePath) - params.Templates[templatePath] = template - } - - // set the client options - if config.AccessToken != "" { - params.ClientOpts = append(opts, client.WithAccessToken(config.AccessToken)) - } - if config.CertPath != "" { - params.ClientOpts = append(opts, client.WithCertPoolFile(config.CertPath)) - } - - // load files that are not to be copied - params.Files, err = LoadFiles(targetInfo.FilePaths...) - if err != nil { - return nil, fmt.Errorf("failed to load files to copy: %v", err) - } - - // run the generator plugin from target passed - return generator.Generate(config, params) -} diff --git a/pkg/generator/hostfile.go b/pkg/generator/hostfile.go deleted file mode 100644 index a9f92e5..0000000 --- a/pkg/generator/hostfile.go +++ /dev/null @@ -1,26 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Hostfile struct{} - -func (g *Hostfile) GetName() string { - return "hostfile" -} - -func (g *Hostfile) GetVersion() string { - return util.GitCommit() -} - -func (g *Hostfile) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Hostfile) Generate(config *config.Config, opts ...Option) (FileMap, error) { - return nil, fmt.Errorf("plugin does not implement generation function") -} diff --git a/pkg/generator/params.go b/pkg/generator/params.go deleted file mode 100644 index e9037ec..0000000 --- a/pkg/generator/params.go +++ /dev/null @@ -1,43 +0,0 @@ -package generator - -import ( - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" -) - -type ( - // Params used by the generator - Params struct { - Templates map[string]Template - Files map[string][]byte - ClientOpts []client.Option - Verbose bool - } - Option func(Params) -) - -func ToParams(opts ...Option) Params { - params := Params{} - for _, opt := range opts { - opt(params) - } - return params -} - -func WithClientOpts(opts ...client.Option) Option { - return func(p Params) { - p.ClientOpts = opts - } -} - -func WithTemplates(templates map[string]Template) Option { - return func(p Params) { - p.Templates = templates - } -} - -// Helper function to get the target in generator.Generate() plugin implementations. -func GetTarget(config *config.Config, key string) configurator.Target { - return config.Targets[key] -} diff --git a/pkg/generator/powerman.go b/pkg/generator/powerman.go deleted file mode 100644 index d8e376f..0000000 --- a/pkg/generator/powerman.go +++ /dev/null @@ -1,26 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Powerman struct{} - -func (g *Powerman) GetName() string { - return "powerman" -} - -func (g *Powerman) GetVersion() string { - return util.GitCommit() -} - -func (g *Powerman) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Powerman) Generate(config *config.Config, opts ...Option) (FileMap, error) { - return nil, fmt.Errorf("plugin does not implement generation function") -} diff --git a/pkg/generator/syslog.go b/pkg/generator/syslog.go deleted file mode 100644 index 450dcbc..0000000 --- a/pkg/generator/syslog.go +++ /dev/null @@ -1,26 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Syslog struct{} - -func (g *Syslog) GetName() string { - return "syslog" -} - -func (g *Syslog) GetVersion() string { - return util.GitCommit() -} - -func (g *Syslog) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Syslog) Generate(config *config.Config, opts ...Option) (FileMap, error) { - return nil, fmt.Errorf("plugin does not implement generation function") -} diff --git a/pkg/generator/templates.go b/pkg/generator/templates.go deleted file mode 100644 index 3028949..0000000 --- a/pkg/generator/templates.go +++ /dev/null @@ -1,98 +0,0 @@ -package generator - -import ( - "bytes" - "fmt" - "os" - - "git.towk2.me/towk/configurator/pkg/util" - "github.com/nikolalohinski/gonja/v2" - "github.com/nikolalohinski/gonja/v2/exec" - "github.com/rs/zerolog/log" -) - -type Template struct { - Contents []byte `json:"contents"` -} - -func (t *Template) LoadFromFile(path string) error { - // skip loading template if path is a directory with no error - if isDir, err := util.IsDirectory(path); err == nil && isDir { - return nil - } else if err != nil { - return fmt.Errorf("failed to test if template path is directory: %w", err) - } - - // try and read the contents of the file - // NOTE: we don't care if this is actually a Jinja template - // or not...at least for now. - contents, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read file: %v", err) - } - t.Contents = contents - return nil -} - -func (t *Template) IsEmpty() bool { - return len(t.Contents) <= 0 -} - -// Wrapper function to slightly abstract away some of the nuances with using gonja -// into a single function call. This function is *mostly* for convenience and -// simplication. If no paths are supplied, then no templates will be applied and -// there will be no output. -// -// The "FileList" returns a slice of byte arrays in the same order as the argument -// list supplied, but with the Jinja templating applied. -func ApplyTemplates(mappings Mappings, templates map[string]Template) (FileMap, error) { - var ( - data = exec.NewContext(mappings) - outputs = FileMap{} - ) - - for path, template := range templates { - // load jinja template from file - t, err := gonja.FromBytes(template.Contents) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %w", err) - } - - // execute/render jinja template - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %w", err) - } - outputs[path] = b.Bytes() - } - - log.Debug().Any("templates", templates).Any("outputs", outputs).Any("mappings", mappings).Msg("apply templates") - - return outputs, nil -} - -// Wrapper function similiar to "ApplyTemplates" but takes file paths as arguments. -// This function will load templates from a file instead of using file contents. -func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) { - var ( - data = exec.NewContext(mappings) - outputs = FileMap{} - ) - - for _, path := range paths { - // load jinja template from file - t, err := gonja.FromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %w", err) - } - - // execute/render jinja template - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %w", err) - } - outputs[path] = b.Bytes() - } - - return outputs, nil -} diff --git a/pkg/generator/warewulf.go b/pkg/generator/warewulf.go deleted file mode 100644 index 5a7c803..0000000 --- a/pkg/generator/warewulf.go +++ /dev/null @@ -1,78 +0,0 @@ -package generator - -import ( - "fmt" - "maps" - "strings" - - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" - "github.com/rs/zerolog/log" -) - -type Warewulf struct{} - -func (g *Warewulf) GetName() string { - return "warewulf" -} - -func (g *Warewulf) GetVersion() string { - return util.GitCommit() -} - -func (g *Warewulf) GetDescription() string { - return "Configurator generator plugin for 'warewulf' config files." -} - -func (g *Warewulf) Generate(config *config.Config, params Params) (FileMap, error) { - var ( - smdClient = client.NewSmdClient(params.ClientOpts...) - outputs = make(FileMap, len(params.Templates)) - nodeEntries = "" - paths = []string{} - ) - - // if we have a client, try making the request for the ethernet interfaces - eths, err := smdClient.FetchEthernetInterfaces(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) - } - - // check if we have the required params first - if eths == nil { - return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") - } - if len(eths) <= 0 { - return nil, fmt.Errorf("no ethernet interfaces found") - } - - // fetch redfish endpoints and handle errors - eps, err := smdClient.FetchRedfishEndpoints(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch redfish endpoints: %v", err) - } - if len(eps) <= 0 { - return nil, fmt.Errorf("no redfish endpoints found") - } - - templates, err := ApplyTemplates(Mappings{ - "node_entries": nodeEntries, - }, params.Templates) - if err != nil { - return nil, fmt.Errorf("failed to load templates: %v", err) - } - - maps.Copy(outputs, params.Files) - maps.Copy(outputs, templates) - - // print message if verbose param is found - if params.Verbose { - for path, _ := range outputs { - paths = append(paths, path) - } - } - log.Info().Str("paths", strings.Join(paths, ":")).Msg("templates and files loaded: \n") - - return outputs, err -} diff --git a/pkg/service/profile.go b/pkg/service/profile.go index e0fb473..6bcdebe 100644 --- a/pkg/service/profile.go +++ b/pkg/service/profile.go @@ -10,6 +10,7 @@ import ( "path/filepath" "github.com/go-chi/chi/v5" + "github.com/tidwall/sjson" ) type Profile struct { @@ -43,7 +44,7 @@ func (s *Service) GetProfiles() http.HandlerFunc { // read file contents var profile *Profile - profile, err = LoadProfile(path) + profile, err = LoadProfileFromFile(path) if err != nil { return err } @@ -91,7 +92,7 @@ func (s *Service) GetProfile() http.HandlerFunc { err error ) - profile, err = LoadProfile(path) + profile, err = LoadProfileFromFile(path) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -114,8 +115,8 @@ func (s *Service) GetProfile() http.HandlerFunc { func (s *Service) CreateProfile() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { type input struct { - path string `json:"path"` - profile *Profile `json:"profile"` + Path string `json:"path"` + Profile *Profile `json:"profile"` } var ( body, contents []byte @@ -137,37 +138,85 @@ func (s *Service) CreateProfile() http.HandlerFunc { } // serialize just the profile part - contents, err = json.Marshal(in.profile) + contents, err = json.Marshal(in.Profile) if err != nil { - + http.Error(w, err.Error(), http.StatusBadRequest) + return } // create a new profile on disk - err = os.WriteFile(in.path, contents, os.ModePerm) + err = os.WriteFile(in.Path, contents, os.ModePerm) if err != nil { - + http.Error(w, err.Error(), http.StatusBadRequest) + return } w.WriteHeader(http.StatusOK) } } -func (s *Service) CreateProfileVar() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} +func (s *Service) SetProfileData() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + body, contents []byte + newContents string + profile *Profile + path string + err error + ) + + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + err = json.Unmarshal(body, &profile) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // make sure the request data sets an ID + if profile.ID == "" { + http.Error(w, "ID must be set to a non-empty value", http.StatusBadRequest) + return + } + + // read the contents the file with profile ID + path = s.BuildProfilePath(profile.ID) + contents, err = os.ReadFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // modify the data of the profile's contents + newContents, err = sjson.Set(string(contents), "data", profile.Data) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // write only the data to the file with ID + err = os.WriteFile(path, []byte(newContents), os.ModePerm) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } } -func (s *Service) DeleteProfileVar() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} +func (s *Service) DeleteProfileData() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + } } func (s *Service) GetProfileData() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) {} } -func (s *Service) GetProfileVar() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - func (s *Service) CreateProfilePath() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) {} } diff --git a/pkg/service/service.go b/pkg/service/service.go index 6c3467d..9a1545a 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -62,10 +62,9 @@ func (s *Service) Serve() error { // router.Post("/profiles", s.CreateProfiles()) router.Get("/profile/{id}", s.GetProfile()) router.Post("/profile/{id}", s.CreateProfile()) - router.Post("/profile/{id}/data/{varname}", s.CreateProfileVar()) - router.Delete("/profile/{id}/data/{varname}", s.DeleteProfileVar()) router.Get("/profile/{id}/data", s.GetProfileData()) - router.Get("/profile/{id}/data/{varname}", s.GetProfileVar()) + router.Post("/profile/{id}/data", s.SetProfileData()) + router.Delete("/profile/{id}/data", s.DeleteProfileData()) router.Post("/profile/{id}/paths/{path}", s.CreateProfilePath()) router.Delete("/profile/{id}/paths/{path}", s.DeleteProfilePath()) router.Get("/profile/{id}/paths/{path}", s.GetProfilePath()) @@ -89,11 +88,11 @@ func (s *Service) FetchJwks(uri string) { } -func LoadProfile(path string) (*Profile, error) { +func LoadProfileFromFile(path string) (*Profile, error) { return LoadFromJSONFile[Profile](path) } -func LoadPlugin(path string) (*configurator.Plugin, error) { +func LoadPluginFromFile(path string) (*configurator.Plugin, error) { return LoadFromJSONFile[configurator.Plugin](path) } From 72be62c78e742f62de0820488f96c5a31e41cece Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 13 Aug 2025 22:52:52 -0600 Subject: [PATCH 05/89] refactor: more implementation to refactor --- pkg/service/profile.go | 131 ++++++++++++++++++++++++++++++++++------- pkg/service/service.go | 129 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 229 insertions(+), 31 deletions(-) diff --git a/pkg/service/profile.go b/pkg/service/profile.go index 6bcdebe..2fa7c2d 100644 --- a/pkg/service/profile.go +++ b/pkg/service/profile.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" + configurator "git.towk2.me/towk/configurator/pkg" "github.com/go-chi/chi/v5" "github.com/tidwall/sjson" ) @@ -86,22 +87,14 @@ func (s *Service) GetProfile() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( id = chi.URLParam(r, "id") - path = s.BuildProfilePath(id) - profile *Profile + path = s.PathForProfileWithID(id) contents []byte err error ) - profile, err = LoadProfileFromFile(path) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - contents, err = json.Marshal(profile) + contents, err = loadProfileContents(path) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) - return } _, err = w.Write(contents) @@ -184,7 +177,7 @@ func (s *Service) SetProfileData() http.HandlerFunc { } // read the contents the file with profile ID - path = s.BuildProfilePath(profile.ID) + path = s.PathForProfileWithID(profile.ID) contents, err = os.ReadFile(path) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -209,28 +202,104 @@ func (s *Service) SetProfileData() http.HandlerFunc { func (s *Service) DeleteProfileData() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var ( + id = chi.URLParam(r, "id") + path = s.PathForProfileWithID(id) + profile *Profile + err error + ) + + // get the profile + profile, err = LoadProfileFromFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + + // delete the profile data + profile.Data = map[string]any{} + + // save the profile back to the file to update + SaveProfileToFile(path, profile) } } func (s *Service) GetProfileData() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} + return func(w http.ResponseWriter, r *http.Request) { + var ( + id = chi.URLParam(r, "id") + path = s.PathForProfileWithID(id) + profile *Profile + body []byte + err error + ) + + profile, err = LoadProfileFromFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // only marshal the profile data and not entire profile + body, err = json.Marshal(profile.Data) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // write body to response + _, err = w.Write(body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } } -func (s *Service) CreateProfilePath() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} +// func (s *Service) CreateProfilePath() http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) { -func (s *Service) DeleteProfilePath() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} +// w.WriteHeader(http.StatusOK) +// } +// } -func (s *Service) GetProfilePath() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} +// func (s *Service) DeleteProfilePath() http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) {} +// } + +// func (s *Service) GetProfilePath() http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) {} +// } func (s *Service) GetPlugins() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} + + return func(w http.ResponseWriter, r *http.Request) { + var ( + plugins map[string]configurator.Plugin + names []string + body []byte + err error + ) + + plugins, err = LoadPluginsFromDir(s.PathForPlugins()) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + for name := range plugins { + names = append(names, name) + } + + body, err = json.Marshal(names) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(body) + } } func (s *Service) CreatePlugins() http.HandlerFunc { @@ -240,3 +309,21 @@ func (s *Service) CreatePlugins() http.HandlerFunc { func (s *Service) DeletePlugins() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) {} } + +func loadProfileContents(path string) ([]byte, error) { + var ( + contents []byte + profile *Profile + err error + ) + profile, err = LoadProfileFromFile(path) + if err != nil { + return nil, fmt.Errorf("failed to load profile from file: %v", err) + } + + contents, err = json.Marshal(profile) + if err != nil { + return nil, fmt.Errorf("failed to marshal profile: %v", err) + } + return contents, nil +} diff --git a/pkg/service/service.go b/pkg/service/service.go index 9a1545a..2247a58 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -3,10 +3,15 @@ package service import ( "encoding/json" "fmt" + "io/fs" "os" + "path/filepath" + "plugin" + "slices" "time" configurator "git.towk2.me/towk/configurator/pkg" + "git.towk2.me/towk/configurator/pkg/util" "github.com/go-chi/chi/middleware" "github.com/go-chi/chi/v5" ) @@ -65,9 +70,9 @@ func (s *Service) Serve() error { router.Get("/profile/{id}/data", s.GetProfileData()) router.Post("/profile/{id}/data", s.SetProfileData()) router.Delete("/profile/{id}/data", s.DeleteProfileData()) - router.Post("/profile/{id}/paths/{path}", s.CreateProfilePath()) - router.Delete("/profile/{id}/paths/{path}", s.DeleteProfilePath()) - router.Get("/profile/{id}/paths/{path}", s.GetProfilePath()) + // router.Post("/profile/{id}/paths/{path}", s.CreateProfilePath()) + // router.Delete("/profile/{id}/paths/{path}", s.DeleteProfilePath()) + // router.Get("/profile/{id}/paths/{path}", s.GetProfilePath()) // plugins router.Get("/plugins", s.GetPlugins()) @@ -89,14 +94,99 @@ func (s *Service) FetchJwks(uri string) { } func LoadProfileFromFile(path string) (*Profile, error) { - return LoadFromJSONFile[Profile](path) + return loadFromJSONFile[Profile](path) } -func LoadPluginFromFile(path string) (*configurator.Plugin, error) { - return LoadFromJSONFile[configurator.Plugin](path) +// LoadPluginFromFile loads a single plugin given a single file path +func LoadPluginFromFile(path string) (configurator.Plugin, error) { + var ( + isDir bool + err error + loadedPlugin *plugin.Plugin + ) + // skip loading plugin if path is a directory with no error + if isDir, err = util.IsDirectory(path); err == nil && isDir { + return nil, nil + } else if err != nil { + return nil, fmt.Errorf("failed to test if plugin path is directory: %v", err) + } + + // try and open the plugin + loadedPlugin, err = plugin.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open plugin at path '%s': %v", path, err) + } + + // load the "Target" symbol from plugin + symbol, err := loadedPlugin.Lookup("Target") + if err != nil { + return nil, fmt.Errorf("failed to look up symbol at path '%s': %v", path, err) + } + + // assert that the plugin is a valid configurator.Plugin + target, ok := symbol.(configurator.Plugin) + if !ok { + return nil, fmt.Errorf("failed to load the correct symbol type at path '%s'", path) + } + return target, nil } -func LoadFromJSONFile[T any](path string) (*T, error) { +// LoadPluginsFromDir loads all plugins in a given directory. +// +// Returns a map of plugins. Each plugin can be accessed by the name +// returned by the plugin.GetName() implemented. +func LoadPluginsFromDir(dirpath string) (map[string]configurator.Plugin, error) { + // check if verbose option is supplied + var ( + cps = make(map[string]configurator.Plugin) + err error + ) + + // helper to check for valid extensions + var hasValidExt = func(path string) bool { + var validExts = []string{".so", ".dylib", ".dll"} + return slices.Contains(validExts, filepath.Ext(path)) + } + + // walk all files in directory only loading *valid* plugins + err = filepath.Walk(dirpath, func(path string, info fs.FileInfo, err error) error { + // skip trying to load generator plugin if directory or error + if info.IsDir() || err != nil { + return nil + } + + // only try loading if file has .so extension + if hasValidExt(path) { + return nil + } + + // load the plugin from current path + p, err := LoadPluginFromFile(path) + if err != nil { + return fmt.Errorf("failed to load plugin in directory '%s': %v", path, err) + } + + // map each plugin by name for lookup + cps[p.Name()] = p + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to walk directory '%s': %v", dirpath, err) + } + + return cps, nil +} + +func SaveProfileToFile(path string, profile *Profile) error { + return saveToJSONFile(path, profile) +} + +func SavePluginToFile(path string, plugin *configurator.Plugin) error { + return saveToJSONFile(path, plugin) +} + +func loadFromJSONFile[T any](path string) (*T, error) { var ( res *T contents []byte @@ -116,6 +206,27 @@ func LoadFromJSONFile[T any](path string) (*T, error) { return res, err } -func (s *Service) BuildProfilePath(id string) string { - return s.RootPath + PLUGINS_RELPATH + "/" + id +func saveToJSONFile[T any](path string, data T) error { + var ( + contents []byte + err error + ) + contents, err = json.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data to JSON: %v", err) + } + err = os.WriteFile(path, contents, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to write JSON to file: %v", err) + } + + return nil +} + +func (s *Service) PathForProfileWithID(id string) string { + return s.RootPath + PROFILES_RELPATH + "/" + id +} + +func (s *Service) PathForPlugins() string { + return s.RootPath + PLUGINS_RELPATH } From a1a9c6407f3af0bbe7f089a8f249d785ade06691 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Aug 2025 07:40:30 -0600 Subject: [PATCH 06/89] refactor: added more implementation details --- pkg/client/client.go | 27 +++++++++++++++++++++++++++ pkg/plugin.go | 6 ++++-- pkg/plugins/jinja2.go | 19 +++++++++++++++++++ pkg/plugins/smd.go | 1 + pkg/service/routes.go | 32 ++++++++++++++++++++++++++++++++ pkg/service/service.go | 5 +++-- 6 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 pkg/client/client.go create mode 100644 pkg/plugins/jinja2.go create mode 100644 pkg/plugins/smd.go diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 0000000..6832193 --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,27 @@ +package client + +import "net/http" + +type HTTPBody []byte +type HTTPHeader map[string]string +type HTTPEnvelope struct { + Path string + Method string + Header HTTPHeader + Body HTTPBody + CACert string +} + +type Client struct { + BaseURI string +} + +func New(uri string) Client { + return Client{ + BaseURI: uri, + } +} + +func (c *Client) MakeRequest(env HTTPEnvelope) (*http.Response, []byte, error) { + http.DefaultTransport.(*http.Transport) +} diff --git a/pkg/plugin.go b/pkg/plugin.go index 602cfce..4ad4544 100644 --- a/pkg/plugin.go +++ b/pkg/plugin.go @@ -1,14 +1,16 @@ package configurator type Plugin interface { - // plugin data Name() string Version() string Description() string Metadata() map[string]string - // run the plugin Init() error Run() error Cleanup() error } + +func RunPlugin() { + +} diff --git a/pkg/plugins/jinja2.go b/pkg/plugins/jinja2.go new file mode 100644 index 0000000..9cb515d --- /dev/null +++ b/pkg/plugins/jinja2.go @@ -0,0 +1,19 @@ +package plugin + +type Jinja2 struct{} + +func Name() string { return "jinja2" } +func Version() string { return "test" } +func Description() string { return "Renders Jinja 2 templates" } +func Metadata() map[string]string { + return map[string]string{ + "author.name": "David J. Allen", + "author.email": "davidallendj@gmail.com", + } +} + +func Init() { + // initialize Jinja2 (gonja) +} +func Run() {} +func Cleanup() {} diff --git a/pkg/plugins/smd.go b/pkg/plugins/smd.go new file mode 100644 index 0000000..b0736c3 --- /dev/null +++ b/pkg/plugins/smd.go @@ -0,0 +1 @@ +package plugin diff --git a/pkg/service/routes.go b/pkg/service/routes.go index f0e55ab..21781df 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -4,10 +4,42 @@ import ( "encoding/json" "fmt" "net/http" + "os" + "path/filepath" + "strings" + + "git.towk2.me/towk/configurator/pkg/util" ) func (s *Service) Download() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var ( + path = strings.TrimPrefix(r.URL.Path, "/download") + fileInfo os.FileInfo + out *os.File + err error + ) + + fmt.Printf("download path: %v\n", path) + + // determine if path is directory, file, or exists + if fileInfo, err = os.Stat(filepath.Clean(path)); err != nil { + if fileInfo.IsDir() { + // recursively walk dir acompressednd get all filenames + // download directory as archive + out, err = os.Create(fmt.Sprintf("%s.tar", path)) + if err != nil { + + } + err = util.CreateArchive([]string{path}, out) + if err != nil { + + } + + } else { + // download individual file + } + } } } diff --git a/pkg/service/service.go b/pkg/service/service.go index 9a1545a..50483af 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -3,6 +3,7 @@ package service import ( "encoding/json" "fmt" + "net/http" "os" "time" @@ -53,7 +54,7 @@ func (s *Service) Serve() error { } else { // general - router.Get("/download", s.Download()) + router.Get("/download/*", s.Download()) router.Post("/upload", s.Upload()) router.Get("/list", s.List()) @@ -77,7 +78,7 @@ func (s *Service) Serve() error { // always available public routes go here router.HandleFunc("/status", s.GetStatus) - return nil + return http.ListenAndServe(":8080", router) } func (s *Service) requireAuth() bool { From 14133128938bd6fb21b490004c96888923033244 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Aug 2025 07:41:09 -0600 Subject: [PATCH 07/89] refactor: updated cmd command funcs --- cmd/download.go | 14 +++++++++++++- cmd/list.go | 10 ++++++++++ cmd/profiles.go | 10 ++++++++++ cmd/render.go | 0 cmd/run.go | 10 ++++++++++ cmd/serve.go | 20 ++++++++++++++++++++ 6 files changed, 63 insertions(+), 1 deletion(-) delete mode 100644 cmd/render.go create mode 100644 cmd/run.go create mode 100644 cmd/serve.go diff --git a/cmd/download.go b/cmd/download.go index 0160d4f..9aca225 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -1,7 +1,19 @@ package cmd -import "github.com/spf13/cobra" +import ( + "git.towk2.me/towk/configurator/pkg/util" + "github.com/spf13/cobra" +) var downloadCmd = cobra.Command{ Use: "download", + Run: func(cmd *cobra.Command, args []string) { + + util.MakeRequest() + + }, +} + +func init() { + rootCmd.AddCommand(&downloadCmd) } diff --git a/cmd/list.go b/cmd/list.go index e69de29..558281d 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -0,0 +1,10 @@ +package cmd + +import "github.com/spf13/cobra" + +var listCmd = &cobra.Command{ + Use: "list", + Run: func(cmd *cobra.Command, args []string) { + + }, +} diff --git a/cmd/profiles.go b/cmd/profiles.go index e69de29..7059cfe 100644 --- a/cmd/profiles.go +++ b/cmd/profiles.go @@ -0,0 +1,10 @@ +package cmd + +import "github.com/spf13/cobra" + +var profilesCmd = &cobra.Command{ + Use: "profiles", + Run: func(cmd *cobra.Command, args []string) { + + }, +} diff --git a/cmd/render.go b/cmd/render.go deleted file mode 100644 index e69de29..0000000 diff --git a/cmd/run.go b/cmd/run.go new file mode 100644 index 0000000..3f7aeff --- /dev/null +++ b/cmd/run.go @@ -0,0 +1,10 @@ +package cmd + +import "github.com/spf13/cobra" + +var runCmd = &cobra.Command{ + Use: "run", + Run: func(cmd *cobra.Command, args []string) { + + }, +} diff --git a/cmd/serve.go b/cmd/serve.go new file mode 100644 index 0000000..0135819 --- /dev/null +++ b/cmd/serve.go @@ -0,0 +1,20 @@ +package cmd + +import ( + "git.towk2.me/towk/configurator/pkg/service" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var serveCmd = &cobra.Command{ + Use: "serve", + Run: func(cmd *cobra.Command, args []string) { + server := service.New() + err := server.Serve() + log.Error().Err(err).Msg("server closed") + }, +} + +func init() { + rootCmd.AddCommand(serveCmd) +} From a8c16ed71521feaf5dfda6419cc12149ddd68eb8 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Aug 2025 07:41:47 -0600 Subject: [PATCH 08/89] chore: added directories and update go deps --- bin/.gitkeep | 0 go.mod | 3 +++ go.sum | 14 ++++++++++++++ lib/.gitkeep | 0 4 files changed, 17 insertions(+) create mode 100644 bin/.gitkeep create mode 100644 lib/.gitkeep diff --git a/bin/.gitkeep b/bin/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/go.mod b/go.mod index 97ef719..bcd1ec8 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/go-chi/chi v1.5.5 github.com/go-chi/chi/v5 v5.1.0 github.com/lestrrat-go/jwx/v2 v2.1.1 + github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.8.0 github.com/tidwall/sjson v1.2.5 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 @@ -21,6 +22,8 @@ require ( github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/gjson v1.14.2 // indirect diff --git a/go.sum b/go.sum index b8090b3..f4508cd 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -12,6 +13,7 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= @@ -26,8 +28,17 @@ github.com/lestrrat-go/jwx/v2 v2.1.1 h1:Y2ltVl8J6izLYFs54BVcpXLv5msSW4o8eXwnzZLI github.com/lestrrat-go/jwx/v2 v2.1.1/go.mod h1:4LvZg7oxu6Q5VJwn7Mk/UwooNRnTHUpXBj2C4j3HNx0= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= @@ -52,6 +63,9 @@ golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/lib/.gitkeep b/lib/.gitkeep new file mode 100644 index 0000000..e69de29 From 1f5775196d6fb25069d28a9fd70cc6d13d40bfe1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 18 Aug 2025 10:59:34 -0600 Subject: [PATCH 09/89] chore: updated license file --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index aa18635..85c3d9c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright © 2024 Triad National Security, LLC. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. +Copyright © 2025 David J. Allen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +OTHER DEALINGS IN THE SOFTWARE. From 419e9781bf7a114d3dcaf806179d9d25cfd09142 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 18 Aug 2025 11:01:19 -0600 Subject: [PATCH 10/89] feat: added root, download, list, and serve cmd implementations --- cmd/download.go | 74 +++++++++++++++++++++++++++++++++++++++++++++++-- cmd/list.go | 59 ++++++++++++++++++++++++++++++++++++++- cmd/root.go | 59 ++++++++++++++++++++++++++++++++------- cmd/serve.go | 53 +++++++++++++++++++++++++++++++++-- 4 files changed, 229 insertions(+), 16 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index 9aca225..1fb211b 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -1,19 +1,89 @@ package cmd import ( - "git.towk2.me/towk/configurator/pkg/util" + "fmt" + "net/http" + "os" + "path/filepath" + "time" + + "git.towk2.me/towk/configurator/pkg/client" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) var downloadCmd = cobra.Command{ Use: "download", + Example: ` + # download a file or directory (as archive) + configurator download + configurator download --host https://example.com --path test + configurator download --plugins smd,jinja2 --profile compute + curl $CONFIGURATOR_HOST/download/test?plugins=smd,jinja2 +`, + Short: "Download and modify files with plugins", + PreRun: func(cmd *cobra.Command, args []string) { + setenv(&host, "CONFIGURATOR_HOST") + setenv(&path, "CONFIGURATOR_PATH") + }, Run: func(cmd *cobra.Command, args []string) { + var ( + c = client.New(host) + res *http.Response + body []byte + err error + ) - util.MakeRequest() + log.Debug(). + Str("host", host). + Str("path", path). + Str("output", outputPath). + Send() + + // set output path to match path if empty + if outputPath == "" { + if path != "." || path != "" { + outputPath = filepath.Base(path) + } else { + outputPath = fmt.Sprintf("%d.file", time.Now().Unix()) + } + } + + // make request to /download endpoint + // _, err = c.Download(outputPath, client.HTTPEnvelope{ + // Path: fmt.Sprintf("/download/%s", path), + // Method: http.MethodGet, + // }) + + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: fmt.Sprintf("/download/%s", path), + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err).Str("host", host).Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error().Int("status", res.StatusCode).Str("host", host).Msg("response returned bad status") + os.Exit(1) + } + if outputPath != "" { + err = os.WriteFile(outputPath, body, 0o755) + if err != nil { + log.Error().Err(err).Msg("failed to write file(s) from download") + os.Exit(1) + } + } else { + fmt.Println(string(body)) + } }, } func init() { + downloadCmd.Flags().StringVar(&host, "host", "http://localhost:5050", "Set the configurator remote host (can be set with CONFIGURATOR_HOST)") + downloadCmd.Flags().StringVarP(&path, "path", "p", ".", "Set the path to list files (can be set with CONFIGURATOR_PATH)") + downloadCmd.Flags().StringVarP(&outputPath, "output", "o", "", "Set the output path to write files") + rootCmd.AddCommand(&downloadCmd) } diff --git a/cmd/list.go b/cmd/list.go index 558281d..f350a43 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -1,10 +1,67 @@ package cmd -import "github.com/spf13/cobra" +import ( + "encoding/json" + "fmt" + "net/http" + "os" + + "git.towk2.me/towk/configurator/pkg/client" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) var listCmd = &cobra.Command{ Use: "list", + Example: ` + # list files in a remote data directory + configurator list --path test + configurator list --host https://example.com --path test + curl https://example.com/list/test +`, + Args: cobra.NoArgs, + Short: "List all files in a remote data directory", + PreRun: func(cmd *cobra.Command, args []string) { + setenv(&host, "CONFIGURATOR_HOST") + setenv(&path, "CONFIGURATOR_PATH") + }, Run: func(cmd *cobra.Command, args []string) { + var ( + c = client.New(host) + body []byte + output []string + err error + ) + log.Debug(). + Str("host", host). + Str("path", path). + Send() + + // make request to /list endpoint + _, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: fmt.Sprintf("/list/%s", path), + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err).Str("url", host).Msg("failed to make request") + os.Exit(1) + } + + err = json.Unmarshal(body, &output) + if err != nil { + log.Error().Err(err).Msg("failed to unmarshal response body") + os.Exit(1) + } + + // show the list of files and directories + log.Info().Strs("output", output).Send() }, } + +func init() { + listCmd.Flags().StringVar(&host, "host", "http://localhost:5050", "Set the configurator remote host (can be set with CONFIGURATOR_HOST)") + listCmd.Flags().StringVarP(&path, "path", "p", ".", "Set the path to list files (can be set with CONFIGURATOR_PATH)") + + rootCmd.AddCommand(listCmd) +} diff --git a/cmd/root.go b/cmd/root.go index 2e70c6a..da93c0c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,21 +3,38 @@ package cmd import ( "fmt" "os" + "slices" + "strings" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) +var ( + host string + path string + outputPath string + rootPath string + logLevel string + timeout int +) var rootCmd = cobra.Command{ - Use: "configurator", - Run: func(cmd *cobra.Command, args []string) { - + Use: "configurator", + Short: "Extensible configuration builder to download files", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + // set the logging level + level, err := strToLogLevel(logLevel) + if err != nil { + log.Error().Err(err).Msg("failed to convert log level argument") + os.Exit(1) + } + zerolog.SetGlobalLevel(level) }, } func Execute() { - // run initialization code first - initEnv() - + // run the main program if err := rootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -26,12 +43,34 @@ func Execute() { func init() { // initialize the config a single time + rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "info", "Set the log level output") } -func initConfigFromFile(path string) { - +func strToLogLevel(ll string) (zerolog.Level, error) { + levels := []string{"debug", "info", "warn", "disabled"} + if index := slices.Index(levels, ll); index >= 0 { + // handle special case to map index == 3 to zerolog.Disabled == 7 + switch index { + case 3: + return zerolog.Disabled, nil + } + return zerolog.Level(index), nil + } + return -100, fmt.Errorf( + "invalid log level (options: %s)", strings.Join(levels, ", "), + ) // use 'info' by default } -func initEnv() { - +func setenv(v *string, key string) { + t := os.Getenv(key) + if t != "" { + *v = t + } } + +// func setenv(cmd *cobra.Command, varname string, envvar string) { +// v := os.Getenv(envvar) +// if v != "" { +// cmd.Flags().Set(varname, v) +// } +// } diff --git a/cmd/serve.go b/cmd/serve.go index 0135819..0358149 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -1,20 +1,67 @@ package cmd import ( + "time" + "git.towk2.me/towk/configurator/pkg/service" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) var serveCmd = &cobra.Command{ - Use: "serve", + Use: "serve", + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + // try and set flags using env vars + setenv(&host, "CONFIGURATOR_HOST") + setenv(&rootPath, "CONFIGURATOR_ROOT") + }, Run: func(cmd *cobra.Command, args []string) { - server := service.New() - err := server.Serve() + var ( + host string + rootPath string + server *service.Service + err error + ) + + // get vars but don't modify + host, _ = cmd.Flags().GetString("host") + rootPath, _ = cmd.Flags().GetString("root") + + // set the server values + server = service.New() + server.Addr = host + server.RootPath = rootPath + server.Timeout = time.Duration(timeout) * time.Second + + // show some debugging information + log.Debug(). + Str("host", host). + Any("paths", map[string]string{ + "root": rootPath, + "data": server.PathForData(), + "profiles": server.PathForProfiles(), + "plugins": server.PathForPlugins(), + "metadata": server.PathForMetadata(), + }). + Send() + + // make the default directories and files if flag is passed + if cmd.Flags().Changed("init") { + server.Init() + } + + // serve and log why the server closed + err = server.Serve() log.Error().Err(err).Msg("server closed") }, } func init() { + serveCmd.Flags().Bool("init", false, "Initializes default files at specified with the '--root' flag") + serveCmd.Flags().StringVar(&host, "host", "localhost:5050", "Set the configurator server host (can be set with CONFIGURATOR_HOST)") + serveCmd.Flags().StringVar(&rootPath, "root", "./", "Set the root path to serve files (can be set with CONFIGURATOR_ROOT)") + serveCmd.Flags().IntVarP(&timeout, "timeout", "t", 60, "Set the timeout in seconds for requests.") + rootCmd.AddCommand(serveCmd) } From 7e9e186138de3af2be2570d8a9c8649af3656b7b Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 18 Aug 2025 11:01:32 -0600 Subject: [PATCH 11/89] chore: updated go deps --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index bcd1ec8..68464fd 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( ) require ( + github.com/cavaliergopher/grab/v3 v3.0.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index f4508cd..90214c9 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8= +github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= +github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 920703fc0ed85f1d60e5931c873ad5e760b4af75 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 18 Aug 2025 11:02:33 -0600 Subject: [PATCH 12/89] feat: added skeleton implementation for run and profiles cmd --- cmd/profiles.go | 4 ++++ cmd/run.go | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/profiles.go b/cmd/profiles.go index 7059cfe..7c4294f 100644 --- a/cmd/profiles.go +++ b/cmd/profiles.go @@ -8,3 +8,7 @@ var profilesCmd = &cobra.Command{ }, } + +func init() { + rootCmd.AddCommand(profilesCmd) +} diff --git a/cmd/run.go b/cmd/run.go index 3f7aeff..369700c 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -3,8 +3,13 @@ package cmd import "github.com/spf13/cobra" var runCmd = &cobra.Command{ - Use: "run", + Use: "run", + Short: "Run the configurator locally", Run: func(cmd *cobra.Command, args []string) { }, } + +func init() { + rootCmd.AddCommand(runCmd) +} From fe67674829c888f57c674d20b1caaa104296e62c Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 18 Aug 2025 11:03:19 -0600 Subject: [PATCH 13/89] fix: issue creating archive that was a directory --- pkg/util/util.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/util/util.go b/pkg/util/util.go index 6ff13b0..2b120e0 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -135,6 +135,11 @@ func addToArchive(tw *tar.Writer, filename string) error { return err } + // skip file if it's a directory + if info.IsDir() { + return nil + } + // create a tar Header from the FileInfo data header, err := tar.FileInfoHeader(info, info.Name()) if err != nil { From fa949baafda2a8de24a222f36a017b9b5071dd8a Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 18 Aug 2025 11:03:54 -0600 Subject: [PATCH 14/89] refactor: updated plugin interface --- pkg/plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin.go b/pkg/plugin.go index 4ad4544..d7ff3e4 100644 --- a/pkg/plugin.go +++ b/pkg/plugin.go @@ -7,7 +7,7 @@ type Plugin interface { Metadata() map[string]string Init() error - Run() error + Run(args []string) error Cleanup() error } From 0d27f07a8b4cd7cbbb20f8c135ea3df3547f5be1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 18 Aug 2025 11:04:44 -0600 Subject: [PATCH 15/89] feat: updated client implementation --- pkg/client/client.go | 97 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 4 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 6832193..d338fd7 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -1,6 +1,16 @@ package client -import "net/http" +import ( + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "strings" + + "git.towk2.me/towk/configurator/pkg/util" + "github.com/cavaliergopher/grab/v3" +) type HTTPBody []byte type HTTPHeader map[string]string @@ -13,15 +23,94 @@ type HTTPEnvelope struct { } type Client struct { - BaseURI string + http.Client + BaseURI string + AccessToken string } func New(uri string) Client { return Client{ - BaseURI: uri, + BaseURI: strings.TrimSuffix(uri, "/"), + } +} + +func NewHTTPEnvelope() HTTPEnvelope { + return HTTPEnvelope{ + Path: "", + Method: http.MethodGet, + Header: nil, + Body: nil, + CACert: "", } } func (c *Client) MakeRequest(env HTTPEnvelope) (*http.Response, []byte, error) { - http.DefaultTransport.(*http.Transport) + return util.MakeRequest(c.BaseURI+env.Path, env.Method, env.Body, env.Header) +} + +func (c *Client) Download(out string, env HTTPEnvelope) (*grab.Response, error) { + if out == "" { + return grab.Get(out, c.BaseURI+env.Path) + } + return grab.Get(out, c.BaseURI+env.Path) +} + +func (c *Client) UploadMultipartFile(uri, key, path string) (*http.Response, error) { + body, writer := io.Pipe() + + req, err := http.NewRequest(http.MethodPost, uri, body) + if err != nil { + return nil, err + } + + mwriter := multipart.NewWriter(writer) + req.Header.Add("Content-Type", mwriter.FormDataContentType()) + + errchan := make(chan error) + + go func() { + defer close(errchan) + defer writer.Close() + defer mwriter.Close() + + w, err := mwriter.CreateFormFile(key, path) + if err != nil { + errchan <- err + return + } + + in, err := os.Open(path) + if err != nil { + errchan <- err + return + } + defer in.Close() + + if written, err := io.Copy(w, in); err != nil { + errchan <- fmt.Errorf("error copying %s (%d bytes written): %v", path, written, err) + return + } + + if err := mwriter.Close(); err != nil { + errchan <- err + return + } + }() + + resp, err := c.Do(req) + merr := <-errchan + + if err != nil || merr != nil { + return resp, fmt.Errorf("http error: %v, multipart error: %v", err, merr) + } + + return resp, nil +} + +func mustOpen(f string) *os.File { + r, err := os.Open(f) + if err != nil { + panic(err) + } + return r } From d56a9e452f51085f0883733486d201bdd0389c93 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 18 Aug 2025 11:09:51 -0600 Subject: [PATCH 16/89] feat: updated server implementation --- pkg/service/constants.go | 51 +++++++++++++++++ pkg/service/plugins.go | 120 +++++++++++++++++++++++++++++++++++++++ pkg/service/profile.go | 41 +------------ pkg/service/routes.go | 112 +++++++++++++++++++++++++++++++----- pkg/service/service.go | 99 +++++++++++++++++++++++++------- 5 files changed, 348 insertions(+), 75 deletions(-) create mode 100644 pkg/service/constants.go create mode 100644 pkg/service/plugins.go diff --git a/pkg/service/constants.go b/pkg/service/constants.go new file mode 100644 index 0000000..52e990a --- /dev/null +++ b/pkg/service/constants.go @@ -0,0 +1,51 @@ +package service + +const ( + RELPATH_PLUGINS = "/plugins" + RELPATH_PROFILES = "/profiles" + RELPATH_DATA = "/data" + RELPATH_METADATA = "/.configurator" + RELPATH_HELP = RELPATH_DATA + "/index.html" + + DEFAULT_TIMEOUT_IN_SECS = 60 + DEFAULT_PLUGINS_MAX_COUNT = 64 + DEFAULT_PROFILES_MAX_COUNT = 256 + DEFAULT_METADATA = `` + DEFAULT_HOME = ` + + + +

+ # setup environment variables
+ export CONFIGURATOR_HOST={{ configurator.host }}
+ export CONFIGURATOR_PATH={{ configurator.path }}
+ export CONFIGURATOR_SERVER_ROOT={{ configurator.server_root }}
+
+ # start the service
+ configurator serve --root $HOME/apps/configurator/server --init
+
+ # download a file or directory (as archive)
+ configurator download
+ configurator download --host http://localhost:5050 --path help.txt
+
+ # download files with rendering using plugins
+ configurator download --plugins smd,jinja2 --profile compute
+ curl $CONFIGURATOR_HOST/download/help.txt?plugins=smd,jinja2
+
+ # upload a file or directory (recursively)
+ configurator upload
+ configurator upload --host http://localhost:5050 --path help.txt
+
+ # list the files in a directory
+ configurator list --path help.txt
+ configurator list --host http://localhost:5050 --path help.txt
+ curl http://localhost:5050/list/test
+

+ + +` +) + +// configurator.host: https://example.com +// configurator.path: test +// configurator.server_root: $HOME/apps/configurator diff --git a/pkg/service/plugins.go b/pkg/service/plugins.go new file mode 100644 index 0000000..c3a6c43 --- /dev/null +++ b/pkg/service/plugins.go @@ -0,0 +1,120 @@ +package service + +import ( + "encoding/json" + "io" + "net/http" + "os" + + configurator "git.towk2.me/towk/configurator/pkg" +) + +func (s *Service) ListPlugins() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + plugins map[string]configurator.Plugin + names []string + body []byte + err error + ) + + plugins, err = LoadPluginsFromDir(s.PathForPlugins()) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + for name := range plugins { + names = append(names, name) + } + + body, err = json.Marshal(names) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(body) + } +} + +func (s *Service) CreatePlugin() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + plugin configurator.Plugin + path string + err error + ) + + plugin, err = getPluginFromRequestBody(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // helper to check for valid plugin name + var hasValidName = func(name string) bool { + return name != "" && len(name) < 64 + } + + // check for a valid plugin name + if !hasValidName(plugin.Name()) { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // save plugin at path using it's name + path = s.PathForPluginWithName(plugin.Name()) + err = SavePluginToFile(path, &plugin) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + } +} + +func (s *Service) DeletePlugin() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + path string + plugin configurator.Plugin + err error + ) + + plugin, err = getPluginFromRequestBody(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + path = s.PathForPluginWithName(plugin.Name()) + err = os.Remove(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + } +} + +func getPluginFromRequestBody(r *http.Request) (configurator.Plugin, error) { + var ( + plugin configurator.Plugin + body []byte + err error + ) + body, err = io.ReadAll(r.Body) + if err != nil { + return nil, err + } + + err = json.Unmarshal(body, &plugin) + if err != nil { + return nil, err + } + + return plugin, nil +} diff --git a/pkg/service/profile.go b/pkg/service/profile.go index 2fa7c2d..9ba7f2e 100644 --- a/pkg/service/profile.go +++ b/pkg/service/profile.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" - configurator "git.towk2.me/towk/configurator/pkg" "github.com/go-chi/chi/v5" "github.com/tidwall/sjson" ) @@ -26,7 +25,7 @@ type Profile struct { func (s *Service) GetProfiles() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - path = s.RootPath + PROFILES_RELPATH + path = s.RootPath + RELPATH_PROFILES profiles []*Profile contents []byte err error @@ -272,44 +271,6 @@ func (s *Service) GetProfileData() http.HandlerFunc { // return func(w http.ResponseWriter, r *http.Request) {} // } -func (s *Service) GetPlugins() http.HandlerFunc { - - return func(w http.ResponseWriter, r *http.Request) { - var ( - plugins map[string]configurator.Plugin - names []string - body []byte - err error - ) - - plugins, err = LoadPluginsFromDir(s.PathForPlugins()) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - for name := range plugins { - names = append(names, name) - } - - body, err = json.Marshal(names) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Write(body) - } -} - -func (s *Service) CreatePlugins() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) DeletePlugins() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - func loadProfileContents(path string) ([]byte, error) { var ( contents []byte diff --git a/pkg/service/routes.go b/pkg/service/routes.go index 21781df..eb6f6cc 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -3,44 +3,91 @@ package service import ( "encoding/json" "fmt" + "io/fs" "net/http" "os" "path/filepath" "strings" + "time" "git.towk2.me/towk/configurator/pkg/util" + "github.com/rs/zerolog/log" ) func (s *Service) Download() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - path = strings.TrimPrefix(r.URL.Path, "/download") + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download") fileInfo os.FileInfo out *os.File + contents []byte err error ) - fmt.Printf("download path: %v\n", path) + log.Debug(). + Str("path", path). + Str("client_host", r.Host). + Msg("Service.Download()") // determine if path is directory, file, or exists - if fileInfo, err = os.Stat(filepath.Clean(path)); err != nil { + if fileInfo, err = os.Stat(path); err == nil { if fileInfo.IsDir() { - // recursively walk dir acompressednd get all filenames - // download directory as archive - out, err = os.Create(fmt.Sprintf("%s.tar", path)) - if err != nil { + // create an archive of the directory and download + log.Debug(). + Str("type", "directory"). + Msg("Service.Download()") + archivePath := fmt.Sprintf("%d.tar.gz", time.Now().Unix()) + out, err = os.Create(archivePath) + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to create named file: %v", err), http.StatusInternalServerError) + return } - err = util.CreateArchive([]string{path}, out) - if err != nil { + filesToArchive := []string{} + filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + filesToArchive = append(filesToArchive, path) + } + return nil + }) + log.Debug().Strs("files", filesToArchive).Send() + err = util.CreateArchive(filesToArchive, out) + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to create archive: %v", err.Error()), http.StatusInternalServerError) + return + } + + contents, err = os.ReadFile(archivePath) + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to read archive: %v", err.Error()), http.StatusInternalServerError) + return + } + + w.Write(contents) + + err = os.Remove(archivePath) + if err != nil { + log.Error().Err(err).Msg("failed to remove temporary archive") + return } } else { // download individual file + log.Debug(). + Str("type", "file"). + Msg("Service.Download()") + contents, err = os.ReadFile(path) + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to read file to download: %v", err.Error()), http.StatusInternalServerError) + return + } + w.Write(contents) } + } else { + s.writeErrorResponse(w, err.Error(), http.StatusBadRequest) + return } - } } @@ -52,19 +99,54 @@ func (s *Service) Upload() http.HandlerFunc { func (s *Service) List() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var ( + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/list") + entries []string + body []byte + err error + ) + // show what we're listing + log.Debug().Str("path", path).Msg("Service.List()") + + // walk directory and show all entries "ls" + err = filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + entries = append(entries, d.Name()) + return nil + }) + if err != nil { + switch err { + case fs.ErrNotExist, fs.ErrInvalid: + http.Error(w, "No such file or directory...", http.StatusBadRequest) + case fs.ErrPermission: + http.Error(w, "Invalid permissions...", http.StatusForbidden) + default: + http.Error(w, "Something went wrong (file or directory *probably* does not exist)...", http.StatusInternalServerError) + } + return + } + + body, err = json.Marshal(entries) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(body) } } func (s *Service) GetStatus(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - data := map[string]any{ - "code": 200, + err := json.NewEncoder(w).Encode(map[string]any{ + "code": http.StatusOK, "message": "Configurator is healthy", - } - err := json.NewEncoder(w).Encode(data) + }) if err != nil { - fmt.Printf("failed to encode JSON: %v\n", err) + fmt.Printf("failed to encode JSON response body: %v\n", err) return } } diff --git a/pkg/service/service.go b/pkg/service/service.go index c208b15..528cdac 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -15,36 +15,69 @@ import ( "git.towk2.me/towk/configurator/pkg/util" "github.com/go-chi/chi/middleware" "github.com/go-chi/chi/v5" -) - -const ( - PLUGINS_RELPATH = "/plugins" - TEMPLATES_RELPATH = "/templates" - PROFILES_RELPATH = "/profiles" + "github.com/rs/zerolog/log" ) type Service struct { + Addr string RootPath string `yaml:"root,omitempty"` Environment map[string]string // max counts PluginsMaxCount int ProfilesMaxCount int + Timeout time.Duration } -// New creates the directories at specified path +// New creates a new Service instance with default values func New() *Service { return &Service{ - RootPath: ".", + Addr: ":5050", + RootPath: "./", Environment: map[string]string{ - "CONFIGURATOR_HOST_URI": "", - "ACCESS_TOKEN": "", + "CONFIGURATOR_HOST": "", + "CONFIGURATOR_ROOT": "", + "ACCESS_TOKEN": "", }, - PluginsMaxCount: 64, - ProfilesMaxCount: 256, + PluginsMaxCount: DEFAULT_PLUGINS_MAX_COUNT, + ProfilesMaxCount: DEFAULT_PROFILES_MAX_COUNT, + Timeout: DEFAULT_TIMEOUT_IN_SECS, } } +// Init() sets up the default files and directories for the service +func (s *Service) Init() error { + // create the default directories + var err error + err = os.MkdirAll(s.RootPath, 0o777) + if err != nil { + return fmt.Errorf("failed to make service root path: %v", err) + } + err = os.MkdirAll(s.PathForPlugins(), 0o777) + if err != nil { + return fmt.Errorf("failed to make service plugin path: %v", err) + } + err = os.MkdirAll(s.PathForProfiles(), 0o777) + if err != nil { + return fmt.Errorf("failed to make service profile path: %v", err) + } + err = os.MkdirAll(s.PathForData(), 0o777) + if err != nil { + return fmt.Errorf("failed to make service profile path: %v", err) + } + + // create the default files + err = os.WriteFile(s.PathForMetadata(), []byte(DEFAULT_METADATA), 0o777) + if err != nil { + return fmt.Errorf("failed to make service metadata file: %v", err) + } + err = os.WriteFile(s.PathForHome(), []byte(DEFAULT_HOME), 0o777) + if err != nil { + return fmt.Errorf("failed to make service metadata file: %v", err) + } + return nil +} + // Serve() starts the configurator service and waits for requests. func (s *Service) Serve() error { router := chi.NewRouter() @@ -53,15 +86,16 @@ func (s *Service) Serve() error { router.Use(middleware.Logger) router.Use(middleware.Recoverer) router.Use(middleware.StripSlashes) - router.Use(middleware.Timeout(60 * time.Second)) + router.Use(middleware.Timeout(s.Timeout * time.Second)) if s.requireAuth() { } else { // general + // router.Handle("/download/*", http.StripPrefix("/download/", http.FileServer(http.Dir(s.PathForData())))) router.Get("/download/*", s.Download()) router.Post("/upload", s.Upload()) - router.Get("/list", s.List()) + router.Get("/list/*", s.List()) // profiles router.Get("/profiles", s.GetProfiles()) @@ -76,14 +110,14 @@ func (s *Service) Serve() error { // router.Get("/profile/{id}/paths/{path}", s.GetProfilePath()) // plugins - router.Get("/plugins", s.GetPlugins()) - router.Post("/plugins", s.CreatePlugins()) - router.Delete("/plugins/{id}", s.DeletePlugins()) + router.Get("/plugins", s.ListPlugins()) + router.Post("/plugins", s.CreatePlugin()) + router.Delete("/plugins/{id}", s.DeletePlugin()) } // always available public routes go here router.HandleFunc("/status", s.GetStatus) - return http.ListenAndServe(":8080", router) + return http.ListenAndServe(s.Addr, router) } func (s *Service) requireAuth() bool { @@ -225,9 +259,34 @@ func saveToJSONFile[T any](path string, data T) error { } func (s *Service) PathForProfileWithID(id string) string { - return s.RootPath + PROFILES_RELPATH + "/" + id + return s.RootPath + RELPATH_PROFILES + "/" + id +} + +func (s *Service) PathForPluginWithName(name string) string { + return s.RootPath + RELPATH_PLUGINS + "/" + name +} + +func (s *Service) PathForProfiles() string { + return s.RootPath + RELPATH_PROFILES + "/" } func (s *Service) PathForPlugins() string { - return s.RootPath + PLUGINS_RELPATH + return s.RootPath + RELPATH_PLUGINS + "/" +} + +func (s *Service) PathForData() string { + return s.RootPath + RELPATH_DATA +} + +func (s *Service) PathForMetadata() string { + return s.RootPath + RELPATH_METADATA +} + +func (s *Service) PathForHome() string { + return s.RootPath + RELPATH_HELP +} + +func (s *Service) writeErrorResponse(w http.ResponseWriter, message string, code int) { + http.Error(w, message, code) + log.Error().Msg(message) } From 97fa0a1062f0862be7dcbd947676425ce9779d9a Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 19 Aug 2025 21:32:13 -0600 Subject: [PATCH 17/89] chore: updated go deps --- go.mod | 20 ++++++++++--- go.sum | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 68464fd..6b1cf7b 100644 --- a/go.mod +++ b/go.mod @@ -1,23 +1,30 @@ module git.towk2.me/towk/configurator -go 1.21.5 +go 1.24.4 + +toolchain go1.24.6 require ( github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 + github.com/cavaliergopher/grab/v3 v3.0.1 github.com/go-chi/chi v1.5.5 github.com/go-chi/chi/v5 v5.1.0 github.com/lestrrat-go/jwx/v2 v2.1.1 + github.com/nikolalohinski/gonja v1.5.3 + github.com/nikolalohinski/gonja/v2 v2.3.5 github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.8.0 github.com/tidwall/sjson v1.2.5 - golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 ) require ( - github.com/cavaliergopher/grab/v3 v3.0.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/goccy/go-json v0.10.3 // indirect + github.com/goph/emperror v0.17.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect @@ -25,11 +32,16 @@ require ( github.com/lestrrat-go/option v1.0.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/segmentio/asm v1.2.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/gjson v1.14.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.23.0 // indirect ) diff --git a/go.sum b/go.sum index 90214c9..db30abb 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,15 @@ +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8= +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -9,15 +17,40 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE= github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= @@ -35,21 +68,50 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/nikolalohinski/gonja/v2 v2.3.5 h1:7ukCnsokmOIGXOjgW/WrM+xqgwjsQcU0ejFrrz4HQXk= +github.com/nikolalohinski/gonja/v2 v2.3.5/go.mod h1:UIzXPVuOsr5h7dZ5DUbqk3/Z7oFA/NLGQGMjqT4L2aU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= +github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= +github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -61,16 +123,38 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 4d33b12fe0e891de6cec3154749edf5ec6812e80 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 19 Aug 2025 21:32:45 -0600 Subject: [PATCH 18/89] feat: added 'profile' and 'plugins' flags --- cmd/download.go | 2 ++ cmd/plugin.go | 11 +++++++++++ cmd/root.go | 3 +++ 3 files changed, 16 insertions(+) create mode 100644 cmd/plugin.go diff --git a/cmd/download.go b/cmd/download.go index 1fb211b..5f787df 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -84,6 +84,8 @@ func init() { downloadCmd.Flags().StringVar(&host, "host", "http://localhost:5050", "Set the configurator remote host (can be set with CONFIGURATOR_HOST)") downloadCmd.Flags().StringVarP(&path, "path", "p", ".", "Set the path to list files (can be set with CONFIGURATOR_PATH)") downloadCmd.Flags().StringVarP(&outputPath, "output", "o", "", "Set the output path to write files") + downloadCmd.Flags().StringVar(&profile, "profile", "", "Set the profile to use to populate data store") + downloadCmd.Flags().StringSliceVar(&plugins, "plugins", []string{}, "Set the plugins to run before downloading files") rootCmd.AddCommand(&downloadCmd) } diff --git a/cmd/plugin.go b/cmd/plugin.go new file mode 100644 index 0000000..bd9c7bd --- /dev/null +++ b/cmd/plugin.go @@ -0,0 +1,11 @@ +package cmd + +import "github.com/spf13/cobra" + +var pluginCmd = &cobra.Command{} + +var pluginCompileCmd = &cobra.Command{} + +func init() { + rootCmd.AddCommand(pluginCmd, pluginCompileCmd) +} diff --git a/cmd/root.go b/cmd/root.go index da93c0c..40ecd25 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,8 +17,11 @@ var ( outputPath string rootPath string logLevel string + profile string + plugins []string timeout int ) + var rootCmd = cobra.Command{ Use: "configurator", Short: "Extensible configuration builder to download files", From 5c4bbe0b58c9133bda162153671b543595b6d242 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 19 Aug 2025 21:33:28 -0600 Subject: [PATCH 19/89] feat: added KV storage interface --- pkg/storage/disk.go | 11 +++++++++++ pkg/storage/memory.go | 20 ++++++++++++++++++++ pkg/storage/storage.go | 13 +++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 pkg/storage/disk.go create mode 100644 pkg/storage/memory.go create mode 100644 pkg/storage/storage.go diff --git a/pkg/storage/disk.go b/pkg/storage/disk.go new file mode 100644 index 0000000..0a17cef --- /dev/null +++ b/pkg/storage/disk.go @@ -0,0 +1,11 @@ +package storage + +type DiskStorage struct{} + +func (ds *DiskStorage) Read(k string) error { + return nil +} + +func (ms *DiskStorage) Write(k string, v any) error { + return nil +} diff --git a/pkg/storage/memory.go b/pkg/storage/memory.go new file mode 100644 index 0000000..d3890c5 --- /dev/null +++ b/pkg/storage/memory.go @@ -0,0 +1,20 @@ +package storage + +import "fmt" + +type MemoryStorage struct { + Data map[string]any +} + +func (ms *MemoryStorage) Read(k string) (any, error) { + v, ok := ms.Data[k] + if ok { + return v, nil + } + return nil, fmt.Errorf("value does not exist") +} + +func (ms *MemoryStorage) Write(k string, v any) error { + ms.Data[k] = v + return nil +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go new file mode 100644 index 0000000..28f437b --- /dev/null +++ b/pkg/storage/storage.go @@ -0,0 +1,13 @@ +package storage + +type KVStore interface { + Get(k string) (any, error) + Set(k string, v any) error + GetData() any +} + +type KVStaticStore[T any] interface { + Get(k string) (T, error) + Set(k string, v T) error + GetData() T +} From 2da5ca37023c2ec5cd9b3e12a2fe7caa3f6925da Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 19 Aug 2025 21:33:58 -0600 Subject: [PATCH 20/89] feat: added jinja2 and userdata plugins --- pkg/plugin.go | 10 ++++---- pkg/plugins/jinja2.go | 55 +++++++++++++++++++++++++++++++++++------ pkg/plugins/userdata.go | 28 +++++++++++++++++++++ 3 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 pkg/plugins/userdata.go diff --git a/pkg/plugin.go b/pkg/plugin.go index d7ff3e4..b3841a7 100644 --- a/pkg/plugin.go +++ b/pkg/plugin.go @@ -1,5 +1,9 @@ package configurator +import ( + "git.towk2.me/towk/configurator/pkg/storage" +) + type Plugin interface { Name() string Version() string @@ -7,10 +11,6 @@ type Plugin interface { Metadata() map[string]string Init() error - Run(args []string) error + Run(data storage.KVStore, args []string) error Cleanup() error } - -func RunPlugin() { - -} diff --git a/pkg/plugins/jinja2.go b/pkg/plugins/jinja2.go index 9cb515d..ffbbd77 100644 --- a/pkg/plugins/jinja2.go +++ b/pkg/plugins/jinja2.go @@ -1,19 +1,58 @@ package plugin +import ( + "bytes" + + "git.towk2.me/towk/configurator/pkg/storage" + "github.com/nikolalohinski/gonja/v2" + "github.com/nikolalohinski/gonja/v2/exec" +) + type Jinja2 struct{} -func Name() string { return "jinja2" } -func Version() string { return "test" } -func Description() string { return "Renders Jinja 2 templates" } -func Metadata() map[string]string { +func (p *Jinja2) Name() string { return "jinja2" } +func (p *Jinja2) Version() string { return "test" } +func (p *Jinja2) Description() string { return "Renders Jinja 2 templates" } +func (p *Jinja2) Metadata() map[string]string { return map[string]string{ "author.name": "David J. Allen", "author.email": "davidallendj@gmail.com", } } -func Init() { - // initialize Jinja2 (gonja) +func (p *Jinja2) Init() error { + // nothing to initialize + return nil +} + +func (p *Jinja2) Run(data storage.KVStore, args []string) error { + // render the files using Jinja 2 from args + newContent := []string{} + for _, arg := range args { + var ( + context *exec.Context + template *exec.Template + output bytes.Buffer + err error + ) + template, err = gonja.FromString(arg) + if err != nil { + panic(err) + } + + context = exec.NewContext(data.GetData().(map[string]any)) + if err = template.Execute(&output, context); err != nil { // Prints: Hello Bob! + panic(err) + } + newContent = append(newContent, output.String()) + } + + // write back to the data storage + data.Set("out", newContent) + return nil +} + +func (p *Jinja2) Cleanup() error { + // nothing to clean up + return nil } -func Run() {} -func Cleanup() {} diff --git a/pkg/plugins/userdata.go b/pkg/plugins/userdata.go new file mode 100644 index 0000000..4b274b9 --- /dev/null +++ b/pkg/plugins/userdata.go @@ -0,0 +1,28 @@ +package plugin + +import "git.towk2.me/towk/configurator/pkg/storage" + +type UserData struct{} + +func (p *UserData) Name() string { return "jinja2" } +func (p *UserData) Version() string { return "test" } +func (p *UserData) Description() string { return "Renders Jinja 2 templates" } +func (p *UserData) Metadata() map[string]string { + return map[string]string{ + "author.name": "David J. Allen", + "author.email": "davidallendj@gmail.com", + } +} + +func (p *UserData) Init() error { + // nothing to initialize + return nil +} + +func (p *UserData) Run(data storage.KVStore, args []string) error { + return nil +} + +func (p *UserData) Clean() error { + return nil +} From a54b39b296ef4156b22fb442b614afe1c3ed6d6f Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:31:10 -0600 Subject: [PATCH 21/89] chore: updated go deps --- go.mod | 5 +---- go.sum | 46 ---------------------------------------------- 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/go.mod b/go.mod index 6b1cf7b..4090b3f 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module git.towk2.me/towk/configurator +module git.towk2.me/towk/makeshift go 1.24.4 @@ -7,10 +7,8 @@ toolchain go1.24.6 require ( github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 github.com/cavaliergopher/grab/v3 v3.0.1 - github.com/go-chi/chi v1.5.5 github.com/go-chi/chi/v5 v5.1.0 github.com/lestrrat-go/jwx/v2 v2.1.1 - github.com/nikolalohinski/gonja v1.5.3 github.com/nikolalohinski/gonja/v2 v2.3.5 github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.8.0 @@ -22,7 +20,6 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/goccy/go-json v0.10.3 // indirect - github.com/goph/emperror v0.17.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect diff --git a/go.sum b/go.sum index db30abb..3c1e2f4 100644 --- a/go.sum +++ b/go.sum @@ -2,14 +2,8 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8= -github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= -github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -19,10 +13,6 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnN github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE= -github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -32,25 +22,15 @@ github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZ github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= -github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= -github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= @@ -68,38 +48,27 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= -github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= github.com/nikolalohinski/gonja/v2 v2.3.5 h1:7ukCnsokmOIGXOjgW/WrM+xqgwjsQcU0ejFrrz4HQXk= github.com/nikolalohinski/gonja/v2 v2.3.5/go.mod h1:UIzXPVuOsr5h7dZ5DUbqk3/Z7oFA/NLGQGMjqT4L2aU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= @@ -107,8 +76,6 @@ github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyh github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -123,38 +90,25 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= -github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From d7a0ddc1c2e26a89be9f43a2eb0e00df8936c2ec Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:31:51 -0600 Subject: [PATCH 22/89] chore: updated changelog --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51332f7..cc94e2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -[0.2.0] - -- Major rewrite of entire code base [0.1.0] -- Initial prerelease of configurator \ No newline at end of file +- Initial prerelease of makeshift \ No newline at end of file From 68d905067f23420a512657ca8e6c1bad0776538e Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:32:12 -0600 Subject: [PATCH 23/89] chore: updated readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ca94345..c7f620d 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ targets: The tool can also run as a service to generate files for clients: ```bash -export CONFIGURATOR_JWKS_URL="http://my.openchami.cluster:8443/key" +export MAKESHIFT_JWKS_URL="http://my.openchami.cluster:8443/key" ./configurator serve --config config.yaml ``` @@ -62,7 +62,7 @@ curl http://127.0.0.1:3334/generate?target=coredhcp -X GET -H "Authorization: Be ./configurator fetch --target coredhcp --host http://127.0.0.1:3334 --cacert ochami.pem ``` -This will do the same thing as the `generate` subcommand, but through a GET request where the file contents is returned in the response. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set when starting the server with `serve`. The `ACCESS_TOKEN` environment variable is passed to `curl` using the `Authorization` header and expects a token as a JWT. +This will do the same thing as the `generate` subcommand, but through a GET request where the file contents is returned in the response. The access token is only required if the `MAKESHIFT_JWKS_URL` environment variable is set when starting the server with `serve`. The `ACCESS_TOKEN` environment variable is passed to `curl` using the `Authorization` header and expects a token as a JWT. ### Docker From a8e9ed95e6475a030d19817e8cbe8577bfbb1600 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:32:31 -0600 Subject: [PATCH 24/89] chore: updated makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b6784ec..c1e7a87 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Unless set otherwise, the container runtime is Docker DOCKER ?= docker -prog ?= configurator +prog ?= makeshift git_tag := $(shell git describe --abbrev=0 --tags --always) sources := main.go $(wildcard cmd/*.go) plugin_source_prefix := pkg/generator/plugins @@ -42,12 +42,12 @@ lib/%.so: pkg/generator/plugins/%/*.go docs: go doc github.com/OpenCHAMI/cmd - go doc github.com/OpenCHAMI/pkg/configurator + go doc github.com/OpenCHAMI/pkg/${prog} # remove executable and all built plugins .PHONY: clean clean: - rm -f configurator + rm -f ${prog} rm -f lib/* # run all of the unit tests From 5767a8fd471a77729d9bdc854bffef47a446b8e1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:33:05 -0600 Subject: [PATCH 25/89] chore: moved archlinux pkgbuild script --- {res => build}/archlinux/PKGBUILD | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) rename {res => build}/archlinux/PKGBUILD (58%) diff --git a/res/archlinux/PKGBUILD b/build/archlinux/PKGBUILD similarity index 58% rename from res/archlinux/PKGBUILD rename to build/archlinux/PKGBUILD index 5796121..3d9d030 100644 --- a/res/archlinux/PKGBUILD +++ b/build/archlinux/PKGBUILD @@ -1,15 +1,15 @@ -# Maintainer: David J. Allen -pkgname=configurator +# Maintainer: David J. Allen +pkgname=makeshift pkgver=v0.1.0alpha pkgrel=1 -pkgdesc="An extensible tool to dynamically generate config files from SMD with Jinja 2 templating support." +pkgdesc="Extensible file cobbler" arch=("x86_64") -url="https://github.com/OpenCHAMI/configurator" +url="https://git.towk2.me/towk/makeshift" license=('MIT') -groups=("openchami") -provides=('configurator') -conflicts=('configurator') -https://git.towk2.me/towk/configurator/releases/download/v0.1.0-alpha/configurator +# groups=("towk") +provides=('makeshift') +conflicts=('makeshift') +# https://git.towk2.me/towk/makeshift/releases/download/v0.1.0-alpha/makeshift source_x86_64=( "${url}/releases/download/v0.1.0-alpha/${pkgname}.tar.gz" ) @@ -27,7 +27,7 @@ package() { # install the binary to /usr/bin mkdir -p "${pkgdir}/usr/bin" mkdir -p "${pkgdir}/usr/lib/${pkgname}" - install -m755 configurator "${pkgdir}/usr/bin/configurator" + install -m755 makeshift "${pkgdir}/usr/bin/makeshift" # install plugins to /usr/lib install -m755 *.so "${pkgdir}/usr/lib/${pkgname}" From c24bcf34d43def608bcd00eabd6e8bce2e5dfbe6 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:33:41 -0600 Subject: [PATCH 26/89] refactor: removed the example plugins from configurator --- examples/templates/conman.jinja | 23 -------------- examples/templates/dhcpd.jinja | 51 ------------------------------- examples/templates/dnsmasq.jinja | 10 ------ examples/templates/powerman.jinja | 18 ----------- examples/templates/test.j2 | 16 ---------- 5 files changed, 118 deletions(-) delete mode 100644 examples/templates/conman.jinja delete mode 100644 examples/templates/dhcpd.jinja delete mode 100644 examples/templates/dnsmasq.jinja delete mode 100644 examples/templates/powerman.jinja delete mode 100644 examples/templates/test.j2 diff --git a/examples/templates/conman.jinja b/examples/templates/conman.jinja deleted file mode 100644 index 69e779c..0000000 --- a/examples/templates/conman.jinja +++ /dev/null @@ -1,23 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# -SERVER keepalive=ON -SERVER logdir="/var/log/conman" -SERVER logfile="/var/log/conman.log" -SERVER loopback=ON -SERVER pidfile="/var/run/conman.pid" -SERVER resetcmd="/usr/bin/powerman -0 \%N; sleep 5; /usr/bin/powerman -1 \%N" -SERVER tcpwrappers=ON -#SERVER timestamp=1h - -GLOBAL seropts="115200,8n1" -GLOBAL log="/var/log/conman/console.\%N" -GLOBAL logopts="sanitize,timestamp" - -{{ consoles }} diff --git a/examples/templates/dhcpd.jinja b/examples/templates/dhcpd.jinja deleted file mode 100644 index 264e590..0000000 --- a/examples/templates/dhcpd.jinja +++ /dev/null @@ -1,51 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# -allow booting; -allow bootp; -ddns-update-style interim; -authoritative; - -option space ipxe; - -# Tell iPXE to not wait for ProxyDHCP requests to speed up boot. -option ipxe.no-pxedhcp code 176 = unsigned integer 8; -option ipxe.no-pxedhcp 1; - -option architecture-type code 93 = unsigned integer 16; - -if exists user-class and option user-class = "iPXE" { - filename "http://%{IPADDR}/WW/ipxe/cfg/${mac}"; -} else { - if option architecture-type = 00:0B { - filename "/warewulf/ipxe/bin-arm64-efi/snp.efi"; - } elsif option architecture-type = 00:0A { - filename "/warewulf/ipxe/bin-arm32-efi/placeholder.efi"; - } elsif option architecture-type = 00:09 { - filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi"; - } elsif option architecture-type = 00:07 { - filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi"; - } elsif option architecture-type = 00:06 { - filename "/warewulf/ipxe/bin-i386-efi/snp.efi"; - } elsif option architecture-type = 00:00 { - filename "/warewulf/ipxe/bin-i386-pcbios/undionly.kpxe"; - } -} - -subnet %{NETWORK} netmask %{NETMASK} { - not authoritative; - # option interface-mtu 9000; - option subnet-mask %{NETMASK}; -} - -# Compute Nodes (WIP - see the dhcpd generator plugin) -{{ compute_nodes }} - -# Node entries will follow below -{{ node_entries }} \ No newline at end of file diff --git a/examples/templates/dnsmasq.jinja b/examples/templates/dnsmasq.jinja deleted file mode 100644 index 1cc4dab..0000000 --- a/examples/templates/dnsmasq.jinja +++ /dev/null @@ -1,10 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# -{{ dhcp_hosts }} diff --git a/examples/templates/powerman.jinja b/examples/templates/powerman.jinja deleted file mode 100644 index 44c13e4..0000000 --- a/examples/templates/powerman.jinja +++ /dev/null @@ -1,18 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# -include "/etc/powerman/ipmipower.dev" -include "/etc/powerman/ipmi.dev" - - -# list of devices -{{ devices }} - -# create nodes based on found nodes in hostfile -{{ nodes }} \ No newline at end of file diff --git a/examples/templates/test.j2 b/examples/templates/test.j2 deleted file mode 100644 index 17bd6b6..0000000 --- a/examples/templates/test.j2 +++ /dev/null @@ -1,16 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# - -# TODO: test variables - -# TODO: test if/else statements - -# TODO: test for loops - From ea4819e97a7538c1d71db15a36368785fd80ff64 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:34:38 -0600 Subject: [PATCH 27/89] refactor: moved plugin files --- pkg/plugins/jinja2.go | 58 ------ pkg/plugins/jinja2/jinja2.go | 98 +++++++++ pkg/plugins/smd.go | 1 - pkg/plugins/smd/smd.go | 270 +++++++++++++++++++++++++ pkg/plugins/{ => userdata}/userdata.go | 6 +- 5 files changed, 372 insertions(+), 61 deletions(-) delete mode 100644 pkg/plugins/jinja2.go create mode 100644 pkg/plugins/jinja2/jinja2.go delete mode 100644 pkg/plugins/smd.go create mode 100644 pkg/plugins/smd/smd.go rename pkg/plugins/{ => userdata}/userdata.go (87%) diff --git a/pkg/plugins/jinja2.go b/pkg/plugins/jinja2.go deleted file mode 100644 index ffbbd77..0000000 --- a/pkg/plugins/jinja2.go +++ /dev/null @@ -1,58 +0,0 @@ -package plugin - -import ( - "bytes" - - "git.towk2.me/towk/configurator/pkg/storage" - "github.com/nikolalohinski/gonja/v2" - "github.com/nikolalohinski/gonja/v2/exec" -) - -type Jinja2 struct{} - -func (p *Jinja2) Name() string { return "jinja2" } -func (p *Jinja2) Version() string { return "test" } -func (p *Jinja2) Description() string { return "Renders Jinja 2 templates" } -func (p *Jinja2) Metadata() map[string]string { - return map[string]string{ - "author.name": "David J. Allen", - "author.email": "davidallendj@gmail.com", - } -} - -func (p *Jinja2) Init() error { - // nothing to initialize - return nil -} - -func (p *Jinja2) Run(data storage.KVStore, args []string) error { - // render the files using Jinja 2 from args - newContent := []string{} - for _, arg := range args { - var ( - context *exec.Context - template *exec.Template - output bytes.Buffer - err error - ) - template, err = gonja.FromString(arg) - if err != nil { - panic(err) - } - - context = exec.NewContext(data.GetData().(map[string]any)) - if err = template.Execute(&output, context); err != nil { // Prints: Hello Bob! - panic(err) - } - newContent = append(newContent, output.String()) - } - - // write back to the data storage - data.Set("out", newContent) - return nil -} - -func (p *Jinja2) Cleanup() error { - // nothing to clean up - return nil -} diff --git a/pkg/plugins/jinja2/jinja2.go b/pkg/plugins/jinja2/jinja2.go new file mode 100644 index 0000000..4b64a11 --- /dev/null +++ b/pkg/plugins/jinja2/jinja2.go @@ -0,0 +1,98 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + + configurator "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/storage" + "github.com/nikolalohinski/gonja/v2" + "github.com/nikolalohinski/gonja/v2/exec" + "github.com/rs/zerolog/log" +) + +type Jinja2 struct{} + +func (p *Jinja2) Name() string { return "jinja2" } +func (p *Jinja2) Version() string { return "v0.0.1-alpha" } +func (p *Jinja2) Description() string { return "Renders Jinja 2 templates" } +func (p *Jinja2) Metadata() configurator.Metadata { + return configurator.Metadata{ + "author.name": "David J. Allen", + "author.email": "davidallendj@gmail.com", + "author.links": []string{ + "https://github.com/davidallendj", + "https://git.towk2.me/towk", + }, + } +} + +func (p *Jinja2) Init() error { + // nothing to initialize + log.Debug().Str("plugin", p.Name()).Msg("jinja2.Init()") + return nil +} + +func (p *Jinja2) Run(data storage.KVStore, args []string) error { + // render the files using Jinja 2 from args + var ( + rendered []string + context *exec.Context + template *exec.Template + mappings map[string]any + input any // must be a byte array + output bytes.Buffer + err error + ) + log.Debug(). + Str("plugin", p.Name()). + Any("data", data). + // Bytes("input", input.([]byte)). + Int("arg_count", len(args)). + Msg("Run()") + + input, err = data.Get("file") + if err != nil { + return fmt.Errorf("(jinja2) failed to get input data: %v", err) + } + + // get the templates provided as args to the plugin + template, err = gonja.FromBytes(input.([]byte)) + if err != nil { + return fmt.Errorf("(jinja2) failed to get template from args: %v", err) + } + + // get mappings from shared data + shared, err := data.Get("shared") + if err != nil { + return fmt.Errorf("(jinja2) failed to get data from store: %v", err) + } + + err = json.Unmarshal(shared.([]byte), &mappings) + if err != nil { + return fmt.Errorf("(jinja2) failed to unmarshal mappings from shared data: %v", err) + } + + data.Set("mappings", mappings) + + // use the provided data in the store to render templates + // NOTE: this may be changed to specifically use "shared" data instead + context = exec.NewContext(data.GetData().(map[string]any)) + if err = template.Execute(&output, context); err != nil { // Prints: Hello Bob! + return fmt.Errorf("(jinja2) failed to render template: %v", err) + } + rendered = append(rendered, output.String()) + + // write render templates to data store output + data.Set("out", rendered) + return nil +} + +func (p *Jinja2) Cleanup() error { + // nothing to clean up + log.Debug().Str("plugin", p.Name()).Msg("jinja2.Cleanup()") + return nil +} + +var Makeshift Jinja2 diff --git a/pkg/plugins/smd.go b/pkg/plugins/smd.go deleted file mode 100644 index b0736c3..0000000 --- a/pkg/plugins/smd.go +++ /dev/null @@ -1 +0,0 @@ -package plugin diff --git a/pkg/plugins/smd/smd.go b/pkg/plugins/smd/smd.go new file mode 100644 index 0000000..cb263ab --- /dev/null +++ b/pkg/plugins/smd/smd.go @@ -0,0 +1,270 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/storage" + "github.com/rs/zerolog/log" +) + +// An struct that's meant to extend functionality of the base HTTP client by +// adding commonly made requests to SMD. The implemented functions are can be +// used in generator plugins to fetch data when it is needed to substitute +// values for the Jinja templates used. +type SmdClient struct { + http.Client `json:"-" yaml:"-"` + Host string `yaml:"host"` + Port int `yaml:"port"` + AccessToken string `yaml:"access-token"` + + RedfishEndpoints []RedfishEndpoint `json:"redfish_endpoints"` + EthernetInterfaces []EthernetInterface `json:"ethernet_interfaces"` + Components []Component `json:"components"` +} + +type IPAddr struct { + IpAddress string `json:"IPAddress"` + Network string `json:"Network"` +} + +type EthernetInterface struct { + ID string `json:"ID"` + Description string `json:"Description"` + MacAddress string `json:"MACAddr"` + LastUpdate string `json:"LastUpdate"` + ComponentID string `json:"ComponentID"` + Type string `json:"Type"` + IpAddresses []IPAddr `json:"IPAddresses"` +} + +type Component struct { + ID string `json:"ID"` + Type string `json:"Type"` + State string `json:"State,omitempty"` + Flag string `json:"Flag,omitempty"` + Enabled *bool `json:"Enabled,omitempty"` + SwStatus string `json:"SoftwareStatus,omitempty"` + Role string `json:"Role,omitempty"` + SubRole string `json:"SubRole,omitempty"` + NID json.Number `json:"NID,omitempty"` + Subtype string `json:"Subtype,omitempty"` + NetType string `json:"NetType,omitempty"` + Arch string `json:"Arch,omitempty"` + Class string `json:"Class,omitempty"` + ReservationDisabled bool `json:"ReservationDisabled,omitempty"` + Locked bool `json:"Locked,omitempty"` +} + +type RedfishEndpoint struct { + ID string `json:"ID"` + Type string `json:"Type"` + Name string `json:"Name,omitempty"` // user supplied descriptive name + Hostname string `json:"Hostname"` + Domain string `json:"Domain"` + FQDN string `json:"FQDN"` + Enabled bool `json:"Enabled"` + UUID string `json:"UUID,omitempty"` + User string `json:"User"` + Password string `json:"Password"` // Temporary until more secure method + UseSSDP bool `json:"UseSSDP,omitempty"` + MACRequired bool `json:"MACRequired,omitempty"` + MACAddr string `json:"MACAddr,omitempty"` + IPAddr string `json:"IPAddress,omitempty"` +} + +func (p *SmdClient) Name() string { return "smd" } +func (p *SmdClient) Version() string { return "v0.0.1-alpha" } +func (p *SmdClient) Description() string { return "Fetchs data from SMD and writes to store" } +func (p *SmdClient) Metadata() makeshift.Metadata { + return makeshift.Metadata{ + "author.name": "David J. Allen", + "author.email": "davidallendj@gmail.com", + "author.links": []string{ + "https://github.com/davidallendj", + "https://git.towk2.me/towk", + }, + } +} + +func (p *SmdClient) Init() error { + log.Debug().Str("plugin", p.Name()).Msg("smd.Init()") + return nil +} + +func (p *SmdClient) Run(data storage.KVStore, args []string) error { + // set all the defaults for variables + var ( + client SmdClient + bytes []byte + err error + ) + + // if we have a client, try making the request for the ethernet interfaces + err = client.FetchEthernetInterfaces() + if err != nil { + return fmt.Errorf("(smd) failed to fetch ethernet interfaces with client: %v", err) + } + + err = client.FetchRedfishEndpoints() + if err != nil { + return fmt.Errorf("(smd) failed to fetch redfish endpoints with client: %v", err) + } + err = client.FetchComponents() + if err != nil { + return fmt.Errorf("(smd) failed to fetch components with client: %v", err) + } + + // write data back to shared data store to be used by other plugins + bytes, err = json.Marshal(client) + if err != nil { + return fmt.Errorf("(smd) failed to marshal SMD client: %v") + } + data.Set("shared", bytes) + + // apply template substitutions and return output as byte array + return nil +} + +func (p *SmdClient) Cleanup() error { + log.Debug().Str("plugin", p.Name()).Msg("smd.Init()") + return nil +} + +// Fetch the ethernet interfaces from SMD service using its API. An access token may be required if the SMD +// service SMD_JWKS_URL envirnoment variable is set. +func (client *SmdClient) FetchEthernetInterfaces() error { + var ( + bytes []byte + err error + ) + // make request to SMD endpoint + bytes, err = client.makeRequest("/Inventory/EthernetInterfaces") + if err != nil { + return fmt.Errorf("failed to read HTTP response: %v", err) + } + + // unmarshal response body JSON and extract in object + err = json.Unmarshal(bytes, &client.EthernetInterfaces) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + + // print what we got if verbose is set + log.Debug().Str("ethernet_interfaces", string(bytes)).Msg("found interfaces") + + return nil +} + +// Fetch the components from SMD using its API. An access token may be required if the SMD +// service SMD_JWKS_URL envirnoment variable is set. +func (client *SmdClient) FetchComponents() error { + var ( + bytes []byte + err error + ) + // make request to SMD endpoint + bytes, err = client.makeRequest("/State/Components") + if err != nil { + return fmt.Errorf("failed to make HTTP request: %v", err) + } + + // make sure our response is actually JSON first + if !json.Valid(bytes) { + return fmt.Errorf("expected valid JSON response: %v", string(bytes)) + } + + // unmarshal response body JSON and extract in object + var tmp map[string]any + err = json.Unmarshal(bytes, &tmp) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + bytes, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) + if err != nil { + return fmt.Errorf("failed to marshal JSON: %v", err) + } + err = json.Unmarshal(bytes, &client.Components) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + + // print what we got if verbose is set + log.Debug().Str("components", string(bytes)).Msg("found components") + + return nil +} + +// TODO: improve implementation of this function +func (client *SmdClient) FetchRedfishEndpoints() error { + var ( + store map[string]any + rfeps []RedfishEndpoint + body []byte + err error + ) + + // make initial request to get JSON with 'RedfishEndpoints' as property + body, err = client.makeRequest("/Inventory/RedfishEndpoints") + if err != nil { + return fmt.Errorf("failed to make HTTP resquest: %v", err) + } + // make sure response is in JSON + if !json.Valid(body) { + return fmt.Errorf("expected valid JSON response: %s", string(body)) + } + err = json.Unmarshal(body, &store) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + + // marshal RedfishEndpoint JSON back to makeshift.RedfishEndpoint + body, err = json.Marshal(store["RedfishEndpoints"].([]any)) + if err != nil { + return fmt.Errorf("failed to marshal JSON: %v", err) + } + err = json.Unmarshal(body, &rfeps) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + + // show the final result + log.Debug().Bytes("redfish_endpoints", body).Msg("found redfish endpoints") + + client.RedfishEndpoints = rfeps + return nil +} + +func (client *SmdClient) makeRequest(endpoint string) ([]byte, error) { + if client == nil { + return nil, fmt.Errorf("client is nil") + } + + // fetch DHCP related information from SMD's endpoint: + url := fmt.Sprintf("%s/hsm/v2%s", client.Host, endpoint) + req, err := http.NewRequest(http.MethodGet, url, bytes.NewBuffer([]byte{})) + if err != nil { + return nil, fmt.Errorf("failed to create new HTTP request: %v", err) + } + + // include access token in authorzation header if found + // NOTE: This shouldn't be needed for this endpoint since it's public + if client.AccessToken != "" { + req.Header.Add("Authorization", "Bearer "+client.AccessToken) + } + + // make the request to SMD + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to make request: %v", err) + } + + // read the contents of the response body + return io.ReadAll(res.Body) +} + +var Makeshift SmdClient diff --git a/pkg/plugins/userdata.go b/pkg/plugins/userdata/userdata.go similarity index 87% rename from pkg/plugins/userdata.go rename to pkg/plugins/userdata/userdata.go index 4b274b9..ae5d31f 100644 --- a/pkg/plugins/userdata.go +++ b/pkg/plugins/userdata/userdata.go @@ -1,6 +1,6 @@ -package plugin +package main -import "git.towk2.me/towk/configurator/pkg/storage" +import "git.towk2.me/towk/makeshift/pkg/storage" type UserData struct{} @@ -26,3 +26,5 @@ func (p *UserData) Run(data storage.KVStore, args []string) error { func (p *UserData) Clean() error { return nil } + +var Makeshift UserData From 0de7beefd0e65777b28d66724a045c78bc1bb7dc Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:35:36 -0600 Subject: [PATCH 28/89] refactor: removed unnecessary cmd commands --- cmd/plugin.go | 11 ----------- cmd/profiles.go | 14 -------------- 2 files changed, 25 deletions(-) delete mode 100644 cmd/plugin.go delete mode 100644 cmd/profiles.go diff --git a/cmd/plugin.go b/cmd/plugin.go deleted file mode 100644 index bd9c7bd..0000000 --- a/cmd/plugin.go +++ /dev/null @@ -1,11 +0,0 @@ -package cmd - -import "github.com/spf13/cobra" - -var pluginCmd = &cobra.Command{} - -var pluginCompileCmd = &cobra.Command{} - -func init() { - rootCmd.AddCommand(pluginCmd, pluginCompileCmd) -} diff --git a/cmd/profiles.go b/cmd/profiles.go deleted file mode 100644 index 7c4294f..0000000 --- a/cmd/profiles.go +++ /dev/null @@ -1,14 +0,0 @@ -package cmd - -import "github.com/spf13/cobra" - -var profilesCmd = &cobra.Command{ - Use: "profiles", - Run: func(cmd *cobra.Command, args []string) { - - }, -} - -func init() { - rootCmd.AddCommand(profilesCmd) -} From 2d6eb1d972400aa24caad757d930015ee464e3f2 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:36:23 -0600 Subject: [PATCH 29/89] refactor: changed name from configurator to makeshift --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 0828d29..ad4d69e 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ package main -import "git.towk2.me/towk/configurator/cmd" +import "git.towk2.me/towk/makeshift/cmd" func main() { cmd.Execute() From 7a96bfd6c7da1a79f07695160ab7e963bc6d66b6 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:38:39 -0600 Subject: [PATCH 30/89] refactor: combined files defining models --- pkg/models.go | 33 +++++ pkg/plugin.go | 16 --- pkg/service/profile.go | 290 ----------------------------------------- 3 files changed, 33 insertions(+), 306 deletions(-) create mode 100644 pkg/models.go delete mode 100644 pkg/plugin.go delete mode 100644 pkg/service/profile.go diff --git a/pkg/models.go b/pkg/models.go new file mode 100644 index 0000000..53a0103 --- /dev/null +++ b/pkg/models.go @@ -0,0 +1,33 @@ +package makeshift + +import "git.towk2.me/towk/makeshift/pkg/storage" + +type Profile struct { + ID string `json:"id"` // profile ID + Description string `json:"description,omitempty"` // profile description + Tags []string `json:"tags,omitempty"` // tags used for ... + Paths []string `json:"paths,omitempty"` // paths to download + Plugins []string `json:"plugins,omitempty"` // plugins to run + Data map[string]any `json:"data,omitempty"` // include render data +} + +type Plugin interface { + Name() string + Version() string + Description() string + Metadata() Metadata + + Init() error + Run(data storage.KVStore, args []string) error + Cleanup() error +} +type Metadata map[string]any +type Hook struct { + Data storage.KVStore + Args []string + Plugin Plugin +} + +func (h *Hook) Run() error { + return h.Plugin.Run(h.Data, h.Args) +} diff --git a/pkg/plugin.go b/pkg/plugin.go deleted file mode 100644 index b3841a7..0000000 --- a/pkg/plugin.go +++ /dev/null @@ -1,16 +0,0 @@ -package configurator - -import ( - "git.towk2.me/towk/configurator/pkg/storage" -) - -type Plugin interface { - Name() string - Version() string - Description() string - Metadata() map[string]string - - Init() error - Run(data storage.KVStore, args []string) error - Cleanup() error -} diff --git a/pkg/service/profile.go b/pkg/service/profile.go deleted file mode 100644 index 9ba7f2e..0000000 --- a/pkg/service/profile.go +++ /dev/null @@ -1,290 +0,0 @@ -package service - -import ( - "encoding/json" - "fmt" - "io" - "io/fs" - "net/http" - "os" - "path/filepath" - - "github.com/go-chi/chi/v5" - "github.com/tidwall/sjson" -) - -type Profile struct { - ID string `json:"id"` // profile ID - Description string `json:"description"` // profile description - Tags []string `json:"tags"` // tags used for ... - Paths []string `json:"paths"` // paths to download - Plugins []string `json:"plugins"` // plugins to run - Data map[string]any `json:"data"` // include render data -} - -func (s *Service) GetProfiles() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ( - path = s.RootPath + RELPATH_PROFILES - profiles []*Profile - contents []byte - err error - ) - - // walk profiles directory to load all profiles - err = filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - // skip directories - if info.IsDir() { - return nil - } - - // read file contents - var profile *Profile - profile, err = LoadProfileFromFile(path) - if err != nil { - return err - } - - profiles = append(profiles, profile) - - fmt.Println(path, info.Size()) - return nil - }) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - // marshal and send all the profiles - contents, err = json.Marshal(profiles) - if err != nil { - http.Error(w, fmt.Sprintf("failed to marshal profiles: %v", err), http.StatusInternalServerError) - } - - _, err = w.Write(contents) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - -// func (s *Service) CreateProfiles() http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) { -// var ( -// path = chi.URLParam(r, "path") -// err error -// ) - -// } -// } - -func (s *Service) GetProfile() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ( - id = chi.URLParam(r, "id") - path = s.PathForProfileWithID(id) - contents []byte - err error - ) - - contents, err = loadProfileContents(path) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - _, err = w.Write(contents) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - -func (s *Service) CreateProfile() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - type input struct { - Path string `json:"path"` - Profile *Profile `json:"profile"` - } - var ( - body, contents []byte - in input - err error - ) - - body, err = io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // use the request info to build profile - err = json.Unmarshal(body, &in) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // serialize just the profile part - contents, err = json.Marshal(in.Profile) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // create a new profile on disk - err = os.WriteFile(in.Path, contents, os.ModePerm) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - w.WriteHeader(http.StatusOK) - } -} - -func (s *Service) SetProfileData() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ( - body, contents []byte - newContents string - profile *Profile - path string - err error - ) - - body, err = io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = json.Unmarshal(body, &profile) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // make sure the request data sets an ID - if profile.ID == "" { - http.Error(w, "ID must be set to a non-empty value", http.StatusBadRequest) - return - } - - // read the contents the file with profile ID - path = s.PathForProfileWithID(profile.ID) - contents, err = os.ReadFile(path) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // modify the data of the profile's contents - newContents, err = sjson.Set(string(contents), "data", profile.Data) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // write only the data to the file with ID - err = os.WriteFile(path, []byte(newContents), os.ModePerm) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - -func (s *Service) DeleteProfileData() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ( - id = chi.URLParam(r, "id") - path = s.PathForProfileWithID(id) - profile *Profile - err error - ) - - // get the profile - profile, err = LoadProfileFromFile(path) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return - } - - // delete the profile data - profile.Data = map[string]any{} - - // save the profile back to the file to update - SaveProfileToFile(path, profile) - - } -} - -func (s *Service) GetProfileData() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ( - id = chi.URLParam(r, "id") - path = s.PathForProfileWithID(id) - profile *Profile - body []byte - err error - ) - - profile, err = LoadProfileFromFile(path) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // only marshal the profile data and not entire profile - body, err = json.Marshal(profile.Data) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // write body to response - _, err = w.Write(body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - -// func (s *Service) CreateProfilePath() http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) { - -// w.WriteHeader(http.StatusOK) -// } -// } - -// func (s *Service) DeleteProfilePath() http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) {} -// } - -// func (s *Service) GetProfilePath() http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) {} -// } - -func loadProfileContents(path string) ([]byte, error) { - var ( - contents []byte - profile *Profile - err error - ) - profile, err = LoadProfileFromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to load profile from file: %v", err) - } - - contents, err = json.Marshal(profile) - if err != nil { - return nil, fmt.Errorf("failed to marshal profile: %v", err) - } - return contents, nil -} From 59a5225b28914d1b981e23a579efbfae7c32a0e4 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:39:39 -0600 Subject: [PATCH 31/89] feat: updated implementations for cmds --- cmd/download.go | 97 ++++++++++++++++++++---------- cmd/list.go | 28 ++++++--- cmd/plugins.go | 157 ++++++++++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 90 ++++++++++++++------------- cmd/run.go | 2 +- cmd/serve.go | 49 +++++++++------ cmd/upload.go | 10 +++ 7 files changed, 331 insertions(+), 102 deletions(-) create mode 100644 cmd/plugins.go diff --git a/cmd/download.go b/cmd/download.go index 5f787df..b431920 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -3,11 +3,13 @@ package cmd import ( "fmt" "net/http" + "net/url" "os" "path/filepath" + "strings" "time" - "git.towk2.me/towk/configurator/pkg/client" + "git.towk2.me/towk/makeshift/pkg/client" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -15,30 +17,40 @@ import ( var downloadCmd = cobra.Command{ Use: "download", Example: ` - # download a file or directory (as archive) - configurator download - configurator download --host https://example.com --path test - configurator download --plugins smd,jinja2 --profile compute - curl $CONFIGURATOR_HOST/download/test?plugins=smd,jinja2 + # set up environment + export MAKESHIFT_HOST=http://localhost:5050 + export MAKESHIFT_PATH=test + + # download a file or directory (as archive) + makeshift download + makeshift download --host http://localhost:5050.com --path test + + # download a file or directory and run plugins with profile data + makeshift download --plugins smd,jinja2 --profile compute + curl $MAKESHIFT_HOST/download/test?plugins=smd,jinja2&profile=test + + # download directory and extract it's contents automatically + makeshift download --extract `, Short: "Download and modify files with plugins", PreRun: func(cmd *cobra.Command, args []string) { - setenv(&host, "CONFIGURATOR_HOST") - setenv(&path, "CONFIGURATOR_PATH") + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") }, Run: func(cmd *cobra.Command, args []string) { var ( - c = client.New(host) - res *http.Response - body []byte - err error - ) + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + outputPath, _ = cmd.Flags().GetString("output") + pluginNames, _ = cmd.Flags().GetStringSlice("plugins") + profileIDs, _ = cmd.Flags().GetStringSlice("profiles") - log.Debug(). - Str("host", host). - Str("path", path). - Str("output", outputPath). - Send() + c = client.New(host) + res *http.Response + query string + body []byte + err error + ) // set output path to match path if empty if outputPath == "" { @@ -49,22 +61,42 @@ var downloadCmd = cobra.Command{ } } - // make request to /download endpoint - // _, err = c.Download(outputPath, client.HTTPEnvelope{ - // Path: fmt.Sprintf("/download/%s", path), - // Method: http.MethodGet, - // }) + query = fmt.Sprintf("/download/%s?", path) + if len(pluginNames) > 0 { + query += "plugins=" + url.QueryEscape(strings.Join(pluginNames, ",")) + } + if len(profileIDs) > 0 { + query += "&profiles=" + url.QueryEscape(strings.Join(profileIDs, ",")) + } + + log.Debug(). + Str("host", host). + Str("path", path). + Str("query", query). + Str("output", outputPath). + Strs("profiles", profileIDs). + Strs("plugins", pluginNames). + Send() res, body, err = c.MakeRequest(client.HTTPEnvelope{ - Path: fmt.Sprintf("/download/%s", path), + Path: query, Method: http.MethodGet, }) if err != nil { - log.Error().Err(err).Str("host", host).Msg("failed to make request") + log.Error().Err(err). + Str("host", host). + Str("path", path). + Str("output", outputPath). + Msg("failed to make request") os.Exit(1) } if res.StatusCode != http.StatusOK { - log.Error().Int("status", res.StatusCode).Str("host", host).Msg("response returned bad status") + log.Error(). + Int("status", res.StatusCode). + Str("host", host). + Str("path", path). + Str("output", outputPath). + Msg("response returned bad status") os.Exit(1) } if outputPath != "" { @@ -80,12 +112,15 @@ var downloadCmd = cobra.Command{ }, } +var downloadProfileCmd = &cobra.Command{} +var downloadPluginCmd = &cobra.Command{} + func init() { - downloadCmd.Flags().StringVar(&host, "host", "http://localhost:5050", "Set the configurator remote host (can be set with CONFIGURATOR_HOST)") - downloadCmd.Flags().StringVarP(&path, "path", "p", ".", "Set the path to list files (can be set with CONFIGURATOR_PATH)") - downloadCmd.Flags().StringVarP(&outputPath, "output", "o", "", "Set the output path to write files") - downloadCmd.Flags().StringVar(&profile, "profile", "", "Set the profile to use to populate data store") - downloadCmd.Flags().StringSliceVar(&plugins, "plugins", []string{}, "Set the plugins to run before downloading files") + downloadCmd.Flags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + downloadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") + downloadCmd.Flags().StringP("output", "o", "", "Set the output path to write files") + downloadCmd.Flags().StringSlice("profiles", []string{}, "Set the profile to use to populate data store") + downloadCmd.Flags().StringSlice("plugins", []string{}, "Set the plugins to run before downloading files") rootCmd.AddCommand(&downloadCmd) } diff --git a/cmd/list.go b/cmd/list.go index f350a43..38cc145 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -6,7 +6,7 @@ import ( "net/http" "os" - "git.towk2.me/towk/configurator/pkg/client" + "git.towk2.me/towk/makeshift/pkg/client" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -14,19 +14,24 @@ import ( var listCmd = &cobra.Command{ Use: "list", Example: ` - # list files in a remote data directory - configurator list --path test - configurator list --host https://example.com --path test - curl https://example.com/list/test + # list files in a remote data directory + configurator list --path test + configurator list --host http://localhost:5050 --path test + + # list files using 'curl' + curl http://localhost:5050/list/test `, Args: cobra.NoArgs, Short: "List all files in a remote data directory", PreRun: func(cmd *cobra.Command, args []string) { - setenv(&host, "CONFIGURATOR_HOST") - setenv(&path, "CONFIGURATOR_PATH") + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") }, Run: func(cmd *cobra.Command, args []string) { var ( + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + c = client.New(host) body []byte output []string @@ -44,7 +49,10 @@ var listCmd = &cobra.Command{ Method: http.MethodGet, }) if err != nil { - log.Error().Err(err).Str("url", host).Msg("failed to make request") + log.Error().Err(err). + Str("host", host). + Str("path", path). + Msg("failed to make request") os.Exit(1) } @@ -60,8 +68,8 @@ var listCmd = &cobra.Command{ } func init() { - listCmd.Flags().StringVar(&host, "host", "http://localhost:5050", "Set the configurator remote host (can be set with CONFIGURATOR_HOST)") - listCmd.Flags().StringVarP(&path, "path", "p", ".", "Set the path to list files (can be set with CONFIGURATOR_PATH)") + listCmd.Flags().String("host", "http://localhost:5050", "Set the configurator remote host (can be set with MAKESHIFT_HOST)") + listCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") rootCmd.AddCommand(listCmd) } diff --git a/cmd/plugins.go b/cmd/plugins.go new file mode 100644 index 0000000..3078e43 --- /dev/null +++ b/cmd/plugins.go @@ -0,0 +1,157 @@ +package cmd + +import ( + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/service" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var pluginsCmd = &cobra.Command{ + Use: "plugins", + Short: "Manage and compile plugins (requires Go build tools)", +} + +var pluginsCompileCmd = &cobra.Command{ + Use: "compile", + Example: ` + # compile plugin using Go build tools + go build -buildmode=plugin -o lib/myplugin.so src/plugins/myplugin.go + + # try to compile all plugins in current directory + cd src/plugins + makeshift plugin compile + + # try to compile all plugins in specified directory + makeshift plugin compile src/plugins + + # compile 'src/plugins/myplugin.go' and save to 'lib/myplugin.so' + makeshift plugin compile src/plugins/myplugin.go -o lib/myplugin.so +`, + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + var ( + outputPath, _ = cmd.Flags().GetString("output") + output []byte + fileInfo os.FileInfo + err error + ) + + // make the directory + err = os.MkdirAll(filepath.Dir(outputPath), 0o777) + if err != nil { + log.Fatal().Err(err).Msg("failed to make output directory") + } + + // one arg passed, so determine if it is file or directory + if len(args) > 0 { + if fileInfo, err = os.Stat(args[0]); err == nil { + if fileInfo.IsDir() { + err = compilePluginsDir(args[0], outputPath) + if err != nil { + log.Fatal().Err(err). + Bytes("output", output). + Msg("failed to compile plugin") + } + } else { + // not a directory so check if Go file so try and compile it + if filepath.Ext(args[0]) == ".go" { + output, err = compilePlugin(outputPath, args[0]) + if err != nil { + log.Fatal().Err(err). + Bytes("output", output). + Msg("failed to compile plugin") + } + } else { + log.Fatal().Msg("argument is not a valid plugin (must be Go file)") + } + } + } else if err != nil { + log.Fatal().Err(err).Msgf("failed to stat provided plugin path") + } + } else { + // no args passed, so use current directory + err = compilePluginsDir(".", outputPath) + if err != nil { + log.Fatal().Err(err). + Bytes("output", output). + Msg("failed to compile plugin") + } + } + }, +} + +var pluginInspectCmd = &cobra.Command{ + Use: "inspect", + Args: cobra.MinimumNArgs(1), + Example: ` + # inspect a plugin and print its information + makeshift plugin inspect lib/jinja2.so +`, + Run: func(cmd *cobra.Command, args []string) { + for _, path := range args { + var ( + plugin makeshift.Plugin + err error + ) + plugin, err = service.LoadPluginFromFile(path) + if err != nil { + log.Error().Err(err). + Str("path", path). + Msg("failed to load plugin from file") + continue + } + + log.Info().Any("plugin", map[string]any{ + "name": plugin.Name(), + "version": plugin.Version(), + "description": plugin.Description(), + "metadata": plugin.Metadata(), + }).Send() + } + }, +} + +func init() { + pluginsCompileCmd.Flags().StringP("output", "o", "", "Set the path to save compiled plugin") + pluginsCmd.AddCommand(pluginsCompileCmd, pluginInspectCmd) + rootCmd.AddCommand(pluginsCmd) +} + +func compilePlugin(outputPath string, srcPath string) ([]byte, error) { + var ( + commandArgs string + command *exec.Cmd + ) + // execute command to build the plugin + commandArgs = fmt.Sprintf("go build -buildmode=plugin -o=%s %s", outputPath, srcPath) + command = exec.Command("bash", "-c", commandArgs) + return command.CombinedOutput() +} + +func compilePluginsDir(dirpath string, outputPath string) error { + err := filepath.WalkDir(dirpath, func(path string, d fs.DirEntry, err error) error { + // not a directory and is Go file, so try and compile it + if !d.IsDir() && filepath.Ext(path) == ".go" { + var ( + localOutputPath string = outputPath + "/" + path + ) + output, err := compilePlugin(localOutputPath, path) + if err != nil { + log.Fatal().Err(err). + Bytes("output", output). + Str("path", localOutputPath). + Msg("failed to compile plugin") + return err + } + } + return nil + }) + return err +} diff --git a/cmd/root.go b/cmd/root.go index 40ecd25..375f4bc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,36 +3,58 @@ package cmd import ( "fmt" "os" - "slices" - "strings" - "github.com/rs/zerolog" + logger "git.towk2.me/towk/makeshift/pkg/log" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) var ( - host string - path string - outputPath string - rootPath string - logLevel string - profile string - plugins []string - timeout int + // host string + // path string + // outputPath string + // rootPath string + // profile string + // plugins []string + // timeout int + // logFile string + logLevel logger.LogLevel = logger.INFO ) var rootCmd = cobra.Command{ - Use: "configurator", - Short: "Extensible configuration builder to download files", + Use: "makeshift", + Short: "Extensible file cobbler", PersistentPreRun: func(cmd *cobra.Command, args []string) { - // set the logging level - level, err := strToLogLevel(logLevel) + var ( + logFile string + err error + ) + + // initialize the logger + logFile, _ = cmd.Flags().GetString("log-file") + err = logger.InitWithLogLevel(logLevel, logFile) if err != nil { - log.Error().Err(err).Msg("failed to convert log level argument") + log.Error().Err(err).Msg("failed to initialize logger") os.Exit(1) } - zerolog.SetGlobalLevel(level) + }, + Run: func(cmd *cobra.Command, args []string) { + // try and set flags using env vars + setenv(cmd, "log-file", "MAKESHIFT_LOG_FILE") + if len(args) == 0 { + err := cmd.Help() + if err != nil { + log.Error().Err(err).Msg("failed to print help") + } + os.Exit(0) + } + }, + PostRun: func(cmd *cobra.Command, args []string) { + log.Debug().Msg("closing log file") + err := logger.LogFile.Close() + if err != nil { + log.Error().Err(err).Msg("failed to close log file") + } }, } @@ -46,34 +68,16 @@ func Execute() { func init() { // initialize the config a single time - rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "info", "Set the log level output") + rootCmd.PersistentFlags().VarP(&logLevel, "log-level", "l", "Set the log level output") + rootCmd.PersistentFlags().String("log-file", "", "Set the log file path (can be set with MAKESHIFT_LOG_FILE)") } -func strToLogLevel(ll string) (zerolog.Level, error) { - levels := []string{"debug", "info", "warn", "disabled"} - if index := slices.Index(levels, ll); index >= 0 { - // handle special case to map index == 3 to zerolog.Disabled == 7 - switch index { - case 3: - return zerolog.Disabled, nil - } - return zerolog.Level(index), nil +func setenv(cmd *cobra.Command, varname string, envvar string) { + if cmd.Flags().Changed(varname) { + return } - return -100, fmt.Errorf( - "invalid log level (options: %s)", strings.Join(levels, ", "), - ) // use 'info' by default -} - -func setenv(v *string, key string) { - t := os.Getenv(key) - if t != "" { - *v = t + val := os.Getenv(envvar) + if val != "" { + cmd.Flags().Set(varname, val) } } - -// func setenv(cmd *cobra.Command, varname string, envvar string) { -// v := os.Getenv(envvar) -// if v != "" { -// cmd.Flags().Set(varname, v) -// } -// } diff --git a/cmd/run.go b/cmd/run.go index 369700c..f932e07 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -4,7 +4,7 @@ import "github.com/spf13/cobra" var runCmd = &cobra.Command{ Use: "run", - Short: "Run the configurator locally", + Short: "Run 'makeshift' locally with plugins", Run: func(cmd *cobra.Command, args []string) { }, diff --git a/cmd/serve.go b/cmd/serve.go index 0358149..252a085 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -1,42 +1,57 @@ package cmd import ( + "net/url" "time" - "git.towk2.me/towk/configurator/pkg/service" + "git.towk2.me/towk/makeshift/pkg/service" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) var serveCmd = &cobra.Command{ - Use: "serve", + Use: "serve", + Example: ` + # start the service in current directory + makeshift serve + + # start the service with root path and initialize + makeshift serve --root ./test --init -l debug +`, Args: cobra.NoArgs, PreRun: func(cmd *cobra.Command, args []string) { - // try and set flags using env vars - setenv(&host, "CONFIGURATOR_HOST") - setenv(&rootPath, "CONFIGURATOR_ROOT") + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "root", "MAKESHIFT_SERVER_ROOT") + setenv(cmd, "timeout", "MAKESHIFT_TIMEOUT") }, Run: func(cmd *cobra.Command, args []string) { var ( - host string - rootPath string - server *service.Service - err error + host, _ = cmd.Flags().GetString("host") + rootPath, _ = cmd.Flags().GetString("root") + timeout, _ = cmd.Flags().GetInt("timeout") + + parsed *url.URL + server *service.Service + err error ) - // get vars but don't modify - host, _ = cmd.Flags().GetString("host") - rootPath, _ = cmd.Flags().GetString("root") + // parse the host to remove scheme if needed + parsed, err = url.Parse(host) + if err != nil { + log.Warn().Err(err). + Str("host", host). + Msg("could not parse host") + } // set the server values server = service.New() - server.Addr = host + server.Addr = parsed.Host server.RootPath = rootPath server.Timeout = time.Duration(timeout) * time.Second // show some debugging information log.Debug(). - Str("host", host). + Str("host", parsed.Host). Any("paths", map[string]string{ "root": rootPath, "data": server.PathForData(), @@ -59,9 +74,9 @@ var serveCmd = &cobra.Command{ func init() { serveCmd.Flags().Bool("init", false, "Initializes default files at specified with the '--root' flag") - serveCmd.Flags().StringVar(&host, "host", "localhost:5050", "Set the configurator server host (can be set with CONFIGURATOR_HOST)") - serveCmd.Flags().StringVar(&rootPath, "root", "./", "Set the root path to serve files (can be set with CONFIGURATOR_ROOT)") - serveCmd.Flags().IntVarP(&timeout, "timeout", "t", 60, "Set the timeout in seconds for requests.") + serveCmd.Flags().String("host", "localhost:5050", "Set the configurator server host (can be set with MAKESHIFT_HOST)") + serveCmd.Flags().String("root", "./", "Set the root path to serve files (can be set with MAKESHIFT_ROOT)") + serveCmd.Flags().IntP("timeout", "t", 60, "Set the timeout in seconds for requests (can be set with MAKESHIFT_TIMEOUT)") rootCmd.AddCommand(serveCmd) } diff --git a/cmd/upload.go b/cmd/upload.go index 9fc1f12..39aa49f 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -9,6 +9,16 @@ var uploadCmd = &cobra.Command{ }, } +var uploadProfileCmd = &cobra.Command{ + Use: "profile", +} + +var uploadPluginCmd = &cobra.Command{ + Use: "plugin", +} + func init() { + + uploadCmd.AddCommand(uploadProfileCmd, uploadPluginCmd) rootCmd.AddCommand(uploadCmd) } From 86f37555b27f4d78dfcc767a4f1db120eee1ff70 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:40:24 -0600 Subject: [PATCH 32/89] feat: added logging implementation --- pkg/log/log.go | 119 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 pkg/log/log.go diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..90ffa8d --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,119 @@ +package log + +import ( + "fmt" + "io" + "os" + "slices" + "strings" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +// string representation that directly corresponds to zerolog.Level +type LogLevel string +type LogLevelList []LogLevel +type LogFilter string + +const ( + DEBUG LogLevel = "debug" + INFO LogLevel = "info" + WARN LogLevel = "warn" + ERROR LogLevel = "error" + DISABLED LogLevel = "disabled" + TRACE LogLevel = "trace" +) + +var Levels = [6]LogLevel{DEBUG, INFO, WARN, ERROR, DISABLED, TRACE} +var LogFile *os.File + +func (ll LogLevel) String() string { + return string(ll) +} + +func (ll *LogLevel) Set(v string) error { + switch LogLevel(v) { + case DEBUG, INFO, WARN, ERROR, DISABLED, TRACE: + *ll = LogLevel(v) + return nil + default: + return fmt.Errorf("must be one of %v", []LogLevel{ + DEBUG, + INFO, + WARN, + ERROR, + DISABLED, + TRACE, + }) + } +} + +func (df LogLevel) Type() string { + return "LogLevel" +} + +func InitWithLogLevel(logLevel LogLevel, logPath string) error { + var ( + logger zerolog.Logger + level zerolog.Level + writer zerolog.LevelWriter + writers []io.Writer + err error + ) + + // set the logging level + level, err = strToLogLevel(logLevel) + if err != nil { + return fmt.Errorf("failed to convert log level: %v", err) + } + + // add the default stderr writer + writers = append(writers, &zerolog.FilteredLevelWriter{ + Writer: &zerolog.LevelWriterAdapter{os.Stderr}, + Level: level, + }) + + // add another writer to write to a log file + if logPath != "" { + LogFile, err = os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664) + if err != nil { + return fmt.Errorf("failed to open log file: %v", err) + } + + // add another write to write to the specified log file + writers = append(writers, &zerolog.FilteredLevelWriter{ + Writer: zerolog.LevelWriterAdapter{LogFile}, + Level: level, + }) + } + writer = zerolog.MultiLevelWriter(writers...) + logger = zerolog.New(writer).Level(level).With().Timestamp().Caller().Logger() + zerolog.SetGlobalLevel(level) + log.Logger = logger + return nil +} + +func strToLogLevel(ll LogLevel) (zerolog.Level, error) { + var tostr = func(lls []LogLevel) []string { + s := []string{} + for _, l := range lls { + s = append(s, string(l)) + } + return s + } + + if index := slices.Index(Levels[:], ll); index >= 0 { + // handle special cases to map index to DISABLED and TRACE + switch index { + case 4: + return zerolog.Disabled, nil + case 5: + return zerolog.TraceLevel, nil + } + return zerolog.Level(index), nil + } + return -100, fmt.Errorf( + "invalid log level (options: %s)", strings.Join(tostr(Levels[:]), ", "), + ) // use 'info' by default +} From 8b161135ff483d3fb3ce6f611f355d7d4c7b7904 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:40:53 -0600 Subject: [PATCH 33/89] feat: updated pkg implementations --- pkg/client/client.go | 2 +- pkg/service/constants.go | 57 +++++--- pkg/service/plugins.go | 38 +++++- pkg/service/profiles.go | 282 +++++++++++++++++++++++++++++++++++++++ pkg/service/routes.go | 162 ++++++++++++++++++++-- pkg/service/service.go | 71 +++++----- pkg/storage/disk.go | 16 ++- pkg/storage/memory.go | 17 ++- pkg/storage/storage.go | 6 + pkg/util/util.go | 68 ---------- 10 files changed, 579 insertions(+), 140 deletions(-) create mode 100644 pkg/service/profiles.go diff --git a/pkg/client/client.go b/pkg/client/client.go index d338fd7..f9cf5d9 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -8,7 +8,7 @@ import ( "os" "strings" - "git.towk2.me/towk/configurator/pkg/util" + "git.towk2.me/towk/makeshift/pkg/util" "github.com/cavaliergopher/grab/v3" ) diff --git a/pkg/service/constants.go b/pkg/service/constants.go index 52e990a..07e0ed2 100644 --- a/pkg/service/constants.go +++ b/pkg/service/constants.go @@ -4,48 +4,67 @@ const ( RELPATH_PLUGINS = "/plugins" RELPATH_PROFILES = "/profiles" RELPATH_DATA = "/data" - RELPATH_METADATA = "/.configurator" + RELPATH_METADATA = "/.makeshift" RELPATH_HELP = RELPATH_DATA + "/index.html" + RELPATH_PROFILE = RELPATH_PROFILES + "/default.json" + + PATH_CONFIG = "$HOME/.config/makeshift/config.yaml" DEFAULT_TIMEOUT_IN_SECS = 60 - DEFAULT_PLUGINS_MAX_COUNT = 64 + DEFAULT_PLUGINS_MAX_COUNT = 32 DEFAULT_PROFILES_MAX_COUNT = 256 - DEFAULT_METADATA = `` - DEFAULT_HOME = ` + + FILE_METADATA = `` + FILE_HOME_PAGE = `

# setup environment variables
- export CONFIGURATOR_HOST={{ configurator.host }}
- export CONFIGURATOR_PATH={{ configurator.path }}
- export CONFIGURATOR_SERVER_ROOT={{ configurator.server_root }}
+ export MAKESHIFT_HOST={{ makeshift.host }}
+ export MAKESHIFT_PATH={{ makeshift.path }}
+ export MAKESHIFT_SERVER_ROOT={{ makeshift.server.root }}

# start the service
- configurator serve --root $HOME/apps/configurator/server --init
+ makeshift serve --root $HOME/apps/makeshift/server --init

# download a file or directory (as archive)
- configurator download
- configurator download --host http://localhost:5050 --path help.txt
+ makeshift download
+ makeshift download --host http://localhost:5050 --path help.txt

# download files with rendering using plugins
- configurator download --plugins smd,jinja2 --profile compute
- curl $CONFIGURATOR_HOST/download/help.txt?plugins=smd,jinja2
+ makeshift download --plugins smd,jinja2 --profile compute
+ curl $MAKESHIFT_HOST/download/help.txt?plugins=smd,jinja2

# upload a file or directory (recursively)
- configurator upload
- configurator upload --host http://localhost:5050 --path help.txt
+ makeshift upload
+ makeshift upload --host http://localhost:5050 --path help.txt

# list the files in a directory
- configurator list --path help.txt
- configurator list --host http://localhost:5050 --path help.txt
+ makeshift list --path help.txt
+ makeshift list --host http://localhost:5050 --path help.txt
curl http://localhost:5050/list/test

+` + FILE_DEFAULT_PROFILE = ` +{ + "id": "default", + "description": "Makeshift default profile", + "data": { + "makeshift": { + "host": "localhost", + "path": "/test", + "server": { + "root": "/test" + } + } + } +} ` ) -// configurator.host: https://example.com -// configurator.path: test -// configurator.server_root: $HOME/apps/configurator +// makeshift.host: https://localhost:5050 +// makeshift.path: test +// makeshift.server.root: $HOME/apps/makeshift diff --git a/pkg/service/plugins.go b/pkg/service/plugins.go index c3a6c43..5b00216 100644 --- a/pkg/service/plugins.go +++ b/pkg/service/plugins.go @@ -5,14 +5,15 @@ import ( "io" "net/http" "os" + "strings" - configurator "git.towk2.me/towk/configurator/pkg" + makeshift "git.towk2.me/towk/makeshift/pkg" ) func (s *Service) ListPlugins() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - plugins map[string]configurator.Plugin + plugins map[string]makeshift.Plugin names []string body []byte err error @@ -38,10 +39,35 @@ func (s *Service) ListPlugins() http.HandlerFunc { } } +func (s *Service) GetPlugin() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + pluginName = strings.TrimPrefix(r.URL.Path, "/plugin") + plugin makeshift.Plugin + body []byte + err error + ) + + plugin, err = LoadPluginFromFile(pluginName) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + body, err = json.Marshal(plugin) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(body) + } +} + func (s *Service) CreatePlugin() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - plugin configurator.Plugin + plugin makeshift.Plugin path string err error ) @@ -79,7 +105,7 @@ func (s *Service) DeletePlugin() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( path string - plugin configurator.Plugin + plugin makeshift.Plugin err error ) @@ -100,9 +126,9 @@ func (s *Service) DeletePlugin() http.HandlerFunc { } } -func getPluginFromRequestBody(r *http.Request) (configurator.Plugin, error) { +func getPluginFromRequestBody(r *http.Request) (makeshift.Plugin, error) { var ( - plugin configurator.Plugin + plugin makeshift.Plugin body []byte err error ) diff --git a/pkg/service/profiles.go b/pkg/service/profiles.go new file mode 100644 index 0000000..08aeff7 --- /dev/null +++ b/pkg/service/profiles.go @@ -0,0 +1,282 @@ +package service + +import ( + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + + makeshift "git.towk2.me/towk/makeshift/pkg" + "github.com/go-chi/chi/v5" + "github.com/tidwall/sjson" +) + +func (s *Service) ListProfiles() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + path = s.RootPath + RELPATH_PROFILES + profiles []*makeshift.Profile + contents []byte + err error + ) + + // walk profiles directory to load all profiles + err = filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + // skip directories + if info.IsDir() { + return nil + } + + // read file contents + var profile *makeshift.Profile + profile, err = LoadProfileFromFile(path) + if err != nil { + return err + } + + profiles = append(profiles, profile) + + fmt.Println(path, info.Size()) + return nil + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + + // marshal and send all the profiles + contents, err = json.Marshal(profiles) + if err != nil { + http.Error(w, fmt.Sprintf("failed to marshal profiles: %v", err), http.StatusInternalServerError) + } + + _, err = w.Write(contents) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +// func (s *Service) CreateProfiles() http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) { +// var ( +// path = chi.URLParam(r, "path") +// err error +// ) + +// } +// } + +func (s *Service) GetProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + id = chi.URLParam(r, "id") + path = s.PathForProfileWithID(id) + contents []byte + err error + ) + + contents, err = loadProfileContents(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + + _, err = w.Write(contents) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +func (s *Service) CreateProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + type input struct { + Path string `json:"path"` + Profile *makeshift.Profile `json:"profile"` + } + var ( + body, contents []byte + in input + err error + ) + + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // use the request info to build profile + err = json.Unmarshal(body, &in) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // serialize just the profile part + contents, err = json.Marshal(in.Profile) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // create a new profile on disk + err = os.WriteFile(in.Path, contents, os.ModePerm) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + w.WriteHeader(http.StatusOK) + } +} + +func (s *Service) SetProfileData() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + body, contents []byte + newContents string + profile *makeshift.Profile + path string + err error + ) + + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + err = json.Unmarshal(body, &profile) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // make sure the request data sets an ID + if profile.ID == "" { + http.Error(w, "ID must be set to a non-empty value", http.StatusBadRequest) + return + } + + // read the contents the file with profile ID + path = s.PathForProfileWithID(profile.ID) + contents, err = os.ReadFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // modify the data of the profile's contents + newContents, err = sjson.Set(string(contents), "data", profile.Data) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // write only the data to the file with ID + err = os.WriteFile(path, []byte(newContents), os.ModePerm) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +func (s *Service) DeleteProfileData() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + id = chi.URLParam(r, "id") + path = s.PathForProfileWithID(id) + profile *makeshift.Profile + err error + ) + + // get the profile + profile, err = LoadProfileFromFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + + // delete the profile data + profile.Data = map[string]any{} + + // save the profile back to the file to update + SaveProfileToFile(path, profile) + + } +} + +func (s *Service) GetProfileData() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + id = chi.URLParam(r, "id") + path = s.PathForProfileWithID(id) + profile *makeshift.Profile + body []byte + err error + ) + + profile, err = LoadProfileFromFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // only marshal the profile data and not entire profile + body, err = json.Marshal(profile.Data) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // write body to response + _, err = w.Write(body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +// func (s *Service) CreateProfilePath() http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) { + +// w.WriteHeader(http.StatusOK) +// } +// } + +// func (s *Service) DeleteProfilePath() http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) {} +// } + +// func (s *Service) GetProfilePath() http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) {} +// } + +func loadProfileContents(path string) ([]byte, error) { + var ( + contents []byte + profile *makeshift.Profile + err error + ) + profile, err = LoadProfileFromFile(path) + if err != nil { + return nil, fmt.Errorf("failed to load profile from file: %v", err) + } + + contents, err = json.Marshal(profile) + if err != nil { + return nil, fmt.Errorf("failed to marshal profile: %v", err) + } + return contents, nil +} diff --git a/pkg/service/routes.go b/pkg/service/routes.go index eb6f6cc..375fbd0 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -10,33 +10,55 @@ import ( "strings" "time" - "git.towk2.me/towk/configurator/pkg/util" + "git.towk2.me/towk/makeshift/internal/archive" + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/storage" "github.com/rs/zerolog/log" ) func (s *Service) Download() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download") + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download") + pluginNames = r.URL.Query()["plugins"] + profileIDs = r.URL.Query()["profiles"] + fileInfo os.FileInfo out *os.File + store *storage.MemoryStorage = new(storage.MemoryStorage) + hooks []makeshift.Hook contents []byte + errs []error err error ) + // initialize storage + store.Init() + log.Debug(). Str("path", path). Str("client_host", r.Host). + Strs("plugins", pluginNames). + Strs("profiles", profileIDs). + Any("query", r.URL.Query()). Msg("Service.Download()") + // prepare profiles + errs = s.loadProfiles(profileIDs, store, errs) + if len(errs) > 0 { + log.Error().Errs("errs", errs).Msg("errors occurred loading profiles") + errs = []error{} + } + // determine if path is directory, file, or exists if fileInfo, err = os.Stat(path); err == nil { if fileInfo.IsDir() { - // create an archive of the directory and download + log.Debug(). Str("type", "directory"). Msg("Service.Download()") + // get the final archive path archivePath := fmt.Sprintf("%d.tar.gz", time.Now().Unix()) out, err = os.Create(archivePath) if err != nil { @@ -44,28 +66,40 @@ func (s *Service) Download() http.HandlerFunc { return } - filesToArchive := []string{} + // get a list of filenames to archive + filenamesToArchive := []string{} filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { if !d.IsDir() { - filesToArchive = append(filesToArchive, path) + filenamesToArchive = append(filenamesToArchive, path) } return nil }) - log.Debug().Strs("files", filesToArchive).Send() - err = util.CreateArchive(filesToArchive, out) + log.Debug().Strs("files", filenamesToArchive).Send() + + // prepare plugins + hooks, errs = s.loadPlugins(pluginNames, store, nil, errs) + if len(errs) > 0 { + log.Error().Errs("errs", errs).Msg("errors occurred loading plugins") + errs = []error{} + } + + // create an archive of the directory, run hooks, and download + err = archive.Create(filenamesToArchive, out, hooks) if err != nil { s.writeErrorResponse(w, fmt.Sprintf("failed to create archive: %v", err.Error()), http.StatusInternalServerError) return } + // load the final archive contents, err = os.ReadFile(archivePath) if err != nil { - s.writeErrorResponse(w, fmt.Sprintf("failed to read archive: %v", err.Error()), http.StatusInternalServerError) + s.writeErrorResponse(w, fmt.Sprintf("failed to read archive contents: %v", err.Error()), http.StatusInternalServerError) return } w.Write(contents) + // clean up the temporary archive err = os.Remove(archivePath) if err != nil { log.Error().Err(err).Msg("failed to remove temporary archive") @@ -77,12 +111,53 @@ func (s *Service) Download() http.HandlerFunc { log.Debug(). Str("type", "file"). Msg("Service.Download()") + contents, err = os.ReadFile(path) if err != nil { - s.writeErrorResponse(w, fmt.Sprintf("failed to read file to download: %v", err.Error()), http.StatusInternalServerError) + s.writeErrorResponse(w, fmt.Sprintf("failed to read file to download: %v", err), http.StatusInternalServerError) return } - w.Write(contents) + + // prepare plugins + + store.Set("file", contents) + hooks, errs = s.loadPlugins(pluginNames, store, nil, errs) + if len(errs) > 0 { + log.Error().Errs("errs", errs).Msg("errors occurred loading plugins") + errs = []error{} + } + + // run pre-hooks to modify the contents of the file before archiving + log.Debug().Int("hook_count", len(hooks)).Msg("running hooks") + for _, hook := range hooks { + log.Debug().Any("hook", map[string]any{ + "data": hook.Data, + "args": hook.Args, + "plugin": map[string]string{ + "name": hook.Plugin.Name(), + "description": hook.Plugin.Description(), + "version": hook.Plugin.Version(), + }, + }).Send() + err = hook.Run() + if err != nil { + log.Error().Err(err).Str("plugin", hook.Plugin.Name()).Msg("failed to run plugin") + continue + } + } + + // take the contents from the last hook and update files + var ( + hook = hooks[len(hooks)-1] + data any + ) + data, err = hook.Data.Get("out") + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to get data from hook: %v", err), http.StatusInternalServerError) + return + } + + w.Write([]byte(data.(string))) } } else { s.writeErrorResponse(w, err.Error(), http.StatusBadRequest) @@ -150,3 +225,70 @@ func (s *Service) GetStatus(w http.ResponseWriter, r *http.Request) { return } } + +func (s *Service) loadProfiles(profileIDs []string, store storage.KVStore, errs []error) []error { + // load data from profiles into the data store + for i, profileID := range profileIDs { + var ( + profilePath = s.PathForProfileWithID(profileID) + profile *makeshift.Profile + err error + ) + if i > DEFAULT_PROFILES_MAX_COUNT { + log.Warn().Msg("max profiles count reached...stopping") + return errs + } + if profileID == "" { + log.Warn().Msg("profile ID is empty...skipping") + continue + } + log.Debug(). + Str("id", profileID). + Str("path", profilePath). + Msg("load profile") + profile, err = LoadProfileFromFile(profilePath) + if err != nil { + errs = append(errs, err) + continue + } + store.Set(profileID, profile) + } + + return errs +} + +func (s *Service) loadPlugins(pluginNames []string, store storage.KVStore, args []string, errs []error) ([]makeshift.Hook, []error) { + // create hooks to run from provided plugins specified + var hooks []makeshift.Hook + for i, pluginName := range pluginNames { + var ( + pluginPath string = s.PathForPluginWithName(pluginName) + plugin makeshift.Plugin + err error + ) + if i > DEFAULT_PLUGINS_MAX_COUNT { + log.Warn().Msg("max plugins count reached...stopping") + return hooks, errs + } + if pluginName == "" { + log.Warn().Msg("plugin name is empty...skipping") + continue + } + log.Debug(). + Str("name", pluginName). + Str("path", pluginPath). + Msg("load plugin") + // load the plugin from disk + plugin, err = LoadPluginFromFile(pluginPath) + if err != nil { + errs = append(errs, err) + continue + } + hooks = append(hooks, makeshift.Hook{ + Data: store, + Args: args, + Plugin: plugin, + }) + } + return hooks, errs +} diff --git a/pkg/service/service.go b/pkg/service/service.go index 528cdac..26d66ea 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -11,10 +11,10 @@ import ( "slices" "time" - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/util" - "github.com/go-chi/chi/middleware" + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/util" "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" "github.com/rs/zerolog/log" ) @@ -35,9 +35,9 @@ func New() *Service { Addr: ":5050", RootPath: "./", Environment: map[string]string{ - "CONFIGURATOR_HOST": "", - "CONFIGURATOR_ROOT": "", - "ACCESS_TOKEN": "", + "MAKESHIFT_HOST": "", + "MAKESHIFT_ROOT": "", + "ACCESS_TOKEN": "", }, PluginsMaxCount: DEFAULT_PLUGINS_MAX_COUNT, ProfilesMaxCount: DEFAULT_PROFILES_MAX_COUNT, @@ -63,22 +63,27 @@ func (s *Service) Init() error { } err = os.MkdirAll(s.PathForData(), 0o777) if err != nil { - return fmt.Errorf("failed to make service profile path: %v", err) + return fmt.Errorf("failed to make service data path: %v", err) } // create the default files - err = os.WriteFile(s.PathForMetadata(), []byte(DEFAULT_METADATA), 0o777) + err = os.WriteFile(s.PathForMetadata(), []byte(FILE_METADATA), 0o777) if err != nil { return fmt.Errorf("failed to make service metadata file: %v", err) } - err = os.WriteFile(s.PathForHome(), []byte(DEFAULT_HOME), 0o777) + err = os.WriteFile(s.PathForHome(), []byte(FILE_HOME_PAGE), 0o777) if err != nil { - return fmt.Errorf("failed to make service metadata file: %v", err) + return fmt.Errorf("failed to make service home page file: %v", err) } + err = os.WriteFile(s.PathForProfileWithID("default"), []byte(FILE_DEFAULT_PROFILE), 0o777) + if err != nil { + return fmt.Errorf("failed to make service default profile file: %v", err) + } + return nil } -// Serve() starts the configurator service and waits for requests. +// Serve() starts the makeshift service and waits for requests. func (s *Service) Serve() error { router := chi.NewRouter() router.Use(middleware.RequestID) @@ -98,7 +103,7 @@ func (s *Service) Serve() error { router.Get("/list/*", s.List()) // profiles - router.Get("/profiles", s.GetProfiles()) + router.Get("/profiles", s.ListProfiles()) // router.Post("/profiles", s.CreateProfiles()) router.Get("/profile/{id}", s.GetProfile()) router.Post("/profile/{id}", s.CreateProfile()) @@ -111,8 +116,9 @@ func (s *Service) Serve() error { // plugins router.Get("/plugins", s.ListPlugins()) - router.Post("/plugins", s.CreatePlugin()) - router.Delete("/plugins/{id}", s.DeletePlugin()) + router.Get("/plugin/{name}", s.GetPlugin()) + router.Post("/plugin/{name}", s.CreatePlugin()) + router.Delete("/plugin/{name}", s.DeletePlugin()) } // always available public routes go here @@ -128,22 +134,23 @@ func (s *Service) FetchJwks(uri string) { } -func LoadProfileFromFile(path string) (*Profile, error) { - return loadFromJSONFile[Profile](path) +func LoadProfileFromFile(path string) (*makeshift.Profile, error) { + return loadFromJSONFile[makeshift.Profile](path) } // LoadPluginFromFile loads a single plugin given a single file path -func LoadPluginFromFile(path string) (configurator.Plugin, error) { +func LoadPluginFromFile(path string) (makeshift.Plugin, error) { var ( isDir bool err error loadedPlugin *plugin.Plugin ) // skip loading plugin if path is a directory with no error - if isDir, err = util.IsDirectory(path); err == nil && isDir { - return nil, nil - } else if err != nil { + isDir, err = util.IsDirectory(path) + if err != nil { return nil, fmt.Errorf("failed to test if plugin path is directory: %v", err) + } else if isDir { + return nil, fmt.Errorf("path is a directory") } // try and open the plugin @@ -153,15 +160,15 @@ func LoadPluginFromFile(path string) (configurator.Plugin, error) { } // load the "Target" symbol from plugin - symbol, err := loadedPlugin.Lookup("Target") + symbol, err := loadedPlugin.Lookup("Makeshift") if err != nil { return nil, fmt.Errorf("failed to look up symbol at path '%s': %v", path, err) } - // assert that the plugin is a valid configurator.Plugin - target, ok := symbol.(configurator.Plugin) + // assert that the plugin is a valid makeshift.Plugin + target, ok := symbol.(makeshift.Plugin) if !ok { - return nil, fmt.Errorf("failed to load the correct symbol type at path '%s'", path) + return nil, fmt.Errorf("failed to assert the correct symbol type at path '%s'", path) } return target, nil } @@ -170,10 +177,10 @@ func LoadPluginFromFile(path string) (configurator.Plugin, error) { // // Returns a map of plugins. Each plugin can be accessed by the name // returned by the plugin.GetName() implemented. -func LoadPluginsFromDir(dirpath string) (map[string]configurator.Plugin, error) { +func LoadPluginsFromDir(dirpath string) (map[string]makeshift.Plugin, error) { // check if verbose option is supplied var ( - cps = make(map[string]configurator.Plugin) + cps = make(map[string]makeshift.Plugin) err error ) @@ -213,11 +220,11 @@ func LoadPluginsFromDir(dirpath string) (map[string]configurator.Plugin, error) return cps, nil } -func SaveProfileToFile(path string, profile *Profile) error { +func SaveProfileToFile(path string, profile *makeshift.Profile) error { return saveToJSONFile(path, profile) } -func SavePluginToFile(path string, plugin *configurator.Plugin) error { +func SavePluginToFile(path string, plugin *makeshift.Plugin) error { return saveToJSONFile(path, plugin) } @@ -230,12 +237,12 @@ func loadFromJSONFile[T any](path string) (*T, error) { contents, err = os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("failed to read plugin file: %v", err) + return nil, fmt.Errorf("failed to read file: %v", err) } err = json.Unmarshal(contents, &res) if err != nil { - return nil, fmt.Errorf("failed to unmarshal plugin: %v", err) + return nil, fmt.Errorf("failed to unmarshal contents from JSON: %v", err) } return res, err @@ -259,11 +266,11 @@ func saveToJSONFile[T any](path string, data T) error { } func (s *Service) PathForProfileWithID(id string) string { - return s.RootPath + RELPATH_PROFILES + "/" + id + return s.RootPath + RELPATH_PROFILES + "/" + id + ".json" } func (s *Service) PathForPluginWithName(name string) string { - return s.RootPath + RELPATH_PLUGINS + "/" + name + return s.RootPath + RELPATH_PLUGINS + "/" + name + ".so" } func (s *Service) PathForProfiles() string { diff --git a/pkg/storage/disk.go b/pkg/storage/disk.go index 0a17cef..46d3f1b 100644 --- a/pkg/storage/disk.go +++ b/pkg/storage/disk.go @@ -2,10 +2,22 @@ package storage type DiskStorage struct{} -func (ds *DiskStorage) Read(k string) error { +func (ds DiskStorage) Init() error { return nil } -func (ms *DiskStorage) Write(k string, v any) error { +func (ds DiskStorage) Cleanup() error { + return nil +} + +func (ds DiskStorage) Get(k string) error { + return nil +} + +func (ds DiskStorage) Set(k string, v any) error { + return nil +} + +func (ds DiskStorage) GetData() any { return nil } diff --git a/pkg/storage/memory.go b/pkg/storage/memory.go index d3890c5..84d4612 100644 --- a/pkg/storage/memory.go +++ b/pkg/storage/memory.go @@ -6,7 +6,16 @@ type MemoryStorage struct { Data map[string]any } -func (ms *MemoryStorage) Read(k string) (any, error) { +func (ms *MemoryStorage) Init() error { + ms.Data = map[string]any{} + return nil +} + +func (ms *MemoryStorage) Cleanup() error { + return nil +} + +func (ms *MemoryStorage) Get(k string) (any, error) { v, ok := ms.Data[k] if ok { return v, nil @@ -14,7 +23,11 @@ func (ms *MemoryStorage) Read(k string) (any, error) { return nil, fmt.Errorf("value does not exist") } -func (ms *MemoryStorage) Write(k string, v any) error { +func (ms *MemoryStorage) Set(k string, v any) error { ms.Data[k] = v return nil } + +func (ms *MemoryStorage) GetData() any { + return ms.Data +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 28f437b..3bc9877 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,12 +1,18 @@ package storage type KVStore interface { + Init() error + Cleanup() error + Get(k string) (any, error) Set(k string, v any) error GetData() any } type KVStaticStore[T any] interface { + Init() error + Cleanup() error + Get(k string) (T, error) Set(k string, v T) error GetData() T diff --git a/pkg/util/util.go b/pkg/util/util.go index 2b120e0..fc53b67 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1,10 +1,8 @@ package util import ( - "archive/tar" "bytes" "cmp" - "compress/gzip" "crypto/tls" "fmt" "io" @@ -99,69 +97,3 @@ func CopyIf[T comparable](s []T, condition func(t T) bool) []T { } return f } - -func CreateArchive(files []string, buf io.Writer) error { - // Create new Writers for gzip and tar - // These writers are chained. Writing to the tar writer will - // write to the gzip writer which in turn will write to - // the "buf" writer - gw := gzip.NewWriter(buf) - defer gw.Close() - tw := tar.NewWriter(gw) - defer tw.Close() - - // Iterate over files and add them to the tar archive - for _, file := range files { - err := addToArchive(tw, file) - if err != nil { - return err - } - } - - return nil -} - -func addToArchive(tw *tar.Writer, filename string) error { - // open file to write to archive - file, err := os.Open(filename) - if err != nil { - return err - } - defer file.Close() - - // get FileInfo for file size, mode, etc. - info, err := file.Stat() - if err != nil { - return err - } - - // skip file if it's a directory - if info.IsDir() { - return nil - } - - // create a tar Header from the FileInfo data - header, err := tar.FileInfoHeader(info, info.Name()) - if err != nil { - return err - } - - // use full path as name (FileInfoHeader only takes the basename) to - // preserve directory structure - // see for more info: https://golang.org/src/archive/tar/common.go?#L626 - header.Name = filename - - // Write file header to the tar archive - err = tw.WriteHeader(header) - if err != nil { - return err - } - - // copy file content to tar archive - _, err = io.Copy(tw, file) - if err != nil { - return err - } - - return nil -} From eb126d5350fc1bb5a04c91e5158fe6d0d0510ad1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:41:34 -0600 Subject: [PATCH 34/89] feat: moved and updated archive utility --- internal/archive/archive.go | 103 ++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 internal/archive/archive.go diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..b173dda --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,103 @@ +package archive + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "strings" + + makeshift "git.towk2.me/towk/makeshift/pkg" +) + +func Create(filenames []string, buf io.Writer, hooks []makeshift.Hook) error { + // Create new Writers for gzip and tar + // These writers are chained. Writing to the tar writer will + // write to the gzip writer which in turn will write to + // the "buf" writer + gw := gzip.NewWriter(buf) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + + // Iterate over files and add them to the tar archive + for _, filename := range filenames { + err := addToArchive(tw, filename, hooks) + if err != nil { + return err + } + } + + return nil +} + +func Expand(path string) error { + return nil +} + +func addToArchive(tw *tar.Writer, filename string, hooks []makeshift.Hook) error { + var ( + hook makeshift.Hook + file *os.File + data any + err error + ) + // open file to write to archive + file, err = os.Open(filename) + if err != nil { + return err + } + defer file.Close() + + // run pre-hooks to modify the contents of the file + // before archiving using plugins + for _, hook := range hooks { + err = hook.Run() + if err != nil { + return err + } + } + + // get FileInfo for file size, mode, etc. + info, err := file.Stat() + if err != nil { + return err + } + + // skip file if it's a directory + if info.IsDir() { + return nil + } + + // create a tar Header from the FileInfo data + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return err + } + + // use full path as name (FileInfoHeader only takes the basename) to + // preserve directory structure + // see for more info: https://golang.org/src/archive/tar/common.go?#L626 + header.Name = filename + + // Write file header to the tar archive + err = tw.WriteHeader(header) + if err != nil { + return err + } + + // take the contents from the last hook and update files + hook = hooks[len(hooks)-1] + data, err = hook.Data.Get("out") + if err != nil { + return err + } + + // copy file content to tar archive + _, err = io.Copy(tw, strings.NewReader(data.(string))) + if err != nil { + return err + } + + return nil +} From 134a0dcac04820d51622c960fdbdbfde32738ee6 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:42:27 -0600 Subject: [PATCH 35/89] feat: added initial hurl tests --- tests/01-download-localhost.hurl | 11 +++++++++++ tests/02-upload-localhost.hurl | 8 ++++++++ tests/03-list-localhost.hurl | 2 ++ tests/04-profiles-localhost.hurl | 6 ++++++ tests/05-plugins-localhost.hurl | 6 ++++++ 5 files changed, 33 insertions(+) create mode 100644 tests/01-download-localhost.hurl create mode 100644 tests/02-upload-localhost.hurl create mode 100644 tests/03-list-localhost.hurl create mode 100644 tests/04-profiles-localhost.hurl create mode 100644 tests/05-plugins-localhost.hurl diff --git a/tests/01-download-localhost.hurl b/tests/01-download-localhost.hurl new file mode 100644 index 0000000..1c1d0c0 --- /dev/null +++ b/tests/01-download-localhost.hurl @@ -0,0 +1,11 @@ +# download single file +GET http://localhost:5050/downloads + +# download directory as archive +GET + +# download single file using plugins + +# download directory as archive using plugins + + diff --git a/tests/02-upload-localhost.hurl b/tests/02-upload-localhost.hurl new file mode 100644 index 0000000..ff0163a --- /dev/null +++ b/tests/02-upload-localhost.hurl @@ -0,0 +1,8 @@ +# upload a single new file +POST http://localhost:5050/upload + +# upload a new directory + +# upload a new plugin + +# upload a new profile \ No newline at end of file diff --git a/tests/03-list-localhost.hurl b/tests/03-list-localhost.hurl new file mode 100644 index 0000000..bfc2558 --- /dev/null +++ b/tests/03-list-localhost.hurl @@ -0,0 +1,2 @@ +GET http://localhost:5050/list +GET http://localhost:5050/status \ No newline at end of file diff --git a/tests/04-profiles-localhost.hurl b/tests/04-profiles-localhost.hurl new file mode 100644 index 0000000..82786b2 --- /dev/null +++ b/tests/04-profiles-localhost.hurl @@ -0,0 +1,6 @@ +GET http://localhost:5050/profiles +GET http://localhost:5050/profiles/test +POST http://localhost:5050/profiles/test +GET http://localhost:5050/profiles/test/data +POST http://localhost:5050/profiles/test/data +DELETE http://localhost:5050/profiles/test/data diff --git a/tests/05-plugins-localhost.hurl b/tests/05-plugins-localhost.hurl new file mode 100644 index 0000000..e52e383 --- /dev/null +++ b/tests/05-plugins-localhost.hurl @@ -0,0 +1,6 @@ +GET http://localhost:5050/plugins +GET http://localhost:5050/plugin/test +POST http://localhost:5050/plugins/test +DELETE http://localhost:5050/plugin/test + + From fbdaf218eb571e3f224692ff0da7d9b8c142a6b0 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 24 Aug 2025 20:44:52 -0600 Subject: [PATCH 36/89] chore: added .gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index a51dd2f..cc2fb4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ **configurator +**makeshift **.yaml **.yml **.so @@ -6,3 +7,7 @@ **.ignore **.tar.gz dist/ +tests/data +tests/downloads +tests/profiles +tests/plugins From 1ebea8cb73fe14d54c23ddc2c2fe18c5d9704f6d Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 26 Aug 2025 22:11:17 -0600 Subject: [PATCH 37/89] feat: added working rendering with plugins --- cmd/download.go | 10 ++++- pkg/models.go | 1 + pkg/plugins/jinja2/jinja2.go | 84 ++++++++++++++++++++++++------------ pkg/plugins/smd/smd.go | 27 ++++++++---- pkg/service/constants.go | 26 ++++++----- pkg/service/routes.go | 68 ++++++++++++++++------------- pkg/storage/memory.go | 4 +- 7 files changed, 139 insertions(+), 81 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index b431920..e8f43b6 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -112,8 +112,12 @@ var downloadCmd = cobra.Command{ }, } -var downloadProfileCmd = &cobra.Command{} -var downloadPluginCmd = &cobra.Command{} +var downloadProfileCmd = &cobra.Command{ + Use: "profile", +} +var downloadPluginCmd = &cobra.Command{ + Use: "plugin", +} func init() { downloadCmd.Flags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") @@ -122,5 +126,7 @@ func init() { downloadCmd.Flags().StringSlice("profiles", []string{}, "Set the profile to use to populate data store") downloadCmd.Flags().StringSlice("plugins", []string{}, "Set the plugins to run before downloading files") + downloadCmd.AddCommand(downloadProfileCmd, downloadPluginCmd) + rootCmd.AddCommand(&downloadCmd) } diff --git a/pkg/models.go b/pkg/models.go index 53a0103..c101203 100644 --- a/pkg/models.go +++ b/pkg/models.go @@ -2,6 +2,7 @@ package makeshift import "git.towk2.me/towk/makeshift/pkg/storage" +type ProfileMap map[string]*Profile type Profile struct { ID string `json:"id"` // profile ID Description string `json:"description,omitempty"` // profile description diff --git a/pkg/plugins/jinja2/jinja2.go b/pkg/plugins/jinja2/jinja2.go index 4b64a11..ff6b98d 100644 --- a/pkg/plugins/jinja2/jinja2.go +++ b/pkg/plugins/jinja2/jinja2.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" - configurator "git.towk2.me/towk/makeshift/pkg" + makeshift "git.towk2.me/towk/makeshift/pkg" "git.towk2.me/towk/makeshift/pkg/storage" "github.com/nikolalohinski/gonja/v2" "github.com/nikolalohinski/gonja/v2/exec" @@ -17,13 +17,15 @@ type Jinja2 struct{} func (p *Jinja2) Name() string { return "jinja2" } func (p *Jinja2) Version() string { return "v0.0.1-alpha" } func (p *Jinja2) Description() string { return "Renders Jinja 2 templates" } -func (p *Jinja2) Metadata() configurator.Metadata { - return configurator.Metadata{ - "author.name": "David J. Allen", - "author.email": "davidallendj@gmail.com", - "author.links": []string{ - "https://github.com/davidallendj", - "https://git.towk2.me/towk", +func (p *Jinja2) Metadata() makeshift.Metadata { + return makeshift.Metadata{ + "author": map[string]any{ + "name": "David J. Allen", + "email": "davidallendj@gmail.com", + "links": []string{ + "https://github.com/davidallendj", + "https://git.towk2.me/towk", + }, }, } } @@ -34,25 +36,32 @@ func (p *Jinja2) Init() error { return nil } -func (p *Jinja2) Run(data storage.KVStore, args []string) error { +func (p *Jinja2) Run(store storage.KVStore, args []string) error { // render the files using Jinja 2 from args var ( - rendered []string + mappings struct { + Data map[string]any `json:"data"` + } context *exec.Context template *exec.Template - mappings map[string]any - input any // must be a byte array + profiles any // makeshift.ProfileMap + input any // []byte output bytes.Buffer err error ) log.Debug(). Str("plugin", p.Name()). - Any("data", data). - // Bytes("input", input.([]byte)). + Any("store", store). + Strs("args", args). Int("arg_count", len(args)). - Msg("Run()") + Msg("(jinja2) Run()") - input, err = data.Get("file") + profiles, err = store.Get("profiles") + if err != nil { + return fmt.Errorf("(jinja2) failed to get profiles: %v", err) + } + + input, err = store.Get("file") if err != nil { return fmt.Errorf("(jinja2) failed to get input data: %v", err) } @@ -63,35 +72,56 @@ func (p *Jinja2) Run(data storage.KVStore, args []string) error { return fmt.Errorf("(jinja2) failed to get template from args: %v", err) } - // get mappings from shared data - shared, err := data.Get("shared") + // get mappings from shared data (optional) + shared, err := store.Get("shared") if err != nil { - return fmt.Errorf("(jinja2) failed to get data from store: %v", err) + log.Warn().Err(err).Msg("(jinja2) could not retrieve shared data") + } else { + err = json.Unmarshal(shared.([]byte), &mappings) + if err != nil { + return fmt.Errorf("(jinja2) failed to unmarshal mappings from shared data: %v", err) + } } - err = json.Unmarshal(shared.([]byte), &mappings) - if err != nil { - return fmt.Errorf("(jinja2) failed to unmarshal mappings from shared data: %v", err) + var ps = make(map[string]any) + for profileID, profile := range profiles.(makeshift.ProfileMap) { + ps[profileID] = map[string]any{ + "id": profile.ID, + "description": profile.Description, + "data": profile.Data, + } } - data.Set("mappings", mappings) + // inject profiles and plugin-specific mapping + mappings.Data = map[string]any{ + "makeshift": map[string]any{ + "profiles": ps, + "plugin": map[string]any{ + "name": p.Name(), + "version": p.Version(), + "description": p.Description(), + "metadata": p.Metadata(), + }, + }, + } + + log.Debug().Any("mappings", mappings).Send() // use the provided data in the store to render templates // NOTE: this may be changed to specifically use "shared" data instead - context = exec.NewContext(data.GetData().(map[string]any)) + context = exec.NewContext(mappings.Data) if err = template.Execute(&output, context); err != nil { // Prints: Hello Bob! return fmt.Errorf("(jinja2) failed to render template: %v", err) } - rendered = append(rendered, output.String()) // write render templates to data store output - data.Set("out", rendered) + store.Set("out", output.String()) return nil } func (p *Jinja2) Cleanup() error { // nothing to clean up - log.Debug().Str("plugin", p.Name()).Msg("jinja2.Cleanup()") + log.Debug().Str("plugin", p.Name()).Msg("(jinja2) Cleanup()") return nil } diff --git a/pkg/plugins/smd/smd.go b/pkg/plugins/smd/smd.go index cb263ab..859c8da 100644 --- a/pkg/plugins/smd/smd.go +++ b/pkg/plugins/smd/smd.go @@ -82,21 +82,23 @@ func (p *SmdClient) Version() string { return "v0.0.1-alpha" } func (p *SmdClient) Description() string { return "Fetchs data from SMD and writes to store" } func (p *SmdClient) Metadata() makeshift.Metadata { return makeshift.Metadata{ - "author.name": "David J. Allen", - "author.email": "davidallendj@gmail.com", - "author.links": []string{ - "https://github.com/davidallendj", - "https://git.towk2.me/towk", + "author": map[string]any{ + "name": "David J. Allen", + "email": "davidallendj@gmail.com", + "links": []string{ + "https://github.com/davidallendj", + "https://git.towk2.me/towk", + }, }, } } func (p *SmdClient) Init() error { - log.Debug().Str("plugin", p.Name()).Msg("smd.Init()") + log.Debug().Str("plugin", p.Name()).Msg("(smd) Init()") return nil } -func (p *SmdClient) Run(data storage.KVStore, args []string) error { +func (p *SmdClient) Run(store storage.KVStore, args []string) error { // set all the defaults for variables var ( client SmdClient @@ -104,6 +106,13 @@ func (p *SmdClient) Run(data storage.KVStore, args []string) error { err error ) + log.Debug(). + Str("plugin", p.Name()). + Strs("args", args). + Int("arg_count", len(args)). + Any("store", store). + Msg("(smd) Run()") + // if we have a client, try making the request for the ethernet interfaces err = client.FetchEthernetInterfaces() if err != nil { @@ -124,14 +133,14 @@ func (p *SmdClient) Run(data storage.KVStore, args []string) error { if err != nil { return fmt.Errorf("(smd) failed to marshal SMD client: %v") } - data.Set("shared", bytes) + store.Set("shared", bytes) // apply template substitutions and return output as byte array return nil } func (p *SmdClient) Cleanup() error { - log.Debug().Str("plugin", p.Name()).Msg("smd.Init()") + log.Debug().Str("plugin", p.Name()).Msg("(smd) Init()") return nil } diff --git a/pkg/service/constants.go b/pkg/service/constants.go index 07e0ed2..719a9e4 100644 --- a/pkg/service/constants.go +++ b/pkg/service/constants.go @@ -20,10 +20,20 @@ const (

+ Plugin Information: + Name: {{ makeshift.plugin.name }} + Version: {{ makeshift.plugin.version }} + Description: {{ makeshift.plugin.description }} + Metadata: {{ makeshift.plugin.metadata }} + + Profile Information: + ID: {{ makeshift.profiles.default.id }} + Description: {{ makeshift.profiles.default.description }} + # setup environment variables
- export MAKESHIFT_HOST={{ makeshift.host }}
- export MAKESHIFT_PATH={{ makeshift.path }}
- export MAKESHIFT_SERVER_ROOT={{ makeshift.server.root }}
+ export MAKESHIFT_HOST={{ makeshift.profiles.default.data.host }}
+ export MAKESHIFT_PATH={{ makeshift.profiles.default.data.path }}
+ export MAKESHIFT_SERVER_ROOT={{ makeshift.profiles.default.data.server_root }}

# start the service
makeshift serve --root $HOME/apps/makeshift/server --init
@@ -53,13 +63,9 @@ const ( "id": "default", "description": "Makeshift default profile", "data": { - "makeshift": { - "host": "localhost", - "path": "/test", - "server": { - "root": "/test" - } - } + "host": "localhost", + "path": "/test", + "server_root": "./test" } } ` diff --git a/pkg/service/routes.go b/pkg/service/routes.go index 375fbd0..28c9886 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -20,8 +20,8 @@ func (s *Service) Download() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download") - pluginNames = r.URL.Query()["plugins"] - profileIDs = r.URL.Query()["profiles"] + pluginNames = strings.Split(r.URL.Query().Get("plugins"), ",") + profileIDs = strings.Split(r.URL.Query().Get("profiles"), ",") fileInfo os.FileInfo out *os.File @@ -126,38 +126,42 @@ func (s *Service) Download() http.HandlerFunc { log.Error().Errs("errs", errs).Msg("errors occurred loading plugins") errs = []error{} } + if len(hooks) > 0 { - // run pre-hooks to modify the contents of the file before archiving - log.Debug().Int("hook_count", len(hooks)).Msg("running hooks") - for _, hook := range hooks { - log.Debug().Any("hook", map[string]any{ - "data": hook.Data, - "args": hook.Args, - "plugin": map[string]string{ - "name": hook.Plugin.Name(), - "description": hook.Plugin.Description(), - "version": hook.Plugin.Version(), - }, - }).Send() - err = hook.Run() - if err != nil { - log.Error().Err(err).Str("plugin", hook.Plugin.Name()).Msg("failed to run plugin") - continue + // run pre-hooks to modify the contents of the file before archiving + log.Debug().Int("hook_count", len(hooks)).Msg("running hooks") + for _, hook := range hooks { + log.Debug().Any("hook", map[string]any{ + "store": hook.Data, + "args": hook.Args, + "plugin": map[string]string{ + "name": hook.Plugin.Name(), + "description": hook.Plugin.Description(), + "version": hook.Plugin.Version(), + }, + }).Send() + err = hook.Run() + if err != nil { + log.Error().Err(err).Str("plugin", hook.Plugin.Name()).Msg("failed to run plugin") + continue + } } + + // take the contents from the last hook and update files + var ( + hook = hooks[len(hooks)-1] + data any + ) + data, err = hook.Data.Get("out") + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to get data from hook: %v", err), http.StatusInternalServerError) + return + } + w.Write([]byte(data.(string))) + } else { + w.Write(contents) } - // take the contents from the last hook and update files - var ( - hook = hooks[len(hooks)-1] - data any - ) - data, err = hook.Data.Get("out") - if err != nil { - s.writeErrorResponse(w, fmt.Sprintf("failed to get data from hook: %v", err), http.StatusInternalServerError) - return - } - - w.Write([]byte(data.(string))) } } else { s.writeErrorResponse(w, err.Error(), http.StatusBadRequest) @@ -228,6 +232,7 @@ func (s *Service) GetStatus(w http.ResponseWriter, r *http.Request) { func (s *Service) loadProfiles(profileIDs []string, store storage.KVStore, errs []error) []error { // load data from profiles into the data store + var profiles = make(makeshift.ProfileMap, len(profileIDs)) for i, profileID := range profileIDs { var ( profilePath = s.PathForProfileWithID(profileID) @@ -251,8 +256,9 @@ func (s *Service) loadProfiles(profileIDs []string, store storage.KVStore, errs errs = append(errs, err) continue } - store.Set(profileID, profile) + profiles[profileID] = profile } + store.Set("profiles", profiles) return errs } diff --git a/pkg/storage/memory.go b/pkg/storage/memory.go index 84d4612..e554fbd 100644 --- a/pkg/storage/memory.go +++ b/pkg/storage/memory.go @@ -3,7 +3,7 @@ package storage import "fmt" type MemoryStorage struct { - Data map[string]any + Data map[string]any `json:"data"` } func (ms *MemoryStorage) Init() error { @@ -20,7 +20,7 @@ func (ms *MemoryStorage) Get(k string) (any, error) { if ok { return v, nil } - return nil, fmt.Errorf("value does not exist") + return nil, fmt.Errorf("value '%s' does not exist", k) } func (ms *MemoryStorage) Set(k string, v any) error { From 98f9acad5d832ad7eef3977aee65a283bfd7904b Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 28 Aug 2025 12:25:45 -0600 Subject: [PATCH 38/89] feat: added downloading templated archives --- cmd/download.go | 6 +++- internal/archive/archive.go | 54 ++++++++++++++++++++++++------------ pkg/plugins/jinja2/jinja2.go | 2 +- pkg/plugins/smd/smd.go | 2 +- pkg/service/routes.go | 7 +++-- 5 files changed, 48 insertions(+), 23 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index e8f43b6..edf09df 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -92,7 +92,10 @@ var downloadCmd = cobra.Command{ } if res.StatusCode != http.StatusOK { log.Error(). - Int("status", res.StatusCode). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). Str("host", host). Str("path", path). Str("output", outputPath). @@ -125,6 +128,7 @@ func init() { downloadCmd.Flags().StringP("output", "o", "", "Set the output path to write files") downloadCmd.Flags().StringSlice("profiles", []string{}, "Set the profile to use to populate data store") downloadCmd.Flags().StringSlice("plugins", []string{}, "Set the plugins to run before downloading files") + downloadCmd.Flags().Bool("extract", false, "Set whether to extract archive locally after downloading") downloadCmd.AddCommand(downloadProfileCmd, downloadPluginCmd) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index b173dda..0e727b0 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -3,6 +3,7 @@ package archive import ( "archive/tar" "compress/gzip" + "fmt" "io" "os" "strings" @@ -37,27 +38,47 @@ func Expand(path string) error { func addToArchive(tw *tar.Writer, filename string, hooks []makeshift.Hook) error { var ( - hook makeshift.Hook - file *os.File - data any - err error + tempfile = fmt.Sprintf("%s.tmp", filename) + file *os.File + contents []byte + data any + err error ) - // open file to write to archive - file, err = os.Open(filename) - if err != nil { - return err - } - defer file.Close() - // run pre-hooks to modify the contents of the file // before archiving using plugins for _, hook := range hooks { + // set the file in the data store before running hook + contents, err = os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read '%s' to download: %v", filename, err) + } + hook.Data.Set("file", contents) + err = hook.Run() if err != nil { return err } + + // create temporary file to use to add to archive + hook = hooks[len(hooks)-1] + data, err = hook.Data.Get("out") + if err != nil { + return fmt.Errorf("failed to get output data from '%s' plugin: %v", hook.Plugin.Name(), err) + } + + err = os.WriteFile(tempfile, data.([]byte), 0o777) + if err != nil { + return fmt.Errorf("failed to write temporary file: %v", err) + } } + // open file to write to archive + file, err = os.Open(tempfile) + if err != nil { + return fmt.Errorf("failed to open temporary file: %v", err) + } + defer file.Close() + // get FileInfo for file size, mode, etc. info, err := file.Stat() if err != nil { @@ -72,7 +93,7 @@ func addToArchive(tw *tar.Writer, filename string, hooks []makeshift.Hook) error // create a tar Header from the FileInfo data header, err := tar.FileInfoHeader(info, info.Name()) if err != nil { - return err + return fmt.Errorf("failed to create FileInfoHeader: %v", err) } // use full path as name (FileInfoHeader only takes the basename) to @@ -86,15 +107,14 @@ func addToArchive(tw *tar.Writer, filename string, hooks []makeshift.Hook) error return err } - // take the contents from the last hook and update files - hook = hooks[len(hooks)-1] - data, err = hook.Data.Get("out") + // copy file content to tar archive + _, err = io.Copy(tw, strings.NewReader(string(data.([]byte)))) if err != nil { return err } - // copy file content to tar archive - _, err = io.Copy(tw, strings.NewReader(data.(string))) + // delete the temporary file since we're done with it + err = os.Remove(tempfile) if err != nil { return err } diff --git a/pkg/plugins/jinja2/jinja2.go b/pkg/plugins/jinja2/jinja2.go index ff6b98d..daf6b20 100644 --- a/pkg/plugins/jinja2/jinja2.go +++ b/pkg/plugins/jinja2/jinja2.go @@ -115,7 +115,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { } // write render templates to data store output - store.Set("out", output.String()) + store.Set("out", output.Bytes()) return nil } diff --git a/pkg/plugins/smd/smd.go b/pkg/plugins/smd/smd.go index 859c8da..0d177be 100644 --- a/pkg/plugins/smd/smd.go +++ b/pkg/plugins/smd/smd.go @@ -131,7 +131,7 @@ func (p *SmdClient) Run(store storage.KVStore, args []string) error { // write data back to shared data store to be used by other plugins bytes, err = json.Marshal(client) if err != nil { - return fmt.Errorf("(smd) failed to marshal SMD client: %v") + return fmt.Errorf("(smd) failed to marshal SMD client: %v", err) } store.Set("shared", bytes) diff --git a/pkg/service/routes.go b/pkg/service/routes.go index 28c9886..cd96218 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -53,13 +53,14 @@ func (s *Service) Download() http.HandlerFunc { // determine if path is directory, file, or exists if fileInfo, err = os.Stat(path); err == nil { if fileInfo.IsDir() { + // get the final archive path + archivePath := fmt.Sprintf("%d.tar.gz", time.Now().Unix()) log.Debug(). + Str("archive_path", archivePath). Str("type", "directory"). Msg("Service.Download()") - // get the final archive path - archivePath := fmt.Sprintf("%d.tar.gz", time.Now().Unix()) out, err = os.Create(archivePath) if err != nil { s.writeErrorResponse(w, fmt.Sprintf("failed to create named file: %v", err), http.StatusInternalServerError) @@ -157,7 +158,7 @@ func (s *Service) Download() http.HandlerFunc { s.writeErrorResponse(w, fmt.Sprintf("failed to get data from hook: %v", err), http.StatusInternalServerError) return } - w.Write([]byte(data.(string))) + w.Write(data.([]byte)) } else { w.Write(contents) } From 2536848541927dacc214dbde056f509f88205946 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 28 Aug 2025 18:20:40 -0600 Subject: [PATCH 39/89] refactor: changed how file/archives are downloaded and saved --- cmd/download.go | 62 +++++++++++++++++++++++++++++-------------- pkg/service/routes.go | 23 +++++++++++----- 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index edf09df..7d00d01 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -5,9 +5,7 @@ import ( "net/http" "net/url" "os" - "path/filepath" "strings" - "time" "git.towk2.me/towk/makeshift/pkg/client" "github.com/rs/zerolog/log" @@ -30,7 +28,7 @@ var downloadCmd = cobra.Command{ curl $MAKESHIFT_HOST/download/test?plugins=smd,jinja2&profile=test # download directory and extract it's contents automatically - makeshift download --extract + makeshift download -xr `, Short: "Download and modify files with plugins", PreRun: func(cmd *cobra.Command, args []string) { @@ -52,15 +50,6 @@ var downloadCmd = cobra.Command{ err error ) - // set output path to match path if empty - if outputPath == "" { - if path != "." || path != "" { - outputPath = filepath.Base(path) - } else { - outputPath = fmt.Sprintf("%d.file", time.Now().Unix()) - } - } - query = fmt.Sprintf("/download/%s?", path) if len(pluginNames) > 0 { query += "plugins=" + url.QueryEscape(strings.Join(pluginNames, ",")) @@ -102,33 +91,66 @@ var downloadCmd = cobra.Command{ Msg("response returned bad status") os.Exit(1) } - if outputPath != "" { + + // helper to write downloaded files + var writeFiles = func(path string, body []byte) { err = os.WriteFile(outputPath, body, 0o755) if err != nil { log.Error().Err(err).Msg("failed to write file(s) from download") os.Exit(1) } - } else { - fmt.Println(string(body)) } + // determine if output path is an archive or file + switch res.Header.Get("FILETYPE") { + case "archive": + // write archive to disk with or without '-o' specified + if outputPath == "" { + outputPath = fmt.Sprintf("%s.tar.gz", path) + writeFiles(outputPath, body) + log.Debug().Str("path", outputPath).Msg("wrote archive to pre-determined path") + } else { + writeFiles(outputPath, body) + log.Debug().Str("path", outputPath).Msg("wrote archive to specified path") + } + case "file": + // write to file if '-o' specified otherwise stdout + if outputPath != "" { + writeFiles(outputPath, body) + log.Debug().Str("path", outputPath).Msg("wrote file to specified path") + } else { + fmt.Println(string(body)) + } + } }, } var downloadProfileCmd = &cobra.Command{ - Use: "profile", + Use: "profile", + Short: "Download a profile", + Run: func(cmd *cobra.Command, args []string) { + + }, } + var downloadPluginCmd = &cobra.Command{ - Use: "plugin", + Use: "plugin", + Short: "Download a plugin", + Run: func(cmd *cobra.Command, args []string) { + + }, } func init() { downloadCmd.Flags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") downloadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") downloadCmd.Flags().StringP("output", "o", "", "Set the output path to write files") - downloadCmd.Flags().StringSlice("profiles", []string{}, "Set the profile to use to populate data store") - downloadCmd.Flags().StringSlice("plugins", []string{}, "Set the plugins to run before downloading files") - downloadCmd.Flags().Bool("extract", false, "Set whether to extract archive locally after downloading") + downloadCmd.Flags().StringSlice("profiles", []string{}, "Set the profile(s) to use to populate data store") + downloadCmd.Flags().StringSlice("plugins", []string{}, "Set the plugin(s) to run before downloading files") + downloadCmd.Flags().BoolP("extract", "x", false, "Set whether to extract archive locally after downloading") + downloadCmd.Flags().BoolP("remove-archive", "r", false, "Set whether to remove the archive after extracting (used with '--extract' flag)") + + downloadCmd.MarkFlagsRequiredTogether("remove-archive", "extract") downloadCmd.AddCommand(downloadProfileCmd, downloadPluginCmd) diff --git a/pkg/service/routes.go b/pkg/service/routes.go index cd96218..3d2db0b 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -8,7 +8,6 @@ import ( "os" "path/filepath" "strings" - "time" "git.towk2.me/towk/makeshift/internal/archive" makeshift "git.towk2.me/towk/makeshift/pkg" @@ -54,7 +53,7 @@ func (s *Service) Download() http.HandlerFunc { if fileInfo, err = os.Stat(path); err == nil { if fileInfo.IsDir() { // get the final archive path - archivePath := fmt.Sprintf("%d.tar.gz", time.Now().Unix()) + archivePath := fmt.Sprintf("%s.tar.gz", path) log.Debug(). Str("archive_path", archivePath). @@ -98,6 +97,8 @@ func (s *Service) Download() http.HandlerFunc { return } + // send the archive back as response + w.Header().Add("FILETYPE", "archive") w.Write(contents) // clean up the temporary archive @@ -128,7 +129,6 @@ func (s *Service) Download() http.HandlerFunc { errs = []error{} } if len(hooks) > 0 { - // run pre-hooks to modify the contents of the file before archiving log.Debug().Int("hook_count", len(hooks)).Msg("running hooks") for _, hook := range hooks { @@ -143,7 +143,10 @@ func (s *Service) Download() http.HandlerFunc { }).Send() err = hook.Run() if err != nil { - log.Error().Err(err).Str("plugin", hook.Plugin.Name()).Msg("failed to run plugin") + log.Error(). + Err(err). + Str("plugin", hook.Plugin.Name()). + Msg("failed to run plugin") continue } } @@ -158,8 +161,13 @@ func (s *Service) Download() http.HandlerFunc { s.writeErrorResponse(w, fmt.Sprintf("failed to get data from hook: %v", err), http.StatusInternalServerError) return } + + // send processed (with plugins) file back as response + w.Header().Add("FILETYPE", "file") w.Write(data.([]byte)) } else { + // send non-processed file back as response + w.Header().Add("FILETYPE", "file") w.Write(contents) } @@ -223,7 +231,7 @@ func (s *Service) GetStatus(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") err := json.NewEncoder(w).Encode(map[string]any{ "code": http.StatusOK, - "message": "Configurator is healthy", + "message": "The makeshift server is healthy", }) if err != nil { fmt.Printf("failed to encode JSON response body: %v\n", err) @@ -274,17 +282,18 @@ func (s *Service) loadPlugins(pluginNames []string, store storage.KVStore, args err error ) if i > DEFAULT_PLUGINS_MAX_COUNT { - log.Warn().Msg("max plugins count reached...stopping") + log.Warn().Msg("max plugins count reached or exceeded...stopping") return hooks, errs } if pluginName == "" { - log.Warn().Msg("plugin name is empty...skipping") + log.Warn().Msgf("no plugin name found with index %d...skipping", i) continue } log.Debug(). Str("name", pluginName). Str("path", pluginPath). Msg("load plugin") + // load the plugin from disk plugin, err = LoadPluginFromFile(pluginPath) if err != nil { From f897bc3ca56f3e6ff4a0bb2f538c0033047c092d Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 28 Aug 2025 19:43:53 -0600 Subject: [PATCH 40/89] fix: issue with downloading without hooks --- internal/archive/archive.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 0e727b0..5145afe 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -72,10 +72,14 @@ func addToArchive(tw *tar.Writer, filename string, hooks []makeshift.Hook) error } } - // open file to write to archive - file, err = os.Open(tempfile) + // use original file if no hooks to write archive + if len(hooks) == 0 { + file, err = os.Open(filename) + } else { + file, err = os.Open(tempfile) + } if err != nil { - return fmt.Errorf("failed to open temporary file: %v", err) + return fmt.Errorf("failed to open archive file: %v", err) } defer file.Close() @@ -108,15 +112,21 @@ func addToArchive(tw *tar.Writer, filename string, hooks []makeshift.Hook) error } // copy file content to tar archive - _, err = io.Copy(tw, strings.NewReader(string(data.([]byte)))) + if len(hooks) == 0 { + _, err = io.Copy(tw, file) + } else { + _, err = io.Copy(tw, strings.NewReader(string(data.([]byte)))) + } if err != nil { return err } // delete the temporary file since we're done with it - err = os.Remove(tempfile) - if err != nil { - return err + if len(hooks) != 0 { + err = os.Remove(tempfile) + if err != nil { + return err + } } return nil From c2d5be5eed00e6989ff9655619297b1c19cfc7bf Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 28 Aug 2025 19:45:17 -0600 Subject: [PATCH 41/89] refactor: changed default help.txt j2 vars --- pkg/service/constants.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/service/constants.go b/pkg/service/constants.go index 719a9e4..3767260 100644 --- a/pkg/service/constants.go +++ b/pkg/service/constants.go @@ -24,7 +24,7 @@ const ( Name: {{ makeshift.plugin.name }} Version: {{ makeshift.plugin.version }} Description: {{ makeshift.plugin.description }} - Metadata: {{ makeshift.plugin.metadata }} + Author: {{ makeshift.plugin.metadata.name }} ({{ makeshift.plugin.metadata.email }}) Profile Information: ID: {{ makeshift.profiles.default.id }} @@ -33,23 +33,24 @@ const ( # setup environment variables
export MAKESHIFT_HOST={{ makeshift.profiles.default.data.host }}
export MAKESHIFT_PATH={{ makeshift.profiles.default.data.path }}
- export MAKESHIFT_SERVER_ROOT={{ makeshift.profiles.default.data.server_root }}
-
+ export MAKESHIFT_SERVER_ROOT={{ makeshift.profiles.default.data.server_root }}

+ # start the service
- makeshift serve --root $HOME/apps/makeshift/server --init
-
+ makeshift serve --root ./tests --init -l debug

+ # download a file or directory (as archive)
makeshift download
- makeshift download --host http://localhost:5050 --path help.txt
-
+ makeshift download --host http://localhost:5050 --path help.txt

+ # download files with rendering using plugins
makeshift download --plugins smd,jinja2 --profile compute
- curl $MAKESHIFT_HOST/download/help.txt?plugins=smd,jinja2
-
+ makeshift download -p templates --plugins jinja --profile io
+ curl $MAKESHIFT_HOST/download/help.txt?plugins=smd,jinja2

+ # upload a file or directory (recursively)
makeshift upload
- makeshift upload --host http://localhost:5050 --path help.txt
-
+ makeshift upload --host http://localhost:5050 --path help.txt

+ # list the files in a directory
makeshift list --path help.txt
makeshift list --host http://localhost:5050 --path help.txt
From e5c1b59bc18b0f1c8198241c4778e0dce78d7d44 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 11:04:16 -0600 Subject: [PATCH 42/89] feat: allow expanding and remove archive after download --- cmd/download.go | 161 ++++++++++++++++++++++++++++++++---- internal/archive/archive.go | 81 +++++++++++++++++- 2 files changed, 223 insertions(+), 19 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index 7d00d01..3bc502f 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -5,8 +5,10 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" + "git.towk2.me/towk/makeshift/internal/archive" "git.towk2.me/towk/makeshift/pkg/client" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -37,11 +39,13 @@ var downloadCmd = cobra.Command{ }, Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") - path, _ = cmd.Flags().GetString("path") - outputPath, _ = cmd.Flags().GetString("output") - pluginNames, _ = cmd.Flags().GetStringSlice("plugins") - profileIDs, _ = cmd.Flags().GetStringSlice("profiles") + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + outputPath, _ = cmd.Flags().GetString("output") + pluginNames, _ = cmd.Flags().GetStringSlice("plugins") + profileIDs, _ = cmd.Flags().GetStringSlice("profiles") + extract, _ = cmd.Flags().GetBool("extract") + removeArchive, _ = cmd.Flags().GetBool("remove-archive") c = client.New(host) res *http.Response @@ -92,15 +96,6 @@ var downloadCmd = cobra.Command{ os.Exit(1) } - // helper to write downloaded files - var writeFiles = func(path string, body []byte) { - err = os.WriteFile(outputPath, body, 0o755) - if err != nil { - log.Error().Err(err).Msg("failed to write file(s) from download") - os.Exit(1) - } - } - // determine if output path is an archive or file switch res.Header.Get("FILETYPE") { case "archive": @@ -113,6 +108,36 @@ var downloadCmd = cobra.Command{ writeFiles(outputPath, body) log.Debug().Str("path", outputPath).Msg("wrote archive to specified path") } + + // extract files if '-x' flag is passed + if extract { + var ( + dir = filepath.Dir(outputPath) + base = strings.TrimSuffix(filepath.Base(outputPath), ".tar.gz") + ) + err = archive.Expand(outputPath, fmt.Sprintf("%s/%s", dir, base)) + if err != nil { + log.Error().Err(err). + Str("path", outputPath). + Msg("failed to expand archive") + os.Exit(1) + } + } + + // optionally, remove archive if '-r' flag is passed + // NOTE: this can only be used if `-x` flag is set + if removeArchive { + if !extract { + log.Warn().Msg("requires '-x/--extract' flag to be set to 'true'") + } else { + err = os.Remove(outputPath) + if err != nil { + log.Error().Err(err). + Str("path", outputPath). + Msg("failed to remove archive") + } + } + } case "file": // write to file if '-o' specified otherwise stdout if outputPath != "" { @@ -126,18 +151,111 @@ var downloadCmd = cobra.Command{ } var downloadProfileCmd = &cobra.Command{ - Use: "profile", + Use: "profile", + Example: ` + // download a profile + makeshift download profile default +`, + Args: cobra.ExactArgs(1), Short: "Download a profile", + PreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") + }, Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + outputPath, _ = cmd.Flags().GetString("output") + c = client.New(host) + res *http.Response + body []byte + query string + err error + ) + for _, profileID := range args { + query = fmt.Sprintf("/profile/{%s}", profileID) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) + } + if outputPath != "" { + writeFiles(outputPath, body) + } else { + fmt.Println(string(body)) + } + } }, } var downloadPluginCmd = &cobra.Command{ - Use: "plugin", + Use: "plugin", + Example: ` + // download a plugin + makeshift download plugin smd jinja2 +`, + Args: cobra.ExactArgs(1), Short: "Download a plugin", + PreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") + }, Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + outputPath, _ = cmd.Flags().GetString("output") + c = client.New(host) + res *http.Response + query string + body []byte + err error + ) + for _, pluginName := range args { + + query = fmt.Sprintf("/profile/%s?", pluginName) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) + } + if outputPath != "" { + writeFiles(outputPath, body) + } else { + writeFiles(fmt.Sprintf("%s.so", pluginName), body) + } + } }, } @@ -150,9 +268,16 @@ func init() { downloadCmd.Flags().BoolP("extract", "x", false, "Set whether to extract archive locally after downloading") downloadCmd.Flags().BoolP("remove-archive", "r", false, "Set whether to remove the archive after extracting (used with '--extract' flag)") - downloadCmd.MarkFlagsRequiredTogether("remove-archive", "extract") - downloadCmd.AddCommand(downloadProfileCmd, downloadPluginCmd) rootCmd.AddCommand(&downloadCmd) } + +// helper to write downloaded files +func writeFiles(path string, body []byte) { + var err = os.WriteFile(path, body, 0o755) + if err != nil { + log.Error().Err(err).Msg("failed to write file(s) from download") + os.Exit(1) + } +} diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 5145afe..eaf6412 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" makeshift "git.towk2.me/towk/makeshift/pkg" @@ -32,7 +33,85 @@ func Create(filenames []string, buf io.Writer, hooks []makeshift.Hook) error { return nil } -func Expand(path string) error { +func Expand(tarname, xpath string) error { + tarfile, err := os.Open(tarname) + if err != nil { + return err + } + defer tarfile.Close() + // absPath, err := filepath.Abs(xpath) + // if err != nil { + // return err + // } + tr := tar.NewReader(tarfile) + if strings.HasSuffix(tarname, ".gz") { + gz, err := gzip.NewReader(tarfile) + if err != nil { + return fmt.Errorf("failed to create new gzip reader: %v", err) + } + defer gz.Close() + tr = tar.NewReader(gz) + } + + // untar each segment + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to get next tar header: %v", err) + } + + // determine proper file path info + var ( + fileinfo = header.FileInfo() + filename = header.Name + file *os.File + abspath string + dirpath string + ) + + // absFileName := filepath.Join(absPath, filename) // if a dir, create it, then go to next segment + if fileinfo.Mode().IsDir() { + if err := os.MkdirAll(filename, 0o755); err != nil { + return fmt.Errorf("failed to make directory '%s': %v", filename, err) + } + continue + } + + dirpath = filepath.Dir(filename) + if err = os.MkdirAll(dirpath, 0o777); err != nil { + return fmt.Errorf("failed to make directory '%s': %v", err) + } + + // create new file with original file mode + abspath, err = filepath.Abs(filename) + if err != nil { + return fmt.Errorf("failed to get absolute path: %v", err) + } + file, err = os.OpenFile( + abspath, + os.O_RDWR|os.O_CREATE|os.O_TRUNC, + fileinfo.Mode().Perm(), + ) + if err != nil { + return fmt.Errorf("failed to open file: %v", err) + } + // fmt.Printf("x %s\n", filename) + + // copy the contents to the new file + n, err := io.Copy(file, tr) + if err != nil { + return fmt.Errorf("failed to copy file: %v", err) + } + if err = file.Close(); err != nil { + return fmt.Errorf("failed to close file: %v", err) + } + if n != fileinfo.Size() { + return fmt.Errorf("wrote %d, want %d", n, fileinfo.Size()) + } + } return nil } From 3244a66f8e38d79ab4267a5450065be33fd9b890 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 16:02:36 -0600 Subject: [PATCH 43/89] fix: issue with logging not initializing correctly --- cmd/download.go | 30 +++++++++++++++++------------- cmd/root.go | 20 +++++++++++++++++--- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index 3bc502f..5e87ea1 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -33,7 +33,7 @@ var downloadCmd = cobra.Command{ makeshift download -xr `, Short: "Download and modify files with plugins", - PreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRun: func(cmd *cobra.Command, args []string) { setenv(cmd, "host", "MAKESHIFT_HOST") setenv(cmd, "path", "MAKESHIFT_PATH") }, @@ -158,10 +158,9 @@ var downloadProfileCmd = &cobra.Command{ `, Args: cobra.ExactArgs(1), Short: "Download a profile", - PreRun: func(cmd *cobra.Command, args []string) { - setenv(cmd, "host", "MAKESHIFT_HOST") - setenv(cmd, "path", "MAKESHIFT_PATH") - }, + // PreRun: func(cmd *cobra.Command, args []string) { + // setenv(cmd, "host", "MAKESHIFT_HOST") + // }, Run: func(cmd *cobra.Command, args []string) { var ( host, _ = cmd.Flags().GetString("host") @@ -173,8 +172,14 @@ var downloadProfileCmd = &cobra.Command{ query string err error ) + + log.Debug(). + Str("host", host). + Str("output", outputPath). + Send() + for _, profileID := range args { - query = fmt.Sprintf("/profile/{%s}", profileID) + query = fmt.Sprintf("/profile/%s", profileID) res, body, err = c.MakeRequest(client.HTTPEnvelope{ Path: query, Method: http.MethodGet, @@ -212,10 +217,9 @@ var downloadPluginCmd = &cobra.Command{ `, Args: cobra.ExactArgs(1), Short: "Download a plugin", - PreRun: func(cmd *cobra.Command, args []string) { - setenv(cmd, "host", "MAKESHIFT_HOST") - setenv(cmd, "path", "MAKESHIFT_PATH") - }, + // PreRun: func(cmd *cobra.Command, args []string) { + // setenv(cmd, "host", "MAKESHIFT_HOST") + // }, Run: func(cmd *cobra.Command, args []string) { var ( host, _ = cmd.Flags().GetString("host") @@ -229,7 +233,7 @@ var downloadPluginCmd = &cobra.Command{ ) for _, pluginName := range args { - query = fmt.Sprintf("/profile/%s?", pluginName) + query = fmt.Sprintf("/profile/%s", pluginName) res, body, err = c.MakeRequest(client.HTTPEnvelope{ Path: query, Method: http.MethodGet, @@ -260,9 +264,9 @@ var downloadPluginCmd = &cobra.Command{ } func init() { - downloadCmd.Flags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + downloadCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + downloadCmd.PersistentFlags().StringP("output", "o", "", "Set the output path to write files") downloadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") - downloadCmd.Flags().StringP("output", "o", "", "Set the output path to write files") downloadCmd.Flags().StringSlice("profiles", []string{}, "Set the profile(s) to use to populate data store") downloadCmd.Flags().StringSlice("plugins", []string{}, "Set the plugin(s) to run before downloading files") downloadCmd.Flags().BoolP("extract", "x", false, "Set whether to extract archive locally after downloading") diff --git a/cmd/root.go b/cmd/root.go index 375f4bc..b068dcd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -18,7 +18,7 @@ var ( // plugins []string // timeout int // logFile string - logLevel logger.LogLevel = logger.INFO + loglevel logger.LogLevel = logger.INFO ) var rootCmd = cobra.Command{ @@ -32,7 +32,7 @@ var rootCmd = cobra.Command{ // initialize the logger logFile, _ = cmd.Flags().GetString("log-file") - err = logger.InitWithLogLevel(logLevel, logFile) + err = logger.InitWithLogLevel(loglevel, logFile) if err != nil { log.Error().Err(err).Msg("failed to initialize logger") os.Exit(1) @@ -67,8 +67,12 @@ func Execute() { } func init() { + cobra.OnInitialize( + initLogger, + // initializeConfig, + ) // initialize the config a single time - rootCmd.PersistentFlags().VarP(&logLevel, "log-level", "l", "Set the log level output") + rootCmd.PersistentFlags().VarP(&loglevel, "log-level", "l", "Set the log level output") rootCmd.PersistentFlags().String("log-file", "", "Set the log file path (can be set with MAKESHIFT_LOG_FILE)") } @@ -81,3 +85,13 @@ func setenv(cmd *cobra.Command, varname string, envvar string) { cmd.Flags().Set(varname, val) } } + +func initLogger() { + // initialize the logger + logfile, _ := rootCmd.PersistentFlags().GetString("log-file") + err := logger.InitWithLogLevel(loglevel, logfile) + if err != nil { + log.Error().Err(err).Msg("failed to initialize logger") + os.Exit(1) + } +} From df8730463e41461caf63e1ed9844d7e74106b717 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 16:07:57 -0600 Subject: [PATCH 44/89] cleanup: removed unused code/comments --- cmd/root.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index b068dcd..f9ac4bc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,14 +10,6 @@ import ( ) var ( - // host string - // path string - // outputPath string - // rootPath string - // profile string - // plugins []string - // timeout int - // logFile string loglevel logger.LogLevel = logger.INFO ) @@ -69,7 +61,6 @@ func Execute() { func init() { cobra.OnInitialize( initLogger, - // initializeConfig, ) // initialize the config a single time rootCmd.PersistentFlags().VarP(&loglevel, "log-level", "l", "Set the log level output") From 135245ca9cb76301831b8caab4c02666e63a7571 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 16:24:23 -0600 Subject: [PATCH 45/89] cleanup: removed unused code and formatting --- cmd/download.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index 5e87ea1..e36ee86 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -158,9 +158,6 @@ var downloadProfileCmd = &cobra.Command{ `, Args: cobra.ExactArgs(1), Short: "Download a profile", - // PreRun: func(cmd *cobra.Command, args []string) { - // setenv(cmd, "host", "MAKESHIFT_HOST") - // }, Run: func(cmd *cobra.Command, args []string) { var ( host, _ = cmd.Flags().GetString("host") @@ -217,9 +214,6 @@ var downloadPluginCmd = &cobra.Command{ `, Args: cobra.ExactArgs(1), Short: "Download a plugin", - // PreRun: func(cmd *cobra.Command, args []string) { - // setenv(cmd, "host", "MAKESHIFT_HOST") - // }, Run: func(cmd *cobra.Command, args []string) { var ( host, _ = cmd.Flags().GetString("host") @@ -231,9 +225,14 @@ var downloadPluginCmd = &cobra.Command{ body []byte err error ) - for _, pluginName := range args { - query = fmt.Sprintf("/profile/%s", pluginName) + log.Debug(). + Str("host", host). + Str("output", outputPath). + Send() + + for _, pluginName := range args { + query = fmt.Sprintf("/plugin/%s", pluginName) res, body, err = c.MakeRequest(client.HTTPEnvelope{ Path: query, Method: http.MethodGet, @@ -241,6 +240,7 @@ var downloadPluginCmd = &cobra.Command{ if err != nil { log.Error().Err(err). Str("host", host). + Str("query", query). Msg("failed to make request") os.Exit(1) } From f917d2b6f85a084d576161e2b39a49a79a64cc39 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 16:24:40 -0600 Subject: [PATCH 46/89] fix: issue with plugins not downloading --- pkg/service/plugins.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/service/plugins.go b/pkg/service/plugins.go index 5b00216..1575f39 100644 --- a/pkg/service/plugins.go +++ b/pkg/service/plugins.go @@ -5,9 +5,9 @@ import ( "io" "net/http" "os" - "strings" makeshift "git.towk2.me/towk/makeshift/pkg" + "github.com/go-chi/chi/v5" ) func (s *Service) ListPlugins() http.HandlerFunc { @@ -42,13 +42,14 @@ func (s *Service) ListPlugins() http.HandlerFunc { func (s *Service) GetPlugin() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - pluginName = strings.TrimPrefix(r.URL.Path, "/plugin") + pluginName = chi.URLParam(r, "name") + path = s.PathForPluginWithName(pluginName) plugin makeshift.Plugin body []byte err error ) - plugin, err = LoadPluginFromFile(pluginName) + plugin, err = LoadPluginFromFile(path) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return From d1e7b275a683ffa7e9b7c749db4cd94760d2b429 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 17:24:44 -0600 Subject: [PATCH 47/89] chore: updated README with makeshift info --- README.md | 320 +++++++++++++++++++++++++----------------------------- 1 file changed, 146 insertions(+), 174 deletions(-) diff --git a/README.md b/README.md index c7f620d..04c4f96 100644 --- a/README.md +++ b/README.md @@ -1,219 +1,191 @@ -# OpenCHAMI Configurator +# << Makeshift >> -The `configurator` is an extensible tool that is capable of dynamically generating files on the fly. The tool includes a built-in generator that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate files based on Jinja 2 template files. The tool and generator plugins are written in Go and plugins can be written by following the ["Creating Generator Plugins"](#creating-generator-plugins) section of this README. +The `makeshift` tool is a service that serves files and CLI that downloads them with a couple of handy features baked-in. Although the CLI and server component function more like a glorified FTP, the power of this tool comes from the plugin system. For example, the file cobbler is built to run external plugins for more advanced processing files before serving them (e.g. fetching from a data source, rendering Jinja 2 templates, etc.). -## Building and Usage -The `configurator` is built using standard `go` build tools. The project separates the client, server, and generator components using build tags. To get started, clone the project, download the dependencies, and build the project: +## Building the Tool + + +The `makeshift` tool is built using standard `go` build tools. To get started, clone the project, download the dependencies, and build the project: ```bash -git clone https://github.com/OpenCHAMI/configurator.git +git clone https://git.towk2.me/towk/makeshift.git go mod tidy -go build --tags all # equivalent to `go build --tags client,server`` +go build ``` This will build the main driver program with the default generators that are found in the `pkg/generators` directory. -> [!WARNING] -> Not all of the plugins have completed generation implementations and are a WIP. +> [!NOTE] +> The project does not current separate the client, server, and plugin components using build tags, but will eventually. This will allow users to only compile and distribute specific parts of the tool with limited functionality. -### Running Configurator with CLI -After you build the program, run the following command to use the tool: +## Basic Examples + +Here are some of the common commands you may want to try right off the bat (aside from `makeshift help` of course). The commands below that do not include the `--host`, `--path`, or `--root` flags are set using the environment variables. ```bash -export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -./configurator generate --config config.yaml --target coredhcp -o coredhcp.conf --cacert ochami.pem +export MAKESHIFT_HOST=localhost +export MAKESHIFT_PATH=/test +export MAKESHIFT_SERVER_ROOT=./test ``` -This will generate a new `coredhcp` config file based on the Jinja 2 template specified in the config file for "coredhcp". The files will be written to `coredhcp.conf` as specified with the `-o/--output` flag. The `--target` flag specifies the type of config file to generate by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). +Start the server. The `--init` flag with create the default files and directory to get started at the `--root` path. -In other words, there should be an entry in the config file that looks like this: +```bash +makeshift serve --root $HOME/apps/makeshift/server --init +``` -```yaml -... -targets: - coredhcp: - plugin: "lib/coredhcp.so" # optional, if we want to use an external plugin instead - templates: - - templates/coredhcp.j2 -... +From here, you might want to see what files are available by default. +```bash +# list the files in the root directory +makeshift list + +# list files store in the template directory with a specified host +makeshift list --host http://localhost:5050 --path templates +``` + +Then, we can start downloading some files or directories (as archives). + +```bash +# download all data (notice --host and --port are not set here) +makeshift download + +# download the 'help.txt' file without processing (i.e. using plugins) +makeshift download --host http://localhost:5050 --path help.txt + +# download files with rendering using Jinja 2 plugin and default profile +makeshift download -p help.txt --plugins jinja2 --profile default + +# download directory with rendering using plugins to fetch data and render +# using a custom 'compute' profile +makeshift download -p templates --plugins smd,jinja2 --profile compute + +# do everything in the above example but extract and remove archive +makeshift download -p templates --plugins smd,jinja2 --profile compute -xr ``` > [!NOTE] -> The `configurator` tool requires a valid access token when making requests to an instance of SMD that has protected routes. +> Plugins are ran in order specified with the `--plugins` flag, which means if you're creating a plugin to write to a data store and then read in a subsequent plugin, the order specified with the CLI matters! -### Running Configurator as a Service - -The tool can also run as a service to generate files for clients: +(WIP) Files, directories, profiles, and plugins will eventually be able to be uploaded to the server. ```bash -export MAKESHIFT_JWKS_URL="http://my.openchami.cluster:8443/key" -./configurator serve --config config.yaml -``` - -Once the server is up and listening for HTTP requests, you can try making a request to it with `curl` or `configurator fetch`. Both commands below are essentially equivalent: - -```bash -export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -curl http://127.0.0.1:3334/generate?target=coredhcp -X GET -H "Authorization: Bearer $ACCESS_TOKEN" --cacert ochami.pem -# ...or... -./configurator fetch --target coredhcp --host http://127.0.0.1:3334 --cacert ochami.pem -``` - -This will do the same thing as the `generate` subcommand, but through a GET request where the file contents is returned in the response. The access token is only required if the `MAKESHIFT_JWKS_URL` environment variable is set when starting the server with `serve`. The `ACCESS_TOKEN` environment variable is passed to `curl` using the `Authorization` header and expects a token as a JWT. - -### Docker - -New images can be built and tested using the `Dockerfile` provided in the project. However, the binary executable and the generator plugins must first be built before building the image since the Docker build copies the binary over. Therefore, build all of the binaries first by following the first section of ["Building and Usage"](#building-and-usage). Running `make docker` from the Makefile will automate this process. Otherwise, run the `docker build` command after building the executable and libraries. - -```bash -docker build -t configurator:testing path/to/configurator/Dockerfile -``` - -If you want to easily include your own external generator plugins, you can build it and copy the `lib.so` file to `lib/`. Make sure that the `Generator` interface is implemented correctly as described in the ["Creating Generator Plugins"](#creating-generator-plugins) or the plugin will not load (you should get an error that specifically says this). Additionally, the name string returned from the `GetName()` method is used for looking up the plugin with the `--target` flag by the main driver program. - -Alternatively, pull the latest existing image/container from the GitHub container repository. - -```bash -docker pull ghcr.io/openchami/configurator:latest -``` - -Then, run the Docker container similarly to running the binary. - -```bash -export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -docker run ghcr.io/openchami/configurator:latest configurator generate --config config.yaml --target coredhcp -o coredhcp.conf --cacert configurator.pem -``` - -### Creating Generator Plugins - -The `configurator` uses built-in and user-defined generators that implement the `Generator` interface to describe how config files should be generated. The interface is defined like so: - -```go -// maps the file path to its contents -type FileMap = map[string][]byte - -// interface for generator plugins -type Generator interface { - GetName() string - GetVersion() string - GetDescription() string - Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) -} -``` - -A new plugin can be created by implementing the methods from interface and exporting a symbol with `Generator` as the name and the plugin struct as the type. The `GetName()` function returns the name that is used for looking up the corresponding target set in your config file. It can also be included in the templated files with the default plugins using the `{{ plugin_name }}` in your template. The `GetVersion()` and `GetDescription()` functions returns the version and description of the plugin which can be included in the templated files using `{{ plugin_version }}` and `{{ plugin_description }}` respectively with the default plugins. The `Generate` function is where the magic happens to build the config file from a template. - -```go -package main - -type MyGenerator struct { - PluginInfo map[string]any -} - -var pluginInfo map[string]any - -// this function is not a part of the `Generator` interface -func (g *MyGenerator) LoadFromFile() map[string]any{ /*...*/ } - -func (g *MyGenerator) GetName() string { - // just an example...this can be done however you want - g.PluginInfo := LoadFromFile("path/to/plugin/info.json") - return g.PluginInfo["name"] -} - -func (g *MyGenerator) GetVersion() string { - return g.PluginInfo["version"] // "v1.0.0" -} - -func (g *MyGenerator) GetDescription() string { - return g.PluginInfo["description"] // "This is an example plugin." -} - -func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { - // do config generation stuff here... - var ( - params = generator.GetParams(opts...) - client = generator.GetClient(params) - output = "" - ) - if client { - eths, err := client.FetchEthernetInterfaces(opts...) - // ... blah, blah, blah, check error, format output, and so on... - - - // apply the substitutions to Jinja template and return output as FileMap (i.e. path and it's contents) - return generator.ApplyTemplate(path, generator.Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - "dhcp_hosts": output, - }) - } -} - -// this MUST be named "Generator" for symbol lookup in main driver -var Generator MyGenerator +# upload a file or directory (recursively)
+ makeshift upload
+ makeshift upload --host http://localhost:5050 --path help.txt
``` > [!NOTE] -> The keys in `generator.ApplyTemplate` must not contain illegal characters such as a `-` or else the templates will not apply correctly. +> Although every command has a `curl` equivalent, it is better to use the CLI since it has other features such as extracting and remove archives after downloading and saving archives as files automatically. -Finally, build the plugin and put it somewhere specified by `plugins` in your config. Make sure that the package is `main` before building. +## Creating Plugins -```bash -go build -buildmode=plugin -o lib/mygenerator.so path/to/mygenerator.go +The `makeshift` tool defines a plugin as an interface that can be implemented and compiled. + +```go +type Plugin interface { + Name() string + Version() string + Description() string + Metadata() Metadata + + Init() error + Run(data storage.KVStore, args []string) error + Cleanup() error +} ``` -Now your plugin should be available to use with the `configurator` main driver program. If you get an error about not loading the correct symbol type, make sure that your generator function definitions match the `Generator` interface entirely and that you don't have a partially implemented interface. +Plugins can *literally* contain whatever you want and is written in Go. Here is a simple example implementation to demonstrate how that is done which we will save at `src/example.go`. + +```go +type Example struct{} + +func (p *Example) Name() string { return "example" } +func (p *Example) Version() string { return "v0.0.1-alpha" } +func (p *Example) Description() string { return "An example plugin" } +func (p *Example) Metadata() map[string]string { + return makeshift.Metadata{ + "author": map[string]any{ + "name": "John Smith", + "email": "john.smith@example", + "links": []string{ + "https://example.com", + }, + }, + } +} + +func (p *Example) Init() error { + // Initialize the plugin if necessary. + return nil +} + +func (p *Example) Run(data storage.KVStore, args []string) error { + // Plugins can read and write to a data stores passed in. + // See the 'jinja2' plugin for reading and 'smd' plugin for writing. + return nil +} + +func (p *Example) Clean() error { + // Clean up resources if necessary. + return nil +} + +// This MUST be included to find the symbol in the main driver executable. +var Makeshift Example +``` + +Then, we can use the built-in `makeshift plugins compile` command to compile it. + +```bash +makeshift plugins compile src/example.go -o $MAKESHIFT_ROOT/plugins/example.so +``` > [!TIP] -> See the `examples/test.go` file for a plugin and template example. +> Make sure you move all of your plugins to `$MAKESHIFT_ROOT/plugins` to use them and should have an `*.so` name for lookup. For example, to use a custom plugin with `makeshift download -p templates/hosts.j2 --plugins my-plugin`, there has to a plugin `$MAKESHIFT_ROOT/plugins/my-plugin.so`. -## Configuration +## Creating Profiles -Here is an example config file to start using configurator: +On the other hand, profiles are simply objects that contain data used to populate data stores. The `makeshift` tool does not currently use all fields of a profile which will likely be removed in the near future. -```yaml -server: # Server-related parameters when using as service - host: 127.0.0.1 - port: 3334 - jwks: # Set the JWKS uri for protected routes - uri: "" - retries: 5 -smd: # SMD-related parameters - host: http://127.0.0.1:27779 -plugins: # path to plugin directories - - "lib/" -targets: # targets to call with --target flag - coredhcp: - templates: - - templates/coredhcp.j2 - files: # files to be copied without templating - - extra/nodes.conf - targets: # additional targets to run (does not run recursively) - - dnsmasq +```go +type Profile struct { + ID string `json:"id"` // profile ID + Description string `json:"description,omitempty"` // profile description + Tags []string `json:"tags,omitempty"` // tags used for ... + Paths []string `json:"paths,omitempty"` // paths to download + Plugins []string `json:"plugins,omitempty"` // plugins to run + Data map[string]any `json:"data,omitempty"` // include render data +} ``` -The `server` section sets the properties for running the `configurator` tool as a service and is not required if you're only using the CLI. Also note that the `jwks.uri` parameter is only needed for protecting endpoints. If it is not set, then all API routes are entirely public. The `smd` section tells the `configurator` tool where to find the SMD service to pull state management data used internally by the client's generator. The `templates` section is where the paths are mapped to each generator by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `plugins` is a list of paths to search for and load external generator plugins. +Profiles can be created using JSON. See the example in `$MAKESHIFT_ROOT/profiles/default.json`. -## Running the Tests - -The `configurator` project includes a collection of tests focused on verifying plugin behavior and generating files. The tests do not include fetching information from any remote sources, can be ran with the following command: - -```bash -go test ./tests/generate_test.go --tags=all +```json +{ + "id": "default", + "description": "Makeshift default profile", + "data": { + "host": "localhost", + "path": "/test", + "server_root": "./test" + } +} ``` -## Known Issues +> [!TIP] +> Make sure that you store your custom profiles in `$MAKESHIFT_ROOT/profiles` and that you set the name you want to use for lookup with a `*.json` extension (e.g. `compute.json`). -- Adds a new `OAuthClient` with every token request -- Plugins are being loaded each time a file is generated +## TODO: Missing Features -## TODO +There are some features still missing that will be added later. -- Add group functionality to create by files by groups -- Extend SMD client functionality (or make extensible?) -- Handle authentication with `OAuthClient`'s correctly +1. Uploading files and directories +2. Uploading new profiles and plugins +3. Running `makeshift` locally with profiles and plugins +4. Plugin to add user data for one-time use without creating a profile +5. Protected routes that require authentication +6. Configuration file for persistent runs +7. `Dockerfile` and `docker-compose.yml` files \ No newline at end of file From cdc441344fc55f08df6976ffe04163de7fbbf1cf Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 17:40:50 -0600 Subject: [PATCH 48/89] chore: fix formatting in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 04c4f96..0b4fd4d 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,8 @@ makeshift download -p templates --plugins smd,jinja2 --profile compute -xr ```bash # upload a file or directory (recursively)
- makeshift upload
- makeshift upload --host http://localhost:5050 --path help.txt
+makeshift upload +makeshift upload --host http://localhost:5050 --path help.txt ``` > [!NOTE] From b18746957d238e98bc1c9d502d09cb452eba5a92 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 17:58:16 -0600 Subject: [PATCH 49/89] refactor: removed unused code and fix typo --- README.md | 2 +- pkg/models.go | 2 -- pkg/service/service.go | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 0b4fd4d..94d4e78 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ makeshift download -p templates --plugins smd,jinja2 --profile compute -xr (WIP) Files, directories, profiles, and plugins will eventually be able to be uploaded to the server. ```bash -# upload a file or directory (recursively)
+# upload a file or directory (recursively) makeshift upload makeshift upload --host http://localhost:5050 --path help.txt ``` diff --git a/pkg/models.go b/pkg/models.go index c101203..b17b6b2 100644 --- a/pkg/models.go +++ b/pkg/models.go @@ -7,8 +7,6 @@ type Profile struct { ID string `json:"id"` // profile ID Description string `json:"description,omitempty"` // profile description Tags []string `json:"tags,omitempty"` // tags used for ... - Paths []string `json:"paths,omitempty"` // paths to download - Plugins []string `json:"plugins,omitempty"` // plugins to run Data map[string]any `json:"data,omitempty"` // include render data } diff --git a/pkg/service/service.go b/pkg/service/service.go index 26d66ea..95f0c92 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -110,9 +110,6 @@ func (s *Service) Serve() error { router.Get("/profile/{id}/data", s.GetProfileData()) router.Post("/profile/{id}/data", s.SetProfileData()) router.Delete("/profile/{id}/data", s.DeleteProfileData()) - // router.Post("/profile/{id}/paths/{path}", s.CreateProfilePath()) - // router.Delete("/profile/{id}/paths/{path}", s.DeleteProfilePath()) - // router.Get("/profile/{id}/paths/{path}", s.GetProfilePath()) // plugins router.Get("/plugins", s.ListPlugins()) From 08a9b9bdcf1404f29d8842aa496d14b07bb5eb06 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 18:23:00 -0600 Subject: [PATCH 50/89] refactor: removed unused code and added routes --- pkg/service/profiles.go | 15 --------------- pkg/service/service.go | 4 +++- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/pkg/service/profiles.go b/pkg/service/profiles.go index 08aeff7..ee7a0d3 100644 --- a/pkg/service/profiles.go +++ b/pkg/service/profiles.go @@ -248,21 +248,6 @@ func (s *Service) GetProfileData() http.HandlerFunc { } } -// func (s *Service) CreateProfilePath() http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) { - -// w.WriteHeader(http.StatusOK) -// } -// } - -// func (s *Service) DeleteProfilePath() http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) {} -// } - -// func (s *Service) GetProfilePath() http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) {} -// } - func loadProfileContents(path string) ([]byte, error) { var ( contents []byte diff --git a/pkg/service/service.go b/pkg/service/service.go index 95f0c92..d6c642f 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -99,7 +99,9 @@ func (s *Service) Serve() error { // general // router.Handle("/download/*", http.StripPrefix("/download/", http.FileServer(http.Dir(s.PathForData())))) router.Get("/download/*", s.Download()) - router.Post("/upload", s.Upload()) + router.Post("/upload/", s.Upload()) + router.Post("/upload/plugin", s.Upload()) + router.Post("/upload/profile", s.Upload()) router.Get("/list/*", s.List()) // profiles From dc6818e1e29fdeb3b1067913fdbb398abeaafb58 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 19:26:13 -0600 Subject: [PATCH 51/89] chore: updated go deps --- go.mod | 3 +++ go.sum | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/go.mod b/go.mod index 4090b3f..4b47295 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,9 @@ require ( github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.8.0 github.com/tidwall/sjson v1.2.5 + go.yaml.in/yaml/v3 v3.0.4 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -22,6 +24,7 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect diff --git a/go.sum b/go.sum index 3c1e2f4..91a82d0 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIH github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -31,6 +32,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= @@ -59,10 +64,13 @@ github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -92,6 +100,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= @@ -109,6 +119,8 @@ golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From e2b400fb12a6399fca8173c1494d5d8bda067c51 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 19:26:50 -0600 Subject: [PATCH 52/89] refactor: added route impls and minor changes --- pkg/service/routes.go | 12 ++++++++++++ pkg/service/service.go | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/service/routes.go b/pkg/service/routes.go index 3d2db0b..42e7338 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -185,6 +185,18 @@ func (s *Service) Upload() http.HandlerFunc { } } +func (s *Service) UploadPlugin() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + } +} + +func (s *Service) UploadProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + } +} + func (s *Service) List() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( diff --git a/pkg/service/service.go b/pkg/service/service.go index d6c642f..c69d68f 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -100,8 +100,8 @@ func (s *Service) Serve() error { // router.Handle("/download/*", http.StripPrefix("/download/", http.FileServer(http.Dir(s.PathForData())))) router.Get("/download/*", s.Download()) router.Post("/upload/", s.Upload()) - router.Post("/upload/plugin", s.Upload()) - router.Post("/upload/profile", s.Upload()) + router.Post("/upload/plugin", s.UploadPlugin()) + router.Post("/upload/profile", s.UploadProfile()) router.Get("/list/*", s.List()) // profiles From ac36201f075b12574617aff167340f636fc86f25 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 19:27:06 -0600 Subject: [PATCH 53/89] feat: initial upload cmd implementation --- cmd/download.go | 1 + cmd/upload.go | 214 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 213 insertions(+), 2 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index e36ee86..10c860f 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -30,6 +30,7 @@ var downloadCmd = cobra.Command{ curl $MAKESHIFT_HOST/download/test?plugins=smd,jinja2&profile=test # download directory and extract it's contents automatically + # then, remove the downloaded archive makeshift download -xr `, Short: "Download and modify files with plugins", diff --git a/cmd/upload.go b/cmd/upload.go index 39aa49f..12b58b1 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -1,24 +1,234 @@ package cmd -import "github.com/spf13/cobra" +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strings" + + "git.towk2.me/towk/makeshift/internal/format" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var ( + inputFormat format.DataFormat = format.JSON + dataArgs []string +) var uploadCmd = &cobra.Command{ Use: "upload", - Run: func(cmd *cobra.Command, args []string) { + Example: ` + # upload a single file + makeshift upload -d @compute-base.yaml -t file + # upload a single file with contents without specify type + makeshift upload -d '{"name": "John Smith", "email": "john.smith@example.com"}' + + # upload a directory + makeshift upload -d @setup/ -t directory + + # upload an archive (extracted and saved on server) + makeshift upload -d @setup.tar.gz -t archive +`, + Run: func(cmd *cobra.Command, args []string) { + // make one request be host positional argument (restricted to 1 for now) + var inputData []map[string]any + temp := append(handleArgs(args), processDataArgs(dataArgs)...) + for _, data := range temp { + if data != nil { + inputData = append(inputData, data) + } + } }, } var uploadProfileCmd = &cobra.Command{ Use: "profile", + Example: ` + # upload a new profile + makeshift upload profile -d @compute.json + + # upload a new profile with a specific name (used for lookups) + makeshift upload profile -d @kubernetes.json -n k8s +`, + Args: cobra.ExactArgs(1), + Short: "Upload a new profile", + Run: func(cmd *cobra.Command, args []string) { + + // make one request be host positional argument (restricted to 1 for now) + var inputData []map[string]any + temp := append(handleArgs(args), processDataArgs(dataArgs)...) + for _, data := range temp { + if data != nil { + inputData = append(inputData, data) + } + } + + }, } var uploadPluginCmd = &cobra.Command{ Use: "plugin", + Example: ` + # upload a new plugin + makeshift upload plugin -d @slurm.so + + # upload a new plugin with a specific name (used for lookups) + makeshift upload plugin -d @cobbler.so -n merge +`, + Args: cobra.ExactArgs(1), + Short: "Upload a new plugin", + Run: func(cmd *cobra.Command, args []string) { + // make one request be host positional argument (restricted to 1 for now) + var inputData []map[string]any + temp := append(handleArgs(args), processDataArgs(dataArgs)...) + for _, data := range temp { + if data != nil { + inputData = append(inputData, data) + } + } + }, } func init() { + uploadProfileCmd.Flags().VarP(&inputFormat, "format", "F", "Set the input format for profile") uploadCmd.AddCommand(uploadProfileCmd, uploadPluginCmd) rootCmd.AddCommand(uploadCmd) } + +// processDataArgs takes a slice of strings that check for the @ symbol and loads +// the contents from the file specified in place (which replaces the path). +// +// NOTE: The purpose is to make the input arguments uniform for our request. This +// function is meant to handle data passed with the `-d/--data` flag and positional +// args from the CLI. +func processDataArgs(args []string) []map[string]any { + // JSON representation + type ( + JSONObject = map[string]any + JSONArray = []JSONObject + ) + + // load data either from file or directly from args + var collection = make(JSONArray, len(args)) + for i, arg := range args { + // if arg is empty string, then skip and continue + if len(arg) > 0 { + // determine if we're reading from file to load contents + if strings.HasPrefix(arg, "@") { + var ( + path string = strings.TrimLeft(arg, "@") + contents []byte + data JSONArray + err error + ) + contents, err = os.ReadFile(path) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("failed to read file") + continue + } + + // skip empty files + if len(contents) == 0 { + log.Warn().Str("path", path).Msg("file is empty") + continue + } + + // convert/validate input data + data, err = parseInput(contents, format.DataFormatFromFileExt(path, inputFormat)) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("failed to validate input from file") + } + + // add loaded data to collection of all data + collection = append(collection, data...) + } else { + // input should be a valid JSON + var ( + data JSONArray + input = []byte(arg) + err error + ) + if !json.Valid(input) { + log.Error().Msgf("argument %d not a valid JSON", i) + continue + } + err = json.Unmarshal(input, &data) + if err != nil { + log.Error().Err(err).Msgf("failed to unmarshal input for argument %d", i) + } + return data + } + } + } + return collection +} + +func handleArgs(args []string) []map[string]any { + // JSON representation + type ( + JSONObject = map[string]any + JSONArray = []JSONObject + ) + // no file to load, so we just use the joined args (since each one is a new line) + // and then stop + var ( + collection JSONArray + data []byte + err error + ) + + if len(dataArgs) > 0 { + return nil + } + data, err = ReadStdin() + if err != nil { + log.Error().Err(err).Msg("failed to read from standard input") + return nil + } + if len(data) == 0 { + log.Warn().Msg("no data found from standard input") + return nil + } + fmt.Println(string(data)) + collection, err = parseInput([]byte(data), inputFormat) + if err != nil { + log.Error().Err(err).Msg("failed to validate input from arg") + } + return collection +} + +func parseInput(contents []byte, dataFormat format.DataFormat) ([]map[string]any, error) { + var ( + data []map[string]any + err error + ) + + // convert/validate JSON input format + err = format.Unmarshal(contents, &data, dataFormat) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + return data, nil +} + +// ReadStdin reads all of standard input and returns the bytes. If an error +// occurs during scanning, it is returned. +func ReadStdin() ([]byte, error) { + var b []byte + input := bufio.NewScanner(os.Stdin) + for input.Scan() { + b = append(b, input.Bytes()...) + b = append(b, byte('\n')) + if len(b) == 0 { + break + } + } + if err := input.Err(); err != nil { + return b, fmt.Errorf("failed to read stdin: %w", err) + } + return b, nil +} From 036bda61b9e49209289029f9d8398dde23a23908 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 19:27:26 -0600 Subject: [PATCH 54/89] feat: added format package for files --- internal/format/format.go | 104 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 internal/format/format.go diff --git a/internal/format/format.go b/internal/format/format.go new file mode 100644 index 0000000..d38aa3e --- /dev/null +++ b/internal/format/format.go @@ -0,0 +1,104 @@ +package format + +import ( + "encoding/json" + "fmt" + "path/filepath" + + "gopkg.in/yaml.v3" +) + +type DataFormat string + +const ( + List DataFormat = "list" + JSON DataFormat = "json" + FORMAT_YAML DataFormat = "yaml" +) + +func (df DataFormat) String() string { + return string(df) +} + +func (df *DataFormat) Set(v string) error { + switch DataFormat(v) { + case List, JSON, FORMAT_YAML: + *df = DataFormat(v) + return nil + default: + return fmt.Errorf("must be one of %v", []DataFormat{ + List, JSON, FORMAT_YAML, + }) + } +} + +func (df DataFormat) Type() string { + return "DataFormat" +} + +// MarshalData marshals arbitrary data into a byte slice formatted as outFormat. +// If a marshalling error occurs or outFormat is unknown, an error is returned. +// +// Supported values are: json, list, yaml +func Marshal(data interface{}, outFormat DataFormat) ([]byte, error) { + switch outFormat { + case JSON: + if bytes, err := json.MarshalIndent(data, "", " "); err != nil { + return nil, fmt.Errorf("failed to marshal data into JSON: %w", err) + } else { + return bytes, nil + } + case FORMAT_YAML: + if bytes, err := yaml.Marshal(data); err != nil { + return nil, fmt.Errorf("failed to marshal data into YAML: %w", err) + } else { + return bytes, nil + } + case List: + return nil, fmt.Errorf("this data format cannot be marshaled") + default: + return nil, fmt.Errorf("unknown data format: %s", outFormat) + } +} + +// UnmarshalData unmarshals a byte slice formatted as inFormat into an interface +// v. If an unmarshalling error occurs or inFormat is unknown, an error is +// returned. +// +// Supported values are: json, list, yaml +func Unmarshal(data []byte, v interface{}, inFormat DataFormat) error { + switch inFormat { + case JSON: + if err := json.Unmarshal(data, v); err != nil { + return fmt.Errorf("failed to unmarshal data into JSON: %w", err) + } + case FORMAT_YAML: + if err := yaml.Unmarshal(data, v); err != nil { + return fmt.Errorf("failed to unmarshal data into YAML: %w", err) + } + case List: + return fmt.Errorf("this data format cannot be unmarshaled") + default: + return fmt.Errorf("unknown data format: %s", inFormat) + } + + return nil +} + +// DataFormatFromFileExt determines the type of the contents +// (JSON or YAML) based on the filname extension. The default +// format is passed in, so if it doesn't match one of the cases, +// that's what we will use. The defaultFmt value takes into account +// both the standard default format (JSON) and any command line +// change to that provided by options. +func DataFormatFromFileExt(path string, defaultFmt DataFormat) DataFormat { + switch filepath.Ext(path) { + case ".json", ".JSON": + // The file is a JSON file + return JSON + case ".yaml", ".yml", ".YAML", ".YML": + // The file is a YAML file + return FORMAT_YAML + } + return defaultFmt +} From a4d1de9a51d4b15adfaaf5d5523b2e37f4f05f89 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 19:39:09 -0600 Subject: [PATCH 55/89] refactor: change MAKESHIFT_SERVER_ROOT to MAKESHIFT_ROOT --- README.md | 2 +- cmd/serve.go | 2 +- pkg/service/constants.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 94d4e78..1f5c2d1 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Here are some of the common commands you may want to try right off the bat (asid ```bash export MAKESHIFT_HOST=localhost export MAKESHIFT_PATH=/test -export MAKESHIFT_SERVER_ROOT=./test +export MAKESHIFT_ROOT=./test ``` Start the server. The `--init` flag with create the default files and directory to get started at the `--root` path. diff --git a/cmd/serve.go b/cmd/serve.go index 252a085..b8a34be 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -21,7 +21,7 @@ var serveCmd = &cobra.Command{ Args: cobra.NoArgs, PreRun: func(cmd *cobra.Command, args []string) { setenv(cmd, "host", "MAKESHIFT_HOST") - setenv(cmd, "root", "MAKESHIFT_SERVER_ROOT") + setenv(cmd, "root", "MAKESHIFT_ROOT") setenv(cmd, "timeout", "MAKESHIFT_TIMEOUT") }, Run: func(cmd *cobra.Command, args []string) { diff --git a/pkg/service/constants.go b/pkg/service/constants.go index 3767260..68c2953 100644 --- a/pkg/service/constants.go +++ b/pkg/service/constants.go @@ -33,7 +33,7 @@ const ( # setup environment variables
export MAKESHIFT_HOST={{ makeshift.profiles.default.data.host }}
export MAKESHIFT_PATH={{ makeshift.profiles.default.data.path }}
- export MAKESHIFT_SERVER_ROOT={{ makeshift.profiles.default.data.server_root }}

+ export MAKESHIFT_ROOT={{ makeshift.profiles.default.data.server_root }}

# start the service
makeshift serve --root ./tests --init -l debug

From e458783061ef8a7ed31dbff356ce695ee1442cd7 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 19:39:40 -0600 Subject: [PATCH 56/89] feat: added run example and args --- cmd/run.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index f932e07..399aa83 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -3,8 +3,19 @@ package cmd import "github.com/spf13/cobra" var runCmd = &cobra.Command{ - Use: "run", - Short: "Run 'makeshift' locally with plugins", + Use: "run", + Example: ` + # set up environment + export MAKESHIFT_HOST=http://localhost:5050 + export MAKESHIFT_PATH=help.txt + export MAKESHIFT_ROOT=./test + + # run locally similar to 'download' + makeshift run --plugins jinja2 --profiles default + makeshift run --root ./test -p help.txt --plugins jinja2 --profiles default +`, + Args: cobra.NoArgs, + Short: "Run locally with plugins and profiles", Run: func(cmd *cobra.Command, args []string) { }, From 85e333289b03d224da64441d26f902f95e127f9c Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 19:43:24 -0600 Subject: [PATCH 57/89] feat: added init cmd with example --- cmd/init.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 cmd/init.go diff --git a/cmd/init.go b/cmd/init.go new file mode 100644 index 0000000..e064ed0 --- /dev/null +++ b/cmd/init.go @@ -0,0 +1,20 @@ +package cmd + +import "github.com/spf13/cobra" + +var initCmd = &cobra.Command{ + Use: "init", + Example: ` + # create default files and directories at specified root path + makeshift init $MAKESHIFT_ROOT +`, + Args: cobra.ExactArgs(1), + Short: "Initialize directory with default files", + Run: func(cmd *cobra.Command, args []string) { + + }, +} + +func init() { + +} From 418889b17fb8f336ee0038e7ad02f9dd891b2a71 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 22:32:37 -0600 Subject: [PATCH 58/89] refactor: added error check for server.Init() --- cmd/serve.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/serve.go b/cmd/serve.go index b8a34be..22c793c 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -63,7 +63,14 @@ var serveCmd = &cobra.Command{ // make the default directories and files if flag is passed if cmd.Flags().Changed("init") { - server.Init() + err = server.Init() + if err != nil { + log.Error().Err(err). + Str("host", parsed.Host). + Str("root", rootPath). + Msg("failed to initialize server root") + return + } } // serve and log why the server closed From 325f77b9d4d665c6047476ba7e532d905597f425 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 22:33:07 -0600 Subject: [PATCH 59/89] feat: added init cmd implementation --- cmd/init.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/cmd/init.go b/cmd/init.go index e064ed0..31c9e9a 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -1,6 +1,10 @@ package cmd -import "github.com/spf13/cobra" +import ( + "git.towk2.me/towk/makeshift/pkg/service" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) var initCmd = &cobra.Command{ Use: "init", @@ -11,10 +15,27 @@ var initCmd = &cobra.Command{ Args: cobra.ExactArgs(1), Short: "Initialize directory with default files", Run: func(cmd *cobra.Command, args []string) { + var ( + rootPath, _ = cmd.Flags().GetString("root") + server *service.Service + err error + ) + + // create the server root files and directories + err = server.Init() + if err != nil { + log.Error().Err(err). + Str("root", rootPath). + Msg("failed to initialize server root") + return + } + log.Debug(). + Str("root", rootPath). + Msg("initialize makeshift files at root path") }, } func init() { - + rootCmd.AddCommand(initCmd) } From 94887aae9e24f270766df97eca19d76095dfdfa9 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 22:39:49 -0600 Subject: [PATCH 60/89] fix: panic with using init cmd --- cmd/init.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/init.go b/cmd/init.go index 31c9e9a..1bac85d 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -10,28 +10,29 @@ var initCmd = &cobra.Command{ Use: "init", Example: ` # create default files and directories at specified root path + # (must be set with positional argument) makeshift init $MAKESHIFT_ROOT `, Args: cobra.ExactArgs(1), Short: "Initialize directory with default files", Run: func(cmd *cobra.Command, args []string) { var ( - rootPath, _ = cmd.Flags().GetString("root") - server *service.Service err error ) // create the server root files and directories + server = service.New() + server.RootPath = args[0] err = server.Init() if err != nil { log.Error().Err(err). - Str("root", rootPath). + Str("root", server.RootPath). Msg("failed to initialize server root") return } log.Debug(). - Str("root", rootPath). + Str("root", server.RootPath). Msg("initialize makeshift files at root path") }, } From 947fbba85496a19ebc31d7a19af854e29010efdc Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 23:51:43 -0600 Subject: [PATCH 61/89] feat: added plugin and profile list commands --- cmd/download.go | 4 +- cmd/list.go | 175 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 3 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index 10c860f..0649e5b 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -177,7 +177,7 @@ var downloadProfileCmd = &cobra.Command{ Send() for _, profileID := range args { - query = fmt.Sprintf("/profile/%s", profileID) + query = fmt.Sprintf("/profiles/%s", profileID) res, body, err = c.MakeRequest(client.HTTPEnvelope{ Path: query, Method: http.MethodGet, @@ -233,7 +233,7 @@ var downloadPluginCmd = &cobra.Command{ Send() for _, pluginName := range args { - query = fmt.Sprintf("/plugin/%s", pluginName) + query = fmt.Sprintf("/plugins/%s", pluginName) res, body, err = c.MakeRequest(client.HTTPEnvelope{ Path: query, Method: http.MethodGet, diff --git a/cmd/list.go b/cmd/list.go index 38cc145..9f0a2f9 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -6,6 +6,7 @@ import ( "net/http" "os" + makeshift "git.towk2.me/towk/makeshift/pkg" "git.towk2.me/towk/makeshift/pkg/client" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -67,9 +68,181 @@ var listCmd = &cobra.Command{ }, } +var listPluginsCmd = &cobra.Command{ + Use: "plugins", + Example: ` + # show all plugins + makeshift list plugins + + # show details for specific plugins + makeshift list plugins smd jinja2 +`, + Short: "Show plugins information", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + + c = client.New(host) + res *http.Response + plugins []string + body []byte + err error + ) + + if len(args) == 0 { + // make request to /list endpoint + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: "/plugins", + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("path", path). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Str("path", path). + Msg("response returned bad status") + os.Exit(1) + } + fmt.Println(string(body)) + err = json.Unmarshal(body, &plugins) + if err != nil { + log.Error().Err(err). + Msg("failed to unmarshal plugins") + return + } + } else { + for _, pluginName := range args { + // make request to /list endpoint + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: fmt.Sprintf("/plugins/%s", pluginName), + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("path", path). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Str("path", path). + Msg("response returned bad status") + os.Exit(1) + } + + plugins = append(plugins, string(body)) + } + } + log.Info().Strs("plugins", plugins).Send() + }, +} + +var listProfilesCmd = &cobra.Command{ + Use: "profiles", + Short: "Show all available profiles", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + + c = client.New(host) + res *http.Response + profiles []makeshift.Profile + body []byte + err error + ) + + if len(args) == 0 { + // make request to /list endpoint + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: "/profiles", + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("path", path). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Str("path", path). + Msg("response returned bad status") + os.Exit(1) + } + + err = json.Unmarshal(body, &profiles) + if err != nil { + log.Error().Err(err). + Msg("failed to unmarshal plugins") + return + } + } else { + for _, profileID := range args { + // make request to /list endpoint + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: fmt.Sprintf("/profiles/%s", profileID), + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("path", path). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Str("path", path). + Msg("response returned bad status") + os.Exit(1) + } + var profile makeshift.Profile + err = json.Unmarshal(body, &profile) + if err != nil { + log.Error().Err(err). + Msg("failed to unmarshal plugin") + continue + } + profiles = append(profiles, profile) + } + } + log.Info().Any("plugins", profiles).Send() + }, +} + func init() { - listCmd.Flags().String("host", "http://localhost:5050", "Set the configurator remote host (can be set with MAKESHIFT_HOST)") + listCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the configurator remote host (can be set with MAKESHIFT_HOST)") listCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") + listCmd.AddCommand(listPluginsCmd, listProfilesCmd) rootCmd.AddCommand(listCmd) } From 4d960101995266499186e03a8b5ce268f8611fec Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 29 Aug 2025 23:52:43 -0600 Subject: [PATCH 62/89] refactor: updated routes and handlers --- pkg/models.go | 14 +++++++++++++- pkg/service/plugins.go | 12 ++++++++++-- pkg/service/service.go | 25 ++++++++++--------------- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/pkg/models.go b/pkg/models.go index b17b6b2..1b45444 100644 --- a/pkg/models.go +++ b/pkg/models.go @@ -1,6 +1,8 @@ package makeshift -import "git.towk2.me/towk/makeshift/pkg/storage" +import ( + "git.towk2.me/towk/makeshift/pkg/storage" +) type ProfileMap map[string]*Profile type Profile struct { @@ -30,3 +32,13 @@ type Hook struct { func (h *Hook) Run() error { return h.Plugin.Run(h.Data, h.Args) } + +func PluginToMap(p Plugin) map[string]any { + return map[string]any{ + "name": p.Name(), + "version": p.Version(), + "description": p.Description(), + "metadata": p.Metadata(), + } + +} diff --git a/pkg/service/plugins.go b/pkg/service/plugins.go index 1575f39..0e1efc5 100644 --- a/pkg/service/plugins.go +++ b/pkg/service/plugins.go @@ -2,6 +2,7 @@ package service import ( "encoding/json" + "fmt" "io" "net/http" "os" @@ -34,12 +35,13 @@ func (s *Service) ListPlugins() http.HandlerFunc { http.Error(w, err.Error(), http.StatusInternalServerError) return } + fmt.Printf("%v", plugins) w.Write(body) } } -func (s *Service) GetPlugin() http.HandlerFunc { +func (s *Service) GetPluginInfo() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( pluginName = chi.URLParam(r, "name") @@ -55,7 +57,7 @@ func (s *Service) GetPlugin() http.HandlerFunc { return } - body, err = json.Marshal(plugin) + body, err = json.Marshal(makeshift.PluginToMap(plugin)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -65,6 +67,12 @@ func (s *Service) GetPlugin() http.HandlerFunc { } } +func (s *Service) GetPluginRaw() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + } +} + func (s *Service) CreatePlugin() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( diff --git a/pkg/service/service.go b/pkg/service/service.go index c69d68f..56e4cae 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -97,7 +97,6 @@ func (s *Service) Serve() error { } else { // general - // router.Handle("/download/*", http.StripPrefix("/download/", http.FileServer(http.Dir(s.PathForData())))) router.Get("/download/*", s.Download()) router.Post("/upload/", s.Upload()) router.Post("/upload/plugin", s.UploadPlugin()) @@ -106,18 +105,18 @@ func (s *Service) Serve() error { // profiles router.Get("/profiles", s.ListProfiles()) - // router.Post("/profiles", s.CreateProfiles()) - router.Get("/profile/{id}", s.GetProfile()) - router.Post("/profile/{id}", s.CreateProfile()) - router.Get("/profile/{id}/data", s.GetProfileData()) - router.Post("/profile/{id}/data", s.SetProfileData()) - router.Delete("/profile/{id}/data", s.DeleteProfileData()) + router.Get("/profiles/{id}", s.GetProfile()) + router.Post("/profiles/{id}", s.CreateProfile()) + router.Get("/profiles/{id}/data", s.GetProfileData()) + router.Post("/profiles/{id}/data", s.SetProfileData()) + router.Delete("/profiles/{id}/data", s.DeleteProfileData()) // plugins router.Get("/plugins", s.ListPlugins()) - router.Get("/plugin/{name}", s.GetPlugin()) - router.Post("/plugin/{name}", s.CreatePlugin()) - router.Delete("/plugin/{name}", s.DeletePlugin()) + router.Get("/plugins/{name}/info", s.GetPluginInfo()) + router.Get("/plugins/{name}/raw", s.GetPluginRaw()) + router.Post("/plugins/{name}", s.CreatePlugin()) + router.Delete("/plugins/{name}", s.DeletePlugin()) } // always available public routes go here @@ -192,12 +191,8 @@ func LoadPluginsFromDir(dirpath string) (map[string]makeshift.Plugin, error) { // walk all files in directory only loading *valid* plugins err = filepath.Walk(dirpath, func(path string, info fs.FileInfo, err error) error { // skip trying to load generator plugin if directory or error - if info.IsDir() || err != nil { - return nil - } - // only try loading if file has .so extension - if hasValidExt(path) { + if info.IsDir() || err != nil || hasValidExt(path) { return nil } From 0f6f8957f6370ad68ef0c65af20089d7d0e1d8d2 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 00:04:17 -0600 Subject: [PATCH 63/89] refactor: updated cmd request and added plugin info --- cmd/download.go | 4 +-- cmd/plugins.go | 59 ++++++++++++++++++++++++++++++++++++++++-- pkg/service/plugins.go | 13 ++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/cmd/download.go b/cmd/download.go index 0649e5b..25e87aa 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -214,7 +214,7 @@ var downloadPluginCmd = &cobra.Command{ makeshift download plugin smd jinja2 `, Args: cobra.ExactArgs(1), - Short: "Download a plugin", + Short: "Download a raw plugin", Run: func(cmd *cobra.Command, args []string) { var ( host, _ = cmd.Flags().GetString("host") @@ -233,7 +233,7 @@ var downloadPluginCmd = &cobra.Command{ Send() for _, pluginName := range args { - query = fmt.Sprintf("/plugins/%s", pluginName) + query = fmt.Sprintf("/plugins/%s/raw", pluginName) res, body, err = c.MakeRequest(client.HTTPEnvelope{ Path: query, Method: http.MethodGet, diff --git a/cmd/plugins.go b/cmd/plugins.go index 3078e43..0308125 100644 --- a/cmd/plugins.go +++ b/cmd/plugins.go @@ -3,11 +3,13 @@ package cmd import ( "fmt" "io/fs" + "net/http" "os" "os/exec" "path/filepath" makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/client" "git.towk2.me/towk/makeshift/pkg/service" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -15,7 +17,7 @@ import ( var pluginsCmd = &cobra.Command{ Use: "plugins", - Short: "Manage and compile plugins (requires Go build tools)", + Short: "Manage, inspect, and compile plugins (requires Go build tools)", } var pluginsCompileCmd = &cobra.Command{ @@ -118,9 +120,62 @@ var pluginInspectCmd = &cobra.Command{ }, } +var pluginInfoCmd = &cobra.Command{ + Use: "info", + Short: "Show plugin information", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + outputPath, _ = cmd.Flags().GetString("output") + + c = client.New(host) + res *http.Response + query string + body []byte + err error + ) + + log.Debug(). + Str("host", host). + Str("output", outputPath). + Send() + + for _, pluginName := range args { + query = fmt.Sprintf("/plugins/%s/info", pluginName) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("query", query). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) + } + if outputPath != "" { + writeFiles(outputPath, body) + } else { + fmt.Println(string(body)) + } + } + }, +} + func init() { pluginsCompileCmd.Flags().StringP("output", "o", "", "Set the path to save compiled plugin") - pluginsCmd.AddCommand(pluginsCompileCmd, pluginInspectCmd) + pluginsCmd.AddCommand(pluginsCompileCmd, pluginInspectCmd, pluginInfoCmd) rootCmd.AddCommand(pluginsCmd) } diff --git a/pkg/service/plugins.go b/pkg/service/plugins.go index 0e1efc5..a8b0b6e 100644 --- a/pkg/service/plugins.go +++ b/pkg/service/plugins.go @@ -69,7 +69,20 @@ func (s *Service) GetPluginInfo() http.HandlerFunc { func (s *Service) GetPluginRaw() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var ( + pluginName = chi.URLParam(r, "name") + path = s.PathForPluginWithName(pluginName) + rawPlugin []byte + err error + ) + rawPlugin, err = os.ReadFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(rawPlugin) } } From 73498a08de7789db7d52437972de16aca2ec70ff Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 00:13:05 -0600 Subject: [PATCH 64/89] fix: issue with host not being set for plugins cmd --- cmd/plugins.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/plugins.go b/cmd/plugins.go index 0308125..812b5a8 100644 --- a/cmd/plugins.go +++ b/cmd/plugins.go @@ -18,6 +18,9 @@ import ( var pluginsCmd = &cobra.Command{ Use: "plugins", Short: "Manage, inspect, and compile plugins (requires Go build tools)", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + }, } var pluginsCompileCmd = &cobra.Command{ @@ -89,7 +92,7 @@ var pluginsCompileCmd = &cobra.Command{ }, } -var pluginInspectCmd = &cobra.Command{ +var pluginsInspectCmd = &cobra.Command{ Use: "inspect", Args: cobra.MinimumNArgs(1), Example: ` @@ -120,7 +123,7 @@ var pluginInspectCmd = &cobra.Command{ }, } -var pluginInfoCmd = &cobra.Command{ +var pluginsInfoCmd = &cobra.Command{ Use: "info", Short: "Show plugin information", Args: cobra.MinimumNArgs(1), @@ -159,6 +162,7 @@ var pluginInfoCmd = &cobra.Command{ Any("status", map[string]any{ "code": res.StatusCode, "message": res.Status, + "body": string(body), }). Str("host", host). Msg("response returned bad status") @@ -174,8 +178,10 @@ var pluginInfoCmd = &cobra.Command{ } func init() { - pluginsCompileCmd.Flags().StringP("output", "o", "", "Set the path to save compiled plugin") - pluginsCmd.AddCommand(pluginsCompileCmd, pluginInspectCmd, pluginInfoCmd) + pluginsCompileCmd.PersistentFlags().StringP("output", "o", "", "Set the path to save compiled plugin") + pluginsInfoCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + + pluginsCmd.AddCommand(pluginsCompileCmd, pluginsInspectCmd, pluginsInfoCmd) rootCmd.AddCommand(pluginsCmd) } From b791b84890427eee00e2521e631c8fb7f16ec0fe Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 00:22:58 -0600 Subject: [PATCH 65/89] chore: updated README examples --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 1f5c2d1..8e3630b 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,18 @@ makeshift list # list files store in the template directory with a specified host makeshift list --host http://localhost:5050 --path templates + +# list all available plugins +makeshift list plugins + +# list specific plugin (for plugin info, use 'makeshift plugin info` instead) +makeshift list plugins jinja2 + +# list all available profiles +makeshift list profiles + +# list specific profile information +makeshift list profiles default ``` Then, we can start downloading some files or directories (as archives). @@ -64,6 +76,12 @@ makeshift download -p templates --plugins smd,jinja2 --profile compute # do everything in the above example but extract and remove archive makeshift download -p templates --plugins smd,jinja2 --profile compute -xr + +# download a raw plugin +makeshift download plugin jinja2 + +# download a profile +makeshift download profile default ``` > [!NOTE] From dc8a9cff2026155075f60391a74d081630efca7b Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 00:23:53 -0600 Subject: [PATCH 66/89] refactor: minor changes and fixes --- cmd/list.go | 3 +-- internal/format/format.go | 16 ++++++++-------- pkg/service/plugins.go | 2 -- pkg/service/profiles.go | 10 ---------- pkg/service/service.go | 5 ++--- 5 files changed, 11 insertions(+), 25 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index 9f0a2f9..0de2797 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -114,7 +114,6 @@ var listPluginsCmd = &cobra.Command{ Msg("response returned bad status") os.Exit(1) } - fmt.Println(string(body)) err = json.Unmarshal(body, &plugins) if err != nil { log.Error().Err(err). @@ -125,7 +124,7 @@ var listPluginsCmd = &cobra.Command{ for _, pluginName := range args { // make request to /list endpoint res, body, err = c.MakeRequest(client.HTTPEnvelope{ - Path: fmt.Sprintf("/plugins/%s", pluginName), + Path: fmt.Sprintf("/plugins/%s/info", pluginName), Method: http.MethodGet, }) if err != nil { diff --git a/internal/format/format.go b/internal/format/format.go index d38aa3e..b753268 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -11,9 +11,9 @@ import ( type DataFormat string const ( - List DataFormat = "list" - JSON DataFormat = "json" - FORMAT_YAML DataFormat = "yaml" + List DataFormat = "list" + JSON DataFormat = "json" + YAML DataFormat = "yaml" ) func (df DataFormat) String() string { @@ -22,12 +22,12 @@ func (df DataFormat) String() string { func (df *DataFormat) Set(v string) error { switch DataFormat(v) { - case List, JSON, FORMAT_YAML: + case List, JSON, YAML: *df = DataFormat(v) return nil default: return fmt.Errorf("must be one of %v", []DataFormat{ - List, JSON, FORMAT_YAML, + List, JSON, YAML, }) } } @@ -48,7 +48,7 @@ func Marshal(data interface{}, outFormat DataFormat) ([]byte, error) { } else { return bytes, nil } - case FORMAT_YAML: + case YAML: if bytes, err := yaml.Marshal(data); err != nil { return nil, fmt.Errorf("failed to marshal data into YAML: %w", err) } else { @@ -72,7 +72,7 @@ func Unmarshal(data []byte, v interface{}, inFormat DataFormat) error { if err := json.Unmarshal(data, v); err != nil { return fmt.Errorf("failed to unmarshal data into JSON: %w", err) } - case FORMAT_YAML: + case YAML: if err := yaml.Unmarshal(data, v); err != nil { return fmt.Errorf("failed to unmarshal data into YAML: %w", err) } @@ -98,7 +98,7 @@ func DataFormatFromFileExt(path string, defaultFmt DataFormat) DataFormat { return JSON case ".yaml", ".yml", ".YAML", ".YML": // The file is a YAML file - return FORMAT_YAML + return YAML } return defaultFmt } diff --git a/pkg/service/plugins.go b/pkg/service/plugins.go index a8b0b6e..0dd516e 100644 --- a/pkg/service/plugins.go +++ b/pkg/service/plugins.go @@ -2,7 +2,6 @@ package service import ( "encoding/json" - "fmt" "io" "net/http" "os" @@ -35,7 +34,6 @@ func (s *Service) ListPlugins() http.HandlerFunc { http.Error(w, err.Error(), http.StatusInternalServerError) return } - fmt.Printf("%v", plugins) w.Write(body) } diff --git a/pkg/service/profiles.go b/pkg/service/profiles.go index ee7a0d3..edef023 100644 --- a/pkg/service/profiles.go +++ b/pkg/service/profiles.go @@ -64,16 +64,6 @@ func (s *Service) ListProfiles() http.HandlerFunc { } } -// func (s *Service) CreateProfiles() http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) { -// var ( -// path = chi.URLParam(r, "path") -// err error -// ) - -// } -// } - func (s *Service) GetProfile() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( diff --git a/pkg/service/service.go b/pkg/service/service.go index 56e4cae..b75bd57 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -184,15 +184,14 @@ func LoadPluginsFromDir(dirpath string) (map[string]makeshift.Plugin, error) { // helper to check for valid extensions var hasValidExt = func(path string) bool { - var validExts = []string{".so", ".dylib", ".dll"} - return slices.Contains(validExts, filepath.Ext(path)) + return slices.Contains([]string{".so", ".dylib", ".dll"}, filepath.Ext(path)) } // walk all files in directory only loading *valid* plugins err = filepath.Walk(dirpath, func(path string, info fs.FileInfo, err error) error { // skip trying to load generator plugin if directory or error // only try loading if file has .so extension - if info.IsDir() || err != nil || hasValidExt(path) { + if info.IsDir() || err != nil || !hasValidExt(path) { return nil } From 7fa685e8625bf947821f0d4bf7ad89b243a60561 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 00:26:50 -0600 Subject: [PATCH 67/89] refactor: correct comment in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e3630b..5d4e7bf 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ makeshift list --host http://localhost:5050 --path templates # list all available plugins makeshift list plugins -# list specific plugin (for plugin info, use 'makeshift plugin info` instead) +# list specific plugin (same as 'makeshift plugins info jinja2') makeshift list plugins jinja2 # list all available profiles From 4771bf45ac078cf00bbdf2b082e28e06f4eab990 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 00:29:18 -0600 Subject: [PATCH 68/89] refactor: added MAKESHIFT_LOG_LEVEL env var --- cmd/root.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/root.go b/cmd/root.go index f9ac4bc..caa383c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -33,6 +33,7 @@ var rootCmd = cobra.Command{ Run: func(cmd *cobra.Command, args []string) { // try and set flags using env vars setenv(cmd, "log-file", "MAKESHIFT_LOG_FILE") + setenv(cmd, "log-level", "MAKESHIFT_LOG_LEVEL") if len(args) == 0 { err := cmd.Help() if err != nil { From d191577ac980dd541af4c36cbc4dffa51be5451e Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 00:29:34 -0600 Subject: [PATCH 69/89] chore: updated README export vars --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5d4e7bf..2470177 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ Here are some of the common commands you may want to try right off the bat (asid export MAKESHIFT_HOST=localhost export MAKESHIFT_PATH=/test export MAKESHIFT_ROOT=./test +export MAKESHIFT_LOG_FILE=logs/makeshift.log +export MAKESHIFT_LOG_LEVEL=debug ``` Start the server. The `--init` flag with create the default files and directory to get started at the `--root` path. From 0349ecaf349f06e6452e8e353ff95f2d8e858a15 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 00:31:45 -0600 Subject: [PATCH 70/89] chore: added TODO in README --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2470177..dadab6f 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,7 @@ There are some features still missing that will be added later. 2. Uploading new profiles and plugins 3. Running `makeshift` locally with profiles and plugins 4. Plugin to add user data for one-time use without creating a profile -5. Protected routes that require authentication -6. Configuration file for persistent runs -7. `Dockerfile` and `docker-compose.yml` files \ No newline at end of file +5. Optionally build plugins directly into the main driver +6. Protected routes that require authentication +7. Configuration file for persistent runs +8. `Dockerfile` and `docker-compose.yml` files \ No newline at end of file From 32d534bcead7ee53e99899f0239f4da0d359485c Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 23:18:47 -0600 Subject: [PATCH 71/89] chore: updated README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dadab6f..7bcfc54 100644 --- a/README.md +++ b/README.md @@ -209,4 +209,5 @@ There are some features still missing that will be added later. 5. Optionally build plugins directly into the main driver 6. Protected routes that require authentication 7. Configuration file for persistent runs -8. `Dockerfile` and `docker-compose.yml` files \ No newline at end of file +8. `Dockerfile` and `docker-compose.yml` files to build containers +9. Including certs with requests \ No newline at end of file From 18edb93d2c8cdb00a7d6dc0b68e2045d4fe7081b Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 23:19:18 -0600 Subject: [PATCH 72/89] feat: added delete cmd --- cmd/delete.go | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 cmd/delete.go diff --git a/cmd/delete.go b/cmd/delete.go new file mode 100644 index 0000000..85dbd71 --- /dev/null +++ b/cmd/delete.go @@ -0,0 +1,123 @@ +package cmd + +import ( + "fmt" + "net/http" + + "git.towk2.me/towk/makeshift/pkg/client" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var deleteCmd = &cobra.Command{ + Use: "delete", + Example: ` + # set up environment + export MAKESHIFT_HOST=http://localhost:5050 + export MAKESHIFT_PATH=test + + # delete a file or directory (cannot delete root) + makeshift delete -p help.txt + makeshift delete --host http://localhost:5555 --path templates +`, + Short: "Delete files and directories", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") + }, + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + paths, _ = cmd.Flags().GetStringSlice("path") + + c = client.New(host) + res *http.Response + query string + err error + ) + for _, path := range paths { + if path == "" { + log.Warn().Msg("skipping empty path") + continue + } + + query = fmt.Sprintf("/delete/%s?", path) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodDelete, + }) + handleResponseError(res, host, query, err) + } + }, +} + +var deleteProfilesCmd = &cobra.Command{ + Use: "profiles", + Example: ` + # delete profile(s) by its ID + makeshift delete profiles kubernetes slurm compute +`, + Args: cobra.MinimumNArgs(1), + Short: "Delete profile(s)", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + + c = client.New(host) + res *http.Response + query string + err error + ) + + for _, profileID := range args { + if profileID == "default" { + log.Warn().Msg("cannot delete the default profile") + continue + } + + query = fmt.Sprintf("/profiles/%s", profileID) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodDelete, + }) + handleResponseError(res, host, query, err) + } + }, +} + +var deletePluginsCmd = &cobra.Command{ + Use: "plugins", + Example: ` + # delete plugin(s) by name + makeshift delete plugins weather slurm user +`, + Args: cobra.MinimumNArgs(1), + Short: "Delete plugin(s)", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + + c = client.New(host) + res *http.Response + query string + err error + ) + + for _, pluginName := range args { + query = fmt.Sprintf("/plugins/%s", pluginName) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodDelete, + }) + handleResponseError(res, host, query, err) + } + }, +} + +func init() { + deleteCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift server host (can be set with MAKESHIFT_HOST)") + deleteCmd.Flags().StringSliceP("path", "p", []string{}, "Set the paths to delete files and directories") + deleteCmd.AddCommand(deleteProfilesCmd, deletePluginsCmd) + + rootCmd.AddCommand(deleteCmd) +} From 5d350717f4e999cc6483401e6dc7e5b530acc204 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 23:19:45 -0600 Subject: [PATCH 73/89] chore: updated references in Makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c1e7a87..5e8d1f1 100644 --- a/Makefile +++ b/Makefile @@ -36,13 +36,13 @@ container-testing: binaries plugins: $(plugin_binaries) # how to make each plugin -lib/%.so: pkg/generator/plugins/%/*.go +lib/%.so: pkg/plugins/%/*.go mkdir -p lib go build -buildmode=plugin -o $@ $< docs: - go doc github.com/OpenCHAMI/cmd - go doc github.com/OpenCHAMI/pkg/${prog} + go doc git.towk2.me/towk/makeshift/cmd + go doc git.towk2.me/towk/makeshift/pkg/${prog} # remove executable and all built plugins .PHONY: clean From ebb95a29ec9ee1ec67fd3aff616e0de3baa25db2 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 23:20:47 -0600 Subject: [PATCH 74/89] refactor: renamed userdata plugin to mapper --- pkg/plugins/mapper/mapper.go | 30 ++++++++++++++++++++++++++++++ pkg/plugins/userdata/userdata.go | 30 ------------------------------ 2 files changed, 30 insertions(+), 30 deletions(-) create mode 100644 pkg/plugins/mapper/mapper.go delete mode 100644 pkg/plugins/userdata/userdata.go diff --git a/pkg/plugins/mapper/mapper.go b/pkg/plugins/mapper/mapper.go new file mode 100644 index 0000000..1fded1f --- /dev/null +++ b/pkg/plugins/mapper/mapper.go @@ -0,0 +1,30 @@ +package main + +import "git.towk2.me/towk/makeshift/pkg/storage" + +type Mapper struct{} + +func (p *Mapper) Name() string { return "jinja2" } +func (p *Mapper) Version() string { return "test" } +func (p *Mapper) Description() string { return "Renders Jinja 2 templates" } +func (p *Mapper) Metadata() map[string]string { + return map[string]string{ + "author.name": "David J. Allen", + "author.email": "davidallendj@gmail.com", + } +} + +func (p *Mapper) Init() error { + // nothing to initialize + return nil +} + +func (p *Mapper) Run(data storage.KVStore, args []string) error { + return nil +} + +func (p *Mapper) Clean() error { + return nil +} + +var Makeshift Mapper diff --git a/pkg/plugins/userdata/userdata.go b/pkg/plugins/userdata/userdata.go deleted file mode 100644 index ae5d31f..0000000 --- a/pkg/plugins/userdata/userdata.go +++ /dev/null @@ -1,30 +0,0 @@ -package main - -import "git.towk2.me/towk/makeshift/pkg/storage" - -type UserData struct{} - -func (p *UserData) Name() string { return "jinja2" } -func (p *UserData) Version() string { return "test" } -func (p *UserData) Description() string { return "Renders Jinja 2 templates" } -func (p *UserData) Metadata() map[string]string { - return map[string]string{ - "author.name": "David J. Allen", - "author.email": "davidallendj@gmail.com", - } -} - -func (p *UserData) Init() error { - // nothing to initialize - return nil -} - -func (p *UserData) Run(data storage.KVStore, args []string) error { - return nil -} - -func (p *UserData) Clean() error { - return nil -} - -var Makeshift UserData From d88ab2c01fab9272d03b6a821642b96b37a0aa77 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 23:21:17 -0600 Subject: [PATCH 75/89] feat: added compile-plugins.sh script --- bin/.gitkeep | 0 bin/compile-plugins.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+) delete mode 100644 bin/.gitkeep create mode 100644 bin/compile-plugins.sh diff --git a/bin/.gitkeep b/bin/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/bin/compile-plugins.sh b/bin/compile-plugins.sh new file mode 100644 index 0000000..278bc7c --- /dev/null +++ b/bin/compile-plugins.sh @@ -0,0 +1,26 @@ +#!/usr/bin bash + + +function compile_default_plugins() { + makeshift_exe=./makeshift + go_exe=go + + # make sure go build tools are installed + if command -v $go_exe >/dev/null 2>&1; then + # make sure that MAKESHIFT_ROOT is set + if [[ ! -v MAKESHIFT_ROOT ]]; then + # Compile the default external plugins + go build + $makeshift_exe compile pkg/plugins/jinja2/jinja2.go -o $MAKESHIFT_ROOT/plugins/jinja2.go + $makeshift_exe compile pkg/plugins/smd/smd.go -o $MAKESHIFT_ROOT/plugins/smd.so + $makeshift_exe compile pkg/plugins/userdata/userdata.go -o $MAKESHIFT_ROOT/plugins/userdata.go + else + echo "requires MAKESHIFT_ROOT to be set" + fi + else + echo "Go build tools must be installed" + fi +} + + +compile_default_plugins \ No newline at end of file From fbed466c3d349619f99f8b9ff9a38f450c1e2235 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sat, 30 Aug 2025 23:30:46 -0600 Subject: [PATCH 76/89] refactor: updated cmd and pkg implementations --- cmd/list.go | 86 ++-------------- cmd/plugins.go | 4 +- cmd/root.go | 31 ++++++ cmd/run.go | 6 +- cmd/upload.go | 218 +++++++++++++++++++++++++--------------- pkg/client/client.go | 41 ++++++++ pkg/service/plugins.go | 16 +-- pkg/service/profiles.go | 43 ++++++-- pkg/service/routes.go | 32 +++--- pkg/service/service.go | 6 +- 10 files changed, 287 insertions(+), 196 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index 0de2797..1fa3fe6 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -81,10 +81,10 @@ var listPluginsCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { var ( host, _ = cmd.Flags().GetString("host") - path, _ = cmd.Flags().GetString("path") c = client.New(host) res *http.Response + query string plugins []string body []byte err error @@ -96,24 +96,7 @@ var listPluginsCmd = &cobra.Command{ Path: "/plugins", Method: http.MethodGet, }) - if err != nil { - log.Error().Err(err). - Str("host", host). - Str("path", path). - Msg("failed to make request") - os.Exit(1) - } - if res.StatusCode != http.StatusOK { - log.Error(). - Any("status", map[string]any{ - "code": res.StatusCode, - "message": res.Status, - }). - Str("host", host). - Str("path", path). - Msg("response returned bad status") - os.Exit(1) - } + handleResponseError(res, host, "/plugins", err) err = json.Unmarshal(body, &plugins) if err != nil { log.Error().Err(err). @@ -123,28 +106,12 @@ var listPluginsCmd = &cobra.Command{ } else { for _, pluginName := range args { // make request to /list endpoint + query = fmt.Sprintf("/plugins/%s/info", pluginName) res, body, err = c.MakeRequest(client.HTTPEnvelope{ - Path: fmt.Sprintf("/plugins/%s/info", pluginName), + Path: query, Method: http.MethodGet, }) - if err != nil { - log.Error().Err(err). - Str("host", host). - Str("path", path). - Msg("failed to make request") - os.Exit(1) - } - if res.StatusCode != http.StatusOK { - log.Error(). - Any("status", map[string]any{ - "code": res.StatusCode, - "message": res.Status, - }). - Str("host", host). - Str("path", path). - Msg("response returned bad status") - os.Exit(1) - } + handleResponseError(res, host, query, err) plugins = append(plugins, string(body)) } @@ -159,12 +126,12 @@ var listProfilesCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { var ( host, _ = cmd.Flags().GetString("host") - path, _ = cmd.Flags().GetString("path") c = client.New(host) res *http.Response profiles []makeshift.Profile body []byte + query string err error ) @@ -174,24 +141,7 @@ var listProfilesCmd = &cobra.Command{ Path: "/profiles", Method: http.MethodGet, }) - if err != nil { - log.Error().Err(err). - Str("host", host). - Str("path", path). - Msg("failed to make request") - os.Exit(1) - } - if res.StatusCode != http.StatusOK { - log.Error(). - Any("status", map[string]any{ - "code": res.StatusCode, - "message": res.Status, - }). - Str("host", host). - Str("path", path). - Msg("response returned bad status") - os.Exit(1) - } + handleResponseError(res, host, "/profiles", err) err = json.Unmarshal(body, &profiles) if err != nil { @@ -202,28 +152,12 @@ var listProfilesCmd = &cobra.Command{ } else { for _, profileID := range args { // make request to /list endpoint + query = fmt.Sprintf("/profiles/%s", profileID) res, body, err = c.MakeRequest(client.HTTPEnvelope{ - Path: fmt.Sprintf("/profiles/%s", profileID), + Path: fmt.Sprintf(query), Method: http.MethodGet, }) - if err != nil { - log.Error().Err(err). - Str("host", host). - Str("path", path). - Msg("failed to make request") - os.Exit(1) - } - if res.StatusCode != http.StatusOK { - log.Error(). - Any("status", map[string]any{ - "code": res.StatusCode, - "message": res.Status, - }). - Str("host", host). - Str("path", path). - Msg("response returned bad status") - os.Exit(1) - } + handleResponseError(res, host, query, err) var profile makeshift.Profile err = json.Unmarshal(body, &profile) if err != nil { diff --git a/cmd/plugins.go b/cmd/plugins.go index 812b5a8..8917b1c 100644 --- a/cmd/plugins.go +++ b/cmd/plugins.go @@ -178,8 +178,8 @@ var pluginsInfoCmd = &cobra.Command{ } func init() { - pluginsCompileCmd.PersistentFlags().StringP("output", "o", "", "Set the path to save compiled plugin") - pluginsInfoCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + pluginsCompileCmd.Flags().StringP("output", "o", "", "Set the path to save compiled plugin (matches source type, i.e. uses files or directory)") + pluginsInfoCmd.Flags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") pluginsCmd.AddCommand(pluginsCompileCmd, pluginsInspectCmd, pluginsInfoCmd) rootCmd.AddCommand(pluginsCmd) diff --git a/cmd/root.go b/cmd/root.go index caa383c..fb354f0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "net/http" "os" logger "git.towk2.me/towk/makeshift/pkg/log" @@ -78,6 +79,16 @@ func setenv(cmd *cobra.Command, varname string, envvar string) { } } +func setenvp(cmd *cobra.Command, varname string, envvar string) { + if cmd.Flags().Changed(varname) { + return + } + val := os.Getenv(envvar) + if val != "" { + cmd.PersistentFlags().Set(varname, val) + } +} + func initLogger() { // initialize the logger logfile, _ := rootCmd.PersistentFlags().GetString("log-file") @@ -87,3 +98,23 @@ func initLogger() { os.Exit(1) } } + +func handleResponseError(res *http.Response, host, query string, err error) { + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("query", query). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) + } +} diff --git a/cmd/run.go b/cmd/run.go index 399aa83..ab698e7 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -5,14 +5,16 @@ import "github.com/spf13/cobra" var runCmd = &cobra.Command{ Use: "run", Example: ` + NOTE: This command is not implemented yet! + # set up environment export MAKESHIFT_HOST=http://localhost:5050 export MAKESHIFT_PATH=help.txt - export MAKESHIFT_ROOT=./test + export MAKESHIFT_ROOT=/opt/makeshift # run locally similar to 'download' makeshift run --plugins jinja2 --profiles default - makeshift run --root ./test -p help.txt --plugins jinja2 --profiles default + makeshift run --root $HOME/apps/makeshift -p help.txt --plugins jinja2 --profiles default `, Args: cobra.NoArgs, Short: "Run locally with plugins and profiles", diff --git a/cmd/upload.go b/cmd/upload.go index 12b58b1..ecdd638 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -4,72 +4,127 @@ import ( "bufio" "encoding/json" "fmt" + "net/http" "os" "strings" "git.towk2.me/towk/makeshift/internal/format" + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/client" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) var ( inputFormat format.DataFormat = format.JSON - dataArgs []string ) var uploadCmd = &cobra.Command{ Use: "upload", Example: ` - # upload a single file - makeshift upload -d @compute-base.yaml -t file - - # upload a single file with contents without specify type - makeshift upload -d '{"name": "John Smith", "email": "john.smith@example.com"}' + # upload a single file in root directory + makeshift upload -d @compute-base.yaml # upload a directory - makeshift upload -d @setup/ -t directory + makeshift upload -d @setup/ # upload an archive (extracted and saved on server) makeshift upload -d @setup.tar.gz -t archive + + # upload a new profile with a specific path (used to set remote location) + makeshift upload profile -d @kubernetes.json -p nodes/kubernetes.json + makeshift upload profile -d @slurm.json -@compute.json -p nodes `, + Short: "Upload files and directories", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") + }, Run: func(cmd *cobra.Command, args []string) { - // make one request be host positional argument (restricted to 1 for now) - var inputData []map[string]any - temp := append(handleArgs(args), processDataArgs(dataArgs)...) - for _, data := range temp { - if data != nil { - inputData = append(inputData, data) + var ( + dataArgs, _ = cmd.Flags().GetStringArray("data") + + inputData = processFiles(dataArgs) + useDirectoryPath = len(inputData) > 1 + ) + for path, contents := range inputData { + log.Info().Str("path", path).Int("size", len(contents)).Send() + if useDirectoryPath { + + } else { + } } }, } -var uploadProfileCmd = &cobra.Command{ - Use: "profile", +var uploadProfilesCmd = &cobra.Command{ + Use: "profile [profileID]", Example: ` # upload a new profile makeshift upload profile -d @compute.json - # upload a new profile with a specific name (used for lookups) + # upload a new profile with a specific path (used for lookup) makeshift upload profile -d @kubernetes.json -n k8s `, - Args: cobra.ExactArgs(1), + Args: cobra.NoArgs, Short: "Upload a new profile", Run: func(cmd *cobra.Command, args []string) { // make one request be host positional argument (restricted to 1 for now) - var inputData []map[string]any - temp := append(handleArgs(args), processDataArgs(dataArgs)...) - for _, data := range temp { - if data != nil { - inputData = append(inputData, data) + var ( + // inputData []map[string]any = append(handleArgs(args), processDataArgs(dataArgs)...) + host, _ = cmd.Flags().GetString("host") + dataArgs, _ = cmd.Flags().GetStringArray("data") + profiles = processProfiles(dataArgs) + + c = client.New(host) + res *http.Response + query string + body []byte + err error + ) + + for _, profile := range profiles { + if profile == nil { + continue + } + + body, err = json.Marshal(profile) + if err != nil { + log.Error().Err(err).Msg("failed to marshal profile") + continue + } + + // send data to server + query = fmt.Sprintf("/profiles/%s", profile.ID) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodPost, + Body: body, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("query", query). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) } } - }, } -var uploadPluginCmd = &cobra.Command{ +var uploadPluginsCmd = &cobra.Command{ Use: "plugin", Example: ` # upload a new plugin @@ -82,8 +137,12 @@ var uploadPluginCmd = &cobra.Command{ Short: "Upload a new plugin", Run: func(cmd *cobra.Command, args []string) { // make one request be host positional argument (restricted to 1 for now) - var inputData []map[string]any - temp := append(handleArgs(args), processDataArgs(dataArgs)...) + // temp := append(handleArgs(args), processDataArgs(dataArgs)...) + var ( + inputData []*makeshift.Profile + dataArgs, _ = cmd.PersistentFlags().GetStringArray("data") + ) + temp := processProfiles(dataArgs) for _, data := range temp { if data != nil { inputData = append(inputData, data) @@ -93,27 +152,62 @@ var uploadPluginCmd = &cobra.Command{ } func init() { - uploadProfileCmd.Flags().VarP(&inputFormat, "format", "F", "Set the input format for profile") + uploadCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + uploadCmd.PersistentFlags().StringArrayP("data", "d", []string{}, "Set the data to send to specified host (prepend @ for files)") + uploadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") - uploadCmd.AddCommand(uploadProfileCmd, uploadPluginCmd) + uploadProfilesCmd.Flags().VarP(&inputFormat, "format", "F", "Set the input format for profile") + + uploadCmd.AddCommand(uploadProfilesCmd, uploadPluginsCmd) rootCmd.AddCommand(uploadCmd) } -// processDataArgs takes a slice of strings that check for the @ symbol and loads +func processFiles(args []string) map[string][]byte { + // load data either from file or directly from args + var collection = make(map[string][]byte, len(args)) + for _, arg := range args { + // if arg is empty string, then skip and continue + if len(arg) > 0 { + // determine if we're reading from file to load contents + if strings.HasPrefix(arg, "@") { + var ( + path string = strings.TrimLeft(arg, "@") + contents []byte + err error + ) + contents, err = os.ReadFile(path) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("failed to read file") + continue + } + + // skip empty files + if len(contents) == 0 { + log.Warn().Str("path", path).Msg("file is empty") + continue + } + + // add loaded data to collection of all data + collection[path] = contents + } else { + log.Warn().Msg("only files can be uploaded") + + continue + } + } + } + return collection +} + +// processProfiles takes a slice of strings that check for the @ symbol and loads // the contents from the file specified in place (which replaces the path). // // NOTE: The purpose is to make the input arguments uniform for our request. This // function is meant to handle data passed with the `-d/--data` flag and positional // args from the CLI. -func processDataArgs(args []string) []map[string]any { - // JSON representation - type ( - JSONObject = map[string]any - JSONArray = []JSONObject - ) - +func processProfiles(args []string) []*makeshift.Profile { // load data either from file or directly from args - var collection = make(JSONArray, len(args)) + var collection = make([]*makeshift.Profile, len(args)) for i, arg := range args { // if arg is empty string, then skip and continue if len(arg) > 0 { @@ -122,7 +216,7 @@ func processDataArgs(args []string) []map[string]any { var ( path string = strings.TrimLeft(arg, "@") contents []byte - data JSONArray + data *makeshift.Profile err error ) contents, err = os.ReadFile(path) @@ -138,17 +232,17 @@ func processDataArgs(args []string) []map[string]any { } // convert/validate input data - data, err = parseInput(contents, format.DataFormatFromFileExt(path, inputFormat)) + data, err = parseProfile(contents, format.DataFormatFromFileExt(path, inputFormat)) if err != nil { log.Error().Err(err).Str("path", path).Msg("failed to validate input from file") } // add loaded data to collection of all data - collection = append(collection, data...) + collection = append(collection, data) } else { // input should be a valid JSON var ( - data JSONArray + data *makeshift.Profile input = []byte(arg) err error ) @@ -160,57 +254,23 @@ func processDataArgs(args []string) []map[string]any { if err != nil { log.Error().Err(err).Msgf("failed to unmarshal input for argument %d", i) } - return data + return []*makeshift.Profile{data} } } } return collection } -func handleArgs(args []string) []map[string]any { - // JSON representation - type ( - JSONObject = map[string]any - JSONArray = []JSONObject - ) - // no file to load, so we just use the joined args (since each one is a new line) - // and then stop +func parseProfile(contents []byte, dataFormat format.DataFormat) (*makeshift.Profile, error) { var ( - collection JSONArray - data []byte - err error - ) - - if len(dataArgs) > 0 { - return nil - } - data, err = ReadStdin() - if err != nil { - log.Error().Err(err).Msg("failed to read from standard input") - return nil - } - if len(data) == 0 { - log.Warn().Msg("no data found from standard input") - return nil - } - fmt.Println(string(data)) - collection, err = parseInput([]byte(data), inputFormat) - if err != nil { - log.Error().Err(err).Msg("failed to validate input from arg") - } - return collection -} - -func parseInput(contents []byte, dataFormat format.DataFormat) ([]map[string]any, error) { - var ( - data []map[string]any + data *makeshift.Profile err error ) // convert/validate JSON input format err = format.Unmarshal(contents, &data, dataFormat) if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) + return nil, fmt.Errorf("failed to unmarshal profile: %v", err) } return data, nil } diff --git a/pkg/client/client.go b/pkg/client/client.go index f9cf5d9..028fced 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -1,12 +1,16 @@ package client import ( + "crypto/tls" + "crypto/x509" "fmt" "io" "mime/multipart" + "net" "net/http" "os" "strings" + "time" "git.towk2.me/towk/makeshift/pkg/util" "github.com/cavaliergopher/grab/v3" @@ -107,6 +111,43 @@ func (c *Client) UploadMultipartFile(uri, key, path string) (*http.Response, err return resp, nil } +func (c *Client) LoadCertificateFromPath(path string) error { + cacert, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read certificate at path: %s", path) + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + err = c.LoadCertificateFromPool(certPool) + if err != nil { + return fmt.Errorf("could not initialize certificate from pool: %v", err) + } + return nil +} + +func (c *Client) LoadCertificateFromPool(certPool *x509.CertPool) error { + // make sure we have a valid cert pool + if certPool == nil { + return fmt.Errorf("invalid cert pool") + } + + // make sure that we can access the internal client + c.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: false, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + return nil +} + func mustOpen(f string) *os.File { r, err := os.Open(f) if err != nil { diff --git a/pkg/service/plugins.go b/pkg/service/plugins.go index 0dd516e..0ab12eb 100644 --- a/pkg/service/plugins.go +++ b/pkg/service/plugins.go @@ -8,6 +8,7 @@ import ( makeshift "git.towk2.me/towk/makeshift/pkg" "github.com/go-chi/chi/v5" + "github.com/rs/zerolog/log" ) func (s *Service) ListPlugins() http.HandlerFunc { @@ -124,24 +125,17 @@ func (s *Service) CreatePlugin() http.HandlerFunc { func (s *Service) DeletePlugin() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - path string - plugin makeshift.Plugin - err error + pluginName = chi.URLParam(r, "name") + path = s.PathForPluginWithName(pluginName) + err error ) - plugin, err = getPluginFromRequestBody(r) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - path = s.PathForPluginWithName(plugin.Name()) + log.Debug().Str("path", path).Send() err = os.Remove(path) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.WriteHeader(http.StatusOK) } } diff --git a/pkg/service/profiles.go b/pkg/service/profiles.go index edef023..e951e26 100644 --- a/pkg/service/profiles.go +++ b/pkg/service/profiles.go @@ -88,13 +88,10 @@ func (s *Service) GetProfile() http.HandlerFunc { func (s *Service) CreateProfile() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - type input struct { - Path string `json:"path"` - Profile *makeshift.Profile `json:"profile"` - } var ( body, contents []byte - in input + path string + profile *makeshift.Profile err error ) @@ -105,23 +102,24 @@ func (s *Service) CreateProfile() http.HandlerFunc { } // use the request info to build profile - err = json.Unmarshal(body, &in) + err = json.Unmarshal(body, &profile) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, fmt.Sprintf("failed to unmarshal profile: %v", err.Error()), http.StatusBadRequest) return } // serialize just the profile part - contents, err = json.Marshal(in.Profile) + contents, err = json.Marshal(profile) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, fmt.Sprintf("failed to marshal profile: %v", err.Error()), http.StatusBadRequest) return } // create a new profile on disk - err = os.WriteFile(in.Path, contents, os.ModePerm) + path = s.PathForProfileWithID(profile.ID) + err = os.WriteFile(path, contents, os.ModePerm) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -129,6 +127,29 @@ func (s *Service) CreateProfile() http.HandlerFunc { } } +func (s *Service) DeleteProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + profileID = chi.URLParam(r, "id") + path string + err error + ) + + if profileID == "default" { + http.Error(w, "cannot delete the default profile", http.StatusBadRequest) + return + } + + path = s.PathForProfileWithID(profileID) + err = os.Remove(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + } +} + func (s *Service) SetProfileData() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( diff --git a/pkg/service/routes.go b/pkg/service/routes.go index 42e7338..23d2d05 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -181,18 +181,9 @@ func (s *Service) Download() http.HandlerFunc { func (s *Service) Upload() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - - } -} - -func (s *Service) UploadPlugin() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - - } -} - -func (s *Service) UploadProfile() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { + var ( + _ = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/upload") + ) } } @@ -239,6 +230,23 @@ func (s *Service) List() http.HandlerFunc { } } +func (s *Service) Delete() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/delete") + err error + ) + + err = os.RemoveAll(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + } +} + func (s *Service) GetStatus(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") err := json.NewEncoder(w).Encode(map[string]any{ diff --git a/pkg/service/service.go b/pkg/service/service.go index b75bd57..362a08c 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -98,15 +98,15 @@ func (s *Service) Serve() error { } else { // general router.Get("/download/*", s.Download()) - router.Post("/upload/", s.Upload()) - router.Post("/upload/plugin", s.UploadPlugin()) - router.Post("/upload/profile", s.UploadProfile()) + router.Post("/upload/*", s.Upload()) router.Get("/list/*", s.List()) + router.Delete("/delete/*", s.Delete()) // profiles router.Get("/profiles", s.ListProfiles()) router.Get("/profiles/{id}", s.GetProfile()) router.Post("/profiles/{id}", s.CreateProfile()) + router.Delete("/profiles/{id}", s.DeleteProfile()) router.Get("/profiles/{id}/data", s.GetProfileData()) router.Post("/profiles/{id}/data", s.SetProfileData()) router.Delete("/profiles/{id}/data", s.DeleteProfileData()) From 13b02c03e8f13387d08c09c0dac4e13309f69a51 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 00:13:33 -0600 Subject: [PATCH 77/89] feat: implemented upload cmd and pkg --- cmd/upload.go | 29 +++++++++++++++++++++++------ pkg/service/routes.go | 30 +++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/cmd/upload.go b/cmd/upload.go index ecdd638..d1a9757 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "os" + "path/filepath" "strings" "git.towk2.me/towk/makeshift/internal/format" @@ -42,18 +43,37 @@ var uploadCmd = &cobra.Command{ }, Run: func(cmd *cobra.Command, args []string) { var ( + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") dataArgs, _ = cmd.Flags().GetStringArray("data") inputData = processFiles(dataArgs) useDirectoryPath = len(inputData) > 1 + c = client.New(host) + res *http.Response + query string + err error ) - for path, contents := range inputData { + for inputPath, contents := range inputData { log.Info().Str("path", path).Int("size", len(contents)).Send() if useDirectoryPath { - + query = path + "/" + filepath.Clean(inputPath) } else { - + // use flag value if supplied + if cmd.Flags().Changed("path") { + query = path + } else { + query = inputPath + } } + + query = fmt.Sprintf("/upload/%s", query) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodPost, + Body: contents, + }) + handleResponseError(res, host, query, err) } }, } @@ -70,10 +90,7 @@ var uploadProfilesCmd = &cobra.Command{ Args: cobra.NoArgs, Short: "Upload a new profile", Run: func(cmd *cobra.Command, args []string) { - - // make one request be host positional argument (restricted to 1 for now) var ( - // inputData []map[string]any = append(handleArgs(args), processDataArgs(dataArgs)...) host, _ = cmd.Flags().GetString("host") dataArgs, _ = cmd.Flags().GetStringArray("data") profiles = processProfiles(dataArgs) diff --git a/pkg/service/routes.go b/pkg/service/routes.go index 23d2d05..b5dd890 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -3,6 +3,7 @@ package service import ( "encoding/json" "fmt" + "io" "io/fs" "net/http" "os" @@ -182,9 +183,36 @@ func (s *Service) Download() http.HandlerFunc { func (s *Service) Upload() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - _ = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/upload") + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/upload") + body []byte + dirpath string + err error ) + // show what we're uploading + log.Debug().Str("path", path).Msg("Service.Upload()") + + // take the provided path and store the file contents + dirpath = filepath.Dir(path) + err = os.MkdirAll(dirpath, 0o777) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // write file to disk + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + err = os.WriteFile(path, body, 0o777) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) } } From c495d10aaddb28414f0e0a78c79381338884d432 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 00:17:00 -0600 Subject: [PATCH 78/89] refactor: change reference in goreleaser --- .goreleaser.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index d91d3e3..badde3a 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -5,7 +5,7 @@ before: - go mod download - make plugins builds: - - id: "configurator" + - id: "makeshift" goos: - linux goarch: @@ -30,10 +30,10 @@ archives: dockers: - image_templates: - - ghcr.io/openchami/{{.ProjectName}}:latest - - ghcr.io/openchami/{{.ProjectName}}:{{ .Tag }} - - ghcr.io/openchami/{{.ProjectName}}:v{{ .Major }} - - ghcr.io/openchami/{{.ProjectName}}:v{{ .Major }}.{{ .Minor }} + - git.towk2.me/towk/{{.ProjectName}}:latest + - git.towk2.me/towk/{{.ProjectName}}:{{ .Tag }} + - git.towk2.me/towk/{{.ProjectName}}:v{{ .Major }} + - git.towk2.me/towk/{{.ProjectName}}:v{{ .Major }}.{{ .Minor }} build_flag_templates: - "--pull" - "--label=org.opencontainers.image.created={{.Date}}" From afc7db53e195bfe53d6de8f2f786fefba6a606d3 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 00:20:23 -0600 Subject: [PATCH 79/89] chore: updated README with upload examples --- README.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7bcfc54..c30b134 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,30 @@ makeshift download profile default (WIP) Files, directories, profiles, and plugins will eventually be able to be uploaded to the server. ```bash -# upload a file or directory (recursively) -makeshift upload -makeshift upload --host http://localhost:5050 --path help.txt +# upload a single file in root directory + makeshift upload -d @compute-base.yaml + + # upload a directory (not working yet...) + makeshift upload -d @setup/ + + # upload an archive (extracted and saved on server - not working yet...) + makeshift upload -d @setup.tar.gz -t archive + + # upload a new profile with a specific path (used to set remote location) + makeshift upload profile -d @kubernetes.json -p nodes/kubernetes.json + makeshift upload profile -d @slurm.json -@compute.json -p nodes + + # upload a new profile + makeshift upload profile -d @compute.json + + # upload a new profile with a specific path (used for lookup) + makeshift upload profile -d @kubernetes.json -n k8s + + # upload a new plugin + makeshift upload plugin -d @slurm.so + + # upload a new plugin with a specific name (used for lookups) + makeshift upload plugin -d @cobbler.so -n merge ``` > [!NOTE] From fa8ef7ab4b3a4baf396d2721a3aef16955df8d7c Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 00:21:19 -0600 Subject: [PATCH 80/89] chore: fix typos in files --- README.md | 2 +- cmd/upload.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c30b134..9104784 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ makeshift download profile default # upload a new profile with a specific path (used to set remote location) makeshift upload profile -d @kubernetes.json -p nodes/kubernetes.json - makeshift upload profile -d @slurm.json -@compute.json -p nodes + makeshift upload profile -d @slurm.json -d @compute.json -p nodes # upload a new profile makeshift upload profile -d @compute.json diff --git a/cmd/upload.go b/cmd/upload.go index d1a9757..260ff3f 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -34,7 +34,7 @@ var uploadCmd = &cobra.Command{ # upload a new profile with a specific path (used to set remote location) makeshift upload profile -d @kubernetes.json -p nodes/kubernetes.json - makeshift upload profile -d @slurm.json -@compute.json -p nodes + makeshift upload profile -d @slurm.json -d @compute.json -p nodes `, Short: "Upload files and directories", PersistentPreRun: func(cmd *cobra.Command, args []string) { From e115319913b77a57392b5f2a536ae657fc4c0687 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 11:01:43 -0600 Subject: [PATCH 81/89] feat: implemented local plugins info --- cmd/plugins.go | 84 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/cmd/plugins.go b/cmd/plugins.go index 8917b1c..15043f0 100644 --- a/cmd/plugins.go +++ b/cmd/plugins.go @@ -124,13 +124,21 @@ var pluginsInspectCmd = &cobra.Command{ } var pluginsInfoCmd = &cobra.Command{ - Use: "info", + Use: "info", + Example: ` + # show information of a remote plugin + makeshift plugins info jinja2 smd + + # show information of a local plugin + makeshift plugins info --local $MAKESHIFT_ROOT/plugins/jinja2.so +`, Short: "Show plugin information", Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { var ( host, _ = cmd.Flags().GetString("host") outputPath, _ = cmd.Flags().GetString("output") + local, _ = cmd.Flags().GetBool("local") c = client.New(host) res *http.Response @@ -144,34 +152,53 @@ var pluginsInfoCmd = &cobra.Command{ Str("output", outputPath). Send() - for _, pluginName := range args { - query = fmt.Sprintf("/plugins/%s/info", pluginName) - res, body, err = c.MakeRequest(client.HTTPEnvelope{ - Path: query, - Method: http.MethodGet, - }) - if err != nil { - log.Error().Err(err). - Str("host", host). - Str("query", query). - Msg("failed to make request") - os.Exit(1) + if local { + var ( + plugins []map[string]any + plugin makeshift.Plugin + err error + ) + for _, path := range args { + plugin, err = service.LoadPluginFromFile(path) + if err != nil { + log.Error().Err(err). + Str("path", path). + Msg("failed to load plugin from path") + continue + } + plugins = append(plugins, makeshift.PluginToMap(plugin)) } - if res.StatusCode != http.StatusOK { - log.Error(). - Any("status", map[string]any{ - "code": res.StatusCode, - "message": res.Status, - "body": string(body), - }). - Str("host", host). - Msg("response returned bad status") - os.Exit(1) - } - if outputPath != "" { - writeFiles(outputPath, body) - } else { - fmt.Println(string(body)) + log.Info().Any("plugins", plugins).Send() + } else { + for _, pluginName := range args { + query = fmt.Sprintf("/plugins/%s/info", pluginName) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("query", query). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + "body": string(body), + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) + } + if outputPath != "" { + writeFiles(outputPath, body) + } else { + fmt.Println(string(body)) + } } } }, @@ -180,6 +207,7 @@ var pluginsInfoCmd = &cobra.Command{ func init() { pluginsCompileCmd.Flags().StringP("output", "o", "", "Set the path to save compiled plugin (matches source type, i.e. uses files or directory)") pluginsInfoCmd.Flags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + pluginsInfoCmd.Flags().Bool("local", false, "Set whether to display information of a local plugin") pluginsCmd.AddCommand(pluginsCompileCmd, pluginsInspectCmd, pluginsInfoCmd) rootCmd.AddCommand(pluginsCmd) From cb3d4ce8db79206b080d851be8c5b7609a3ae5f1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 11:03:25 -0600 Subject: [PATCH 82/89] feat: added tip about local plugins info --- cmd/plugins.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/plugins.go b/cmd/plugins.go index 15043f0..99b8908 100644 --- a/cmd/plugins.go +++ b/cmd/plugins.go @@ -129,7 +129,7 @@ var pluginsInfoCmd = &cobra.Command{ # show information of a remote plugin makeshift plugins info jinja2 smd - # show information of a local plugin + # show information of a local plugin (same as 'makeshift inspect') makeshift plugins info --local $MAKESHIFT_ROOT/plugins/jinja2.so `, Short: "Show plugin information", From 8e1fa3d2ab759084c42276fdde8042a38cc827f9 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 11:48:49 -0600 Subject: [PATCH 83/89] feat: updated cmd/pkg implementations and cleanup --- cmd/upload.go | 109 ++++++++++++++++++++++++++++------------- pkg/service/plugins.go | 47 ++++++------------ 2 files changed, 88 insertions(+), 68 deletions(-) diff --git a/cmd/upload.go b/cmd/upload.go index 260ff3f..ef0c9c7 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -12,6 +12,7 @@ import ( "git.towk2.me/towk/makeshift/internal/format" makeshift "git.towk2.me/towk/makeshift/pkg" "git.towk2.me/towk/makeshift/pkg/client" + "git.towk2.me/towk/makeshift/pkg/service" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -32,9 +33,9 @@ var uploadCmd = &cobra.Command{ # upload an archive (extracted and saved on server) makeshift upload -d @setup.tar.gz -t archive - # upload a new profile with a specific path (used to set remote location) - makeshift upload profile -d @kubernetes.json -p nodes/kubernetes.json - makeshift upload profile -d @slurm.json -d @compute.json -p nodes + # upload multiple files with a specific path (used to set remote location) + makeshift upload -d @kubernetes.json -p nodes/kubernetes.json + makeshift upload -d @slurm.json -d @compute.json -p nodes `, Short: "Upload files and directories", PersistentPreRun: func(cmd *cobra.Command, args []string) { @@ -79,15 +80,15 @@ var uploadCmd = &cobra.Command{ } var uploadProfilesCmd = &cobra.Command{ - Use: "profile [profileID]", + Use: "profile [profile_id]", Example: ` # upload a new profile - makeshift upload profile -d @compute.json + makeshift upload profile -d @compute.json kubernetes.json - # upload a new profile with a specific path (used for lookup) - makeshift upload profile -d @kubernetes.json -n k8s + # upload a new profile with a specific path + makeshift upload profile -d @kubernetes.json + makeshift upload profile -d '{"id": "custom", "data": {}}' kubernetes.json `, - Args: cobra.NoArgs, Short: "Upload a new profile", Run: func(cmd *cobra.Command, args []string) { var ( @@ -102,6 +103,28 @@ var uploadProfilesCmd = &cobra.Command{ err error ) + // load files from args + for i, path := range args { + body, err = os.ReadFile(path) + if err != nil { + log.Error().Err(err). + Int("index", i). + Str("path", path). + Msg("failed to read profile file") + continue + } + var profile *makeshift.Profile + err = json.Unmarshal(body, &profile) + if err != nil { + log.Error().Err(err). + Int("index", i). + Str("path", path). + Msg("failed to unmarshal profile") + } + profiles = append(profiles, profile) + } + + // send each loaded profile to server for _, profile := range profiles { if profile == nil { continue @@ -113,42 +136,25 @@ var uploadProfilesCmd = &cobra.Command{ continue } - // send data to server query = fmt.Sprintf("/profiles/%s", profile.ID) res, body, err = c.MakeRequest(client.HTTPEnvelope{ Path: query, Method: http.MethodPost, Body: body, }) - if err != nil { - log.Error().Err(err). - Str("host", host). - Str("query", query). - Msg("failed to make request") - os.Exit(1) - } - if res.StatusCode != http.StatusOK { - log.Error(). - Any("status", map[string]any{ - "code": res.StatusCode, - "message": res.Status, - }). - Str("host", host). - Msg("response returned bad status") - os.Exit(1) - } + handleResponseError(res, host, query, err) } }, } var uploadPluginsCmd = &cobra.Command{ - Use: "plugin", + Use: "plugin [plugin_name]", Example: ` # upload a new plugin makeshift upload plugin -d @slurm.so # upload a new plugin with a specific name (used for lookups) - makeshift upload plugin -d @cobbler.so -n merge + makeshift upload plugin -d @cobbler.so `, Args: cobra.ExactArgs(1), Short: "Upload a new plugin", @@ -156,14 +162,47 @@ var uploadPluginsCmd = &cobra.Command{ // make one request be host positional argument (restricted to 1 for now) // temp := append(handleArgs(args), processDataArgs(dataArgs)...) var ( - inputData []*makeshift.Profile - dataArgs, _ = cmd.PersistentFlags().GetStringArray("data") + host, _ = cmd.Flags().GetString("host") + dataArgs, _ = cmd.Flags().GetStringArray("data") + + plugins = processFiles(dataArgs) + c = client.New(host) + res *http.Response + query string + body []byte + plugin makeshift.Plugin + err error ) - temp := processProfiles(dataArgs) - for _, data := range temp { - if data != nil { - inputData = append(inputData, data) + + // load files from args + for i, path := range args { + body, err = os.ReadFile(path) + if err != nil { + log.Error().Err(err). + Int("index", i). + Str("path", path). + Msg("failed to read plugin file") + continue } + + plugins[path] = body + } + + for path, contents := range plugins { + plugin, err = service.LoadPluginFromFile(path) + if err != nil { + log.Error().Err(err). + Str("path", path). + Msg("failed to load plugin from file") + } + + query = fmt.Sprintf("/plugins/%s", plugin.Name()) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodPost, + Body: contents, + }) + handleResponseError(res, host, query, err) } }, } @@ -207,7 +246,7 @@ func processFiles(args []string) map[string][]byte { // add loaded data to collection of all data collection[path] = contents } else { - log.Warn().Msg("only files can be uploaded") + log.Warn().Msg("only files can be uploaded (add @ before the path)") continue } diff --git a/pkg/service/plugins.go b/pkg/service/plugins.go index 0ab12eb..29d2f16 100644 --- a/pkg/service/plugins.go +++ b/pkg/service/plugins.go @@ -88,33 +88,33 @@ func (s *Service) GetPluginRaw() http.HandlerFunc { func (s *Service) CreatePlugin() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - plugin makeshift.Plugin - path string - err error + pluginName = chi.URLParam(r, "name") + body []byte + path string + err error ) - plugin, err = getPluginFromRequestBody(r) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - // helper to check for valid plugin name var hasValidName = func(name string) bool { return name != "" && len(name) < 64 } // check for a valid plugin name - if !hasValidName(plugin.Name()) { + if !hasValidName(pluginName) { + http.Error(w, "invalid name for plugin", http.StatusBadRequest) + return + } + + body, err = io.ReadAll(r.Body) + if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - // save plugin at path using it's name - path = s.PathForPluginWithName(plugin.Name()) - err = SavePluginToFile(path, &plugin) + path = s.PathForPluginWithName(pluginName) + err = os.WriteFile(path, body, 0o777) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -139,22 +139,3 @@ func (s *Service) DeletePlugin() http.HandlerFunc { w.WriteHeader(http.StatusOK) } } - -func getPluginFromRequestBody(r *http.Request) (makeshift.Plugin, error) { - var ( - plugin makeshift.Plugin - body []byte - err error - ) - body, err = io.ReadAll(r.Body) - if err != nil { - return nil, err - } - - err = json.Unmarshal(body, &plugin) - if err != nil { - return nil, err - } - - return plugin, nil -} From c9f40e3857dea11ddf18b2ad07197509cf63c081 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 11:56:11 -0600 Subject: [PATCH 84/89] chore: added server root section to README --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9104784..04a2749 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ The `makeshift` tool is a service that serves files and CLI that downloads them with a couple of handy features baked-in. Although the CLI and server component function more like a glorified FTP, the power of this tool comes from the plugin system. For example, the file cobbler is built to run external plugins for more advanced processing files before serving them (e.g. fetching from a data source, rendering Jinja 2 templates, etc.). -## Building the Tool +## Building and Go! The `makeshift` tool is built using standard `go` build tools. To get started, clone the project, download the dependencies, and build the project: @@ -121,6 +121,23 @@ makeshift download profile default > [!NOTE] > Although every command has a `curl` equivalent, it is better to use the CLI since it has other features such as extracting and remove archives after downloading and saving archives as files automatically. +## Server Root Structure + +The `makeshift` server serves files at the specified `--root` path (also set with `MAKESHIFT_ROOT` environment variable). The directory structure looks like the following by default with initializing with `makeshift init $MAKESHIFT_ROOT`. + +```bash +├── data +├── plugins +└── profiles +``` + +Each directory holds specific files for different purposes: + +- `data` - Stores any and all miscellaenous files and directories. +- `plugins` - Stores plugins defined in the ["Creating Plugins"](#creating-plugins) section. +- `profiles` - Stores profiles in JSON format as defined in the ["Creating Profiles"](#creating-profiles) section. + + ## Creating Plugins The `makeshift` tool defines a plugin as an interface that can be implemented and compiled. From c799dc78382c5ee5b6a110036158ffc1b8a48396 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 11:58:17 -0600 Subject: [PATCH 85/89] chore: updated examples in README and cmd --- README.md | 17 ++++++----------- cmd/upload.go | 4 +--- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 04a2749..916cd52 100644 --- a/README.md +++ b/README.md @@ -101,21 +101,16 @@ makeshift download profile default # upload an archive (extracted and saved on server - not working yet...) makeshift upload -d @setup.tar.gz -t archive - # upload a new profile with a specific path (used to set remote location) - makeshift upload profile -d @kubernetes.json -p nodes/kubernetes.json - makeshift upload profile -d @slurm.json -d @compute.json -p nodes + # upload a new profile + makeshift upload profile -d @compute.json kubernetes.json - # upload a new profile - makeshift upload profile -d @compute.json - - # upload a new profile with a specific path (used for lookup) - makeshift upload profile -d @kubernetes.json -n k8s + # upload a new profile with a specific path + makeshift upload profile -d @kubernetes.json + makeshift upload profile -d '{"id": "custom", "data": {}}' kubernetes.json # upload a new plugin makeshift upload plugin -d @slurm.so - - # upload a new plugin with a specific name (used for lookups) - makeshift upload plugin -d @cobbler.so -n merge + makeshift upload plugin slurm.so ``` > [!NOTE] diff --git a/cmd/upload.go b/cmd/upload.go index ef0c9c7..4472a18 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -152,9 +152,7 @@ var uploadPluginsCmd = &cobra.Command{ Example: ` # upload a new plugin makeshift upload plugin -d @slurm.so - - # upload a new plugin with a specific name (used for lookups) - makeshift upload plugin -d @cobbler.so + makeshift upload plugin slurm.so `, Args: cobra.ExactArgs(1), Short: "Upload a new plugin", From eac73ada69ad50f72ddada6afbe5902df9c092d1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 21:59:14 -0600 Subject: [PATCH 86/89] chore: removed TODO and update plugin struct --- README.md | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 916cd52..380d435 100644 --- a/README.md +++ b/README.md @@ -207,14 +207,12 @@ On the other hand, profiles are simply objects that contain data used to populat type Profile struct { ID string `json:"id"` // profile ID Description string `json:"description,omitempty"` // profile description - Tags []string `json:"tags,omitempty"` // tags used for ... - Paths []string `json:"paths,omitempty"` // paths to download - Plugins []string `json:"plugins,omitempty"` // plugins to run + Tags []string `json:"tags,omitempty"` // tags used for filtering (not implemented yet) Data map[string]any `json:"data,omitempty"` // include render data } ``` -Profiles can be created using JSON. See the example in `$MAKESHIFT_ROOT/profiles/default.json`. +Profiles can be created using JSON and only require an `id` with optional `data`. See the example in `$MAKESHIFT_ROOT/profiles/default.json`. ```json { @@ -235,12 +233,9 @@ Profiles can be created using JSON. See the example in `$MAKESHIFT_ROOT/profiles There are some features still missing that will be added later. -1. Uploading files and directories -2. Uploading new profiles and plugins -3. Running `makeshift` locally with profiles and plugins -4. Plugin to add user data for one-time use without creating a profile -5. Optionally build plugins directly into the main driver -6. Protected routes that require authentication -7. Configuration file for persistent runs -8. `Dockerfile` and `docker-compose.yml` files to build containers -9. Including certs with requests \ No newline at end of file +1. Running `makeshift` locally with profiles and plugins +2. Plugin to add user data for one-time use without creating a profile +3. Optionally build plugins directly into the main driver +4. Protected routes that require authentication +5. Configuration file for persistent runs +6. `Dockerfile` and `docker-compose.yml` files to build containers \ No newline at end of file From 2112e7eefd3ada3fd9fdc0023afc8a96eec60844 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 21:59:40 -0600 Subject: [PATCH 87/89] chore: changed permission of bin/ helper script --- bin/compile-plugins.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/compile-plugins.sh diff --git a/bin/compile-plugins.sh b/bin/compile-plugins.sh old mode 100644 new mode 100755 From bdd85b01ff04b986b0590204e228c607ed1c47f1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 22:02:10 -0600 Subject: [PATCH 88/89] feat: added cacerts and some tidying --- cmd/delete.go | 40 +++++++++- cmd/download.go | 17 ++++ cmd/list.go | 49 ++++++++++-- cmd/serve.go | 18 ++++- cmd/upload.go | 176 +++++++++++++++++++++++++++++++++-------- pkg/models.go | 8 ++ pkg/service/routes.go | 8 +- pkg/service/service.go | 22 +++--- 8 files changed, 279 insertions(+), 59 deletions(-) diff --git a/cmd/delete.go b/cmd/delete.go index 85dbd71..4bdb767 100644 --- a/cmd/delete.go +++ b/cmd/delete.go @@ -24,17 +24,29 @@ var deleteCmd = &cobra.Command{ PersistentPreRun: func(cmd *cobra.Command, args []string) { setenv(cmd, "host", "MAKESHIFT_HOST") setenv(cmd, "path", "MAKESHIFT_PATH") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") }, Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") - paths, _ = cmd.Flags().GetStringSlice("path") + host, _ = cmd.Flags().GetString("host") + paths, _ = cmd.Flags().GetStringSlice("path") + cacertPath, _ = cmd.Flags().GetString("cacert") c = client.New(host) res *http.Response query string err error ) + + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + for _, path := range paths { if path == "" { log.Warn().Msg("skipping empty path") @@ -61,7 +73,8 @@ var deleteProfilesCmd = &cobra.Command{ Short: "Delete profile(s)", Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") + host, _ = cmd.Flags().GetString("host") + cacertPath, _ = cmd.Flags().GetString("cacert") c = client.New(host) res *http.Response @@ -69,6 +82,14 @@ var deleteProfilesCmd = &cobra.Command{ err error ) + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } for _, profileID := range args { if profileID == "default" { log.Warn().Msg("cannot delete the default profile") @@ -95,7 +116,8 @@ var deletePluginsCmd = &cobra.Command{ Short: "Delete plugin(s)", Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") + host, _ = cmd.Flags().GetString("host") + cacertPath, _ = cmd.Flags().GetString("cacert") c = client.New(host) res *http.Response @@ -103,6 +125,15 @@ var deletePluginsCmd = &cobra.Command{ err error ) + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + for _, pluginName := range args { query = fmt.Sprintf("/plugins/%s", pluginName) res, _, err = c.MakeRequest(client.HTTPEnvelope{ @@ -116,6 +147,7 @@ var deletePluginsCmd = &cobra.Command{ func init() { deleteCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift server host (can be set with MAKESHIFT_HOST)") + deleteCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load") deleteCmd.Flags().StringSliceP("path", "p", []string{}, "Set the paths to delete files and directories") deleteCmd.AddCommand(deleteProfilesCmd, deletePluginsCmd) diff --git a/cmd/download.go b/cmd/download.go index 25e87aa..03bf061 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -37,12 +37,14 @@ var downloadCmd = cobra.Command{ PersistentPreRun: func(cmd *cobra.Command, args []string) { setenv(cmd, "host", "MAKESHIFT_HOST") setenv(cmd, "path", "MAKESHIFT_PATH") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") }, Run: func(cmd *cobra.Command, args []string) { var ( host, _ = cmd.Flags().GetString("host") path, _ = cmd.Flags().GetString("path") outputPath, _ = cmd.Flags().GetString("output") + cacertPath, _ = cmd.Flags().GetString("cacert") pluginNames, _ = cmd.Flags().GetStringSlice("plugins") profileIDs, _ = cmd.Flags().GetStringSlice("profiles") extract, _ = cmd.Flags().GetBool("extract") @@ -72,6 +74,10 @@ var downloadCmd = cobra.Command{ Strs("plugins", pluginNames). Send() + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + res, body, err = c.MakeRequest(client.HTTPEnvelope{ Path: query, Method: http.MethodGet, @@ -163,6 +169,7 @@ var downloadProfileCmd = &cobra.Command{ var ( host, _ = cmd.Flags().GetString("host") outputPath, _ = cmd.Flags().GetString("output") + cacertPath, _ = cmd.Flags().GetString("cacert") c = client.New(host) res *http.Response @@ -176,6 +183,10 @@ var downloadProfileCmd = &cobra.Command{ Str("output", outputPath). Send() + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + for _, profileID := range args { query = fmt.Sprintf("/profiles/%s", profileID) res, body, err = c.MakeRequest(client.HTTPEnvelope{ @@ -219,6 +230,7 @@ var downloadPluginCmd = &cobra.Command{ var ( host, _ = cmd.Flags().GetString("host") outputPath, _ = cmd.Flags().GetString("output") + cacertPath, _ = cmd.Flags().GetString("cacert") c = client.New(host) res *http.Response @@ -232,6 +244,10 @@ var downloadPluginCmd = &cobra.Command{ Str("output", outputPath). Send() + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + for _, pluginName := range args { query = fmt.Sprintf("/plugins/%s/raw", pluginName) res, body, err = c.MakeRequest(client.HTTPEnvelope{ @@ -267,6 +283,7 @@ var downloadPluginCmd = &cobra.Command{ func init() { downloadCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") downloadCmd.PersistentFlags().StringP("output", "o", "", "Set the output path to write files") + downloadCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load") downloadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") downloadCmd.Flags().StringSlice("profiles", []string{}, "Set the profile(s) to use to populate data store") downloadCmd.Flags().StringSlice("plugins", []string{}, "Set the plugin(s) to run before downloading files") diff --git a/cmd/list.go b/cmd/list.go index 1fa3fe6..4c5898c 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -24,14 +24,16 @@ var listCmd = &cobra.Command{ `, Args: cobra.NoArgs, Short: "List all files in a remote data directory", - PreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRun: func(cmd *cobra.Command, args []string) { setenv(cmd, "host", "MAKESHIFT_HOST") setenv(cmd, "path", "MAKESHIFT_PATH") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") }, Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") - path, _ = cmd.Flags().GetString("path") + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + cacertPath, _ = cmd.Flags().GetString("cacert") c = client.New(host) body []byte @@ -42,8 +44,13 @@ var listCmd = &cobra.Command{ log.Debug(). Str("host", host). Str("path", path). + Str("cacert", cacertPath). Send() + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + // make request to /list endpoint _, body, err = c.MakeRequest(client.HTTPEnvelope{ Path: fmt.Sprintf("/list/%s", path), @@ -80,7 +87,8 @@ var listPluginsCmd = &cobra.Command{ Short: "Show plugins information", Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") + host, _ = cmd.Flags().GetString("host") + cacertPath, _ = cmd.Flags().GetString("cacert") c = client.New(host) res *http.Response @@ -90,6 +98,15 @@ var listPluginsCmd = &cobra.Command{ err error ) + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + if len(args) == 0 { // make request to /list endpoint res, body, err = c.MakeRequest(client.HTTPEnvelope{ @@ -121,11 +138,19 @@ var listPluginsCmd = &cobra.Command{ } var listProfilesCmd = &cobra.Command{ - Use: "profiles", + Use: "profiles", + Example: ` + # list all profiles + makeshift list profiles + + # live individual profiles + makeshift list profiles default custom +`, Short: "Show all available profiles", Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") + host, _ = cmd.Flags().GetString("host") + cacertPath, _ = cmd.Flags().GetString("cacert") c = client.New(host) res *http.Response @@ -135,6 +160,15 @@ var listProfilesCmd = &cobra.Command{ err error ) + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + if len(args) == 0 { // make request to /list endpoint res, body, err = c.MakeRequest(client.HTTPEnvelope{ @@ -154,7 +188,7 @@ var listProfilesCmd = &cobra.Command{ // make request to /list endpoint query = fmt.Sprintf("/profiles/%s", profileID) res, body, err = c.MakeRequest(client.HTTPEnvelope{ - Path: fmt.Sprintf(query), + Path: query, Method: http.MethodGet, }) handleResponseError(res, host, query, err) @@ -174,6 +208,7 @@ var listProfilesCmd = &cobra.Command{ func init() { listCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the configurator remote host (can be set with MAKESHIFT_HOST)") + listCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load") listCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") listCmd.AddCommand(listPluginsCmd, listProfilesCmd) diff --git a/cmd/serve.go b/cmd/serve.go index 22c793c..9c8c046 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -23,12 +23,16 @@ var serveCmd = &cobra.Command{ setenv(cmd, "host", "MAKESHIFT_HOST") setenv(cmd, "root", "MAKESHIFT_ROOT") setenv(cmd, "timeout", "MAKESHIFT_TIMEOUT") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") + setenv(cmd, "keyfile", "MAKESHIFT_KEYFILE") }, Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") - rootPath, _ = cmd.Flags().GetString("root") - timeout, _ = cmd.Flags().GetInt("timeout") + host, _ = cmd.Flags().GetString("host") + rootPath, _ = cmd.Flags().GetString("root") + cacertPath, _ = cmd.Flags().GetString("cacert") + keyfile, _ = cmd.Flags().GetString("keyfile") + timeout, _ = cmd.Flags().GetInt("timeout") parsed *url.URL server *service.Service @@ -47,6 +51,8 @@ var serveCmd = &cobra.Command{ server = service.New() server.Addr = parsed.Host server.RootPath = rootPath + server.CACertFile = cacertPath + server.CACertKeyfile = keyfile server.Timeout = time.Duration(timeout) * time.Second // show some debugging information @@ -54,6 +60,8 @@ var serveCmd = &cobra.Command{ Str("host", parsed.Host). Any("paths", map[string]string{ "root": rootPath, + "cacert": cacertPath, + "keyfile": keyfile, "data": server.PathForData(), "profiles": server.PathForProfiles(), "plugins": server.PathForPlugins(), @@ -84,6 +92,10 @@ func init() { serveCmd.Flags().String("host", "localhost:5050", "Set the configurator server host (can be set with MAKESHIFT_HOST)") serveCmd.Flags().String("root", "./", "Set the root path to serve files (can be set with MAKESHIFT_ROOT)") serveCmd.Flags().IntP("timeout", "t", 60, "Set the timeout in seconds for requests (can be set with MAKESHIFT_TIMEOUT)") + serveCmd.Flags().String("cacert", "", "Set the CA certificate path to load (can be set with MAKESHIFT_CACERT)") + serveCmd.Flags().String("keyfile", "", "Set the CA key file to use (can be set with MAKESHIFT_KEYFILE)") + + serveCmd.MarkFlagsRequiredTogether("cacert", "keyfile") rootCmd.AddCommand(serveCmd) } diff --git a/cmd/upload.go b/cmd/upload.go index 4472a18..b4d09cb 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -4,6 +4,8 @@ import ( "bufio" "encoding/json" "fmt" + "io/fs" + "maps" "net/http" "os" "path/filepath" @@ -41,12 +43,14 @@ var uploadCmd = &cobra.Command{ PersistentPreRun: func(cmd *cobra.Command, args []string) { setenv(cmd, "host", "MAKESHIFT_HOST") setenv(cmd, "path", "MAKESHIFT_PATH") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") }, Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") - path, _ = cmd.Flags().GetString("path") - dataArgs, _ = cmd.Flags().GetStringArray("data") + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + cacertPath, _ = cmd.Flags().GetString("cacert") + dataArgs, _ = cmd.Flags().GetStringArray("data") inputData = processFiles(dataArgs) useDirectoryPath = len(inputData) > 1 @@ -55,8 +59,21 @@ var uploadCmd = &cobra.Command{ query string err error ) + + log.Debug(). + Str("host", host). + Str("path", path). + Str("query", query). + Str("cacert", cacertPath). + Any("input", inputData). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + for inputPath, contents := range inputData { - log.Info().Str("path", path).Int("size", len(contents)).Send() + log.Debug().Str("path", path).Int("size", len(contents)).Send() if useDirectoryPath { query = path + "/" + filepath.Clean(inputPath) } else { @@ -92,9 +109,10 @@ var uploadProfilesCmd = &cobra.Command{ Short: "Upload a new profile", Run: func(cmd *cobra.Command, args []string) { var ( - host, _ = cmd.Flags().GetString("host") - dataArgs, _ = cmd.Flags().GetStringArray("data") - profiles = processProfiles(dataArgs) + host, _ = cmd.Flags().GetString("host") + dataArgs, _ = cmd.Flags().GetStringArray("data") + cacertPath, _ = cmd.Flags().GetString("cacert") + profiles = processProfiles(dataArgs) c = client.New(host) res *http.Response @@ -103,6 +121,16 @@ var uploadProfilesCmd = &cobra.Command{ err error ) + log.Debug(). + Str("host", host). + Str("query", query). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + // load files from args for i, path := range args { body, err = os.ReadFile(path) @@ -160,8 +188,9 @@ var uploadPluginsCmd = &cobra.Command{ // make one request be host positional argument (restricted to 1 for now) // temp := append(handleArgs(args), processDataArgs(dataArgs)...) var ( - host, _ = cmd.Flags().GetString("host") - dataArgs, _ = cmd.Flags().GetStringArray("data") + host, _ = cmd.Flags().GetString("host") + dataArgs, _ = cmd.Flags().GetStringArray("data") + cacertPath, _ = cmd.Flags().GetString("cacert") plugins = processFiles(dataArgs) c = client.New(host) @@ -172,6 +201,16 @@ var uploadPluginsCmd = &cobra.Command{ err error ) + log.Debug(). + Str("host", host). + Str("query", query). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + // load files from args for i, path := range args { body, err = os.ReadFile(path) @@ -208,6 +247,8 @@ var uploadPluginsCmd = &cobra.Command{ func init() { uploadCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") uploadCmd.PersistentFlags().StringArrayP("data", "d", []string{}, "Set the data to send to specified host (prepend @ for files)") + uploadCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load") + uploadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") uploadProfilesCmd.Flags().VarP(&inputFormat, "format", "F", "Set the input format for profile") @@ -220,31 +261,27 @@ func processFiles(args []string) map[string][]byte { // load data either from file or directly from args var collection = make(map[string][]byte, len(args)) for _, arg := range args { - // if arg is empty string, then skip and continue + // skip empty string args if len(arg) > 0 { // determine if we're reading from file to load contents if strings.HasPrefix(arg, "@") { - var ( - path string = strings.TrimLeft(arg, "@") - contents []byte - err error - ) - contents, err = os.ReadFile(path) + var path string = strings.TrimLeft(arg, "@") + + // process sub-directories recursively + newCollection, err := processDir(path) if err != nil { - log.Error().Err(err).Str("path", path).Msg("failed to read file") - continue + log.Warn(). + Err(err). + Str("path", path). + Msg("failed to process directory at path") } + log.Trace(). + Str("path", path). + Msg("new collection added at path") + maps.Copy(collection, newCollection) - // skip empty files - if len(contents) == 0 { - log.Warn().Str("path", path).Msg("file is empty") - continue - } - - // add loaded data to collection of all data - collection[path] = contents } else { - log.Warn().Msg("only files can be uploaded (add @ before the path)") + log.Warn().Msg("only files can be uploaded (add @ before the path with '--data' flag)") continue } @@ -275,20 +312,28 @@ func processProfiles(args []string) []*makeshift.Profile { ) contents, err = os.ReadFile(path) if err != nil { - log.Error().Err(err).Str("path", path).Msg("failed to read file") + log.Error(). + Err(err). + Str("path", path). + Msg("failed to read file") continue } // skip empty files if len(contents) == 0 { - log.Warn().Str("path", path).Msg("file is empty") + log.Warn(). + Str("path", path). + Msg("file is empty") continue } // convert/validate input data data, err = parseProfile(contents, format.DataFormatFromFileExt(path, inputFormat)) if err != nil { - log.Error().Err(err).Str("path", path).Msg("failed to validate input from file") + log.Error(). + Err(err). + Str("path", path). + Msg("failed to validate input from file") } // add loaded data to collection of all data @@ -306,7 +351,9 @@ func processProfiles(args []string) []*makeshift.Profile { } err = json.Unmarshal(input, &data) if err != nil { - log.Error().Err(err).Msgf("failed to unmarshal input for argument %d", i) + log.Error(). + Err(err). + Msgf("failed to unmarshal input for argument %d", i) } return []*makeshift.Profile{data} } @@ -315,6 +362,73 @@ func processProfiles(args []string) []*makeshift.Profile { return collection } +func processDir(path string) (map[string][]byte, error) { + var ( + collection = map[string][]byte{} + fileInfo os.FileInfo + contents []byte + err error + ) + // determine if path is directory + if fileInfo, err = os.Stat(path); err == nil { + if fileInfo.IsDir() { + filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + contents, err = os.ReadFile(path) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("failed to read file") + return nil + } + + // skip empty files + if len(contents) == 0 { + log.Warn().Str("path", path).Msg("file is empty") + return nil + } + + log.Debug(). + Str("path", path). + Msg("file added to collection") + + // add loaded data to collection of all data + collection[path] = contents + } else { + // process sub-directories recursively + newCollection, err := processDir(path) + if err != nil { + return fmt.Errorf("failed to process directory at path '%s': %v", path, err) + } + log.Trace(). + Str("path", path). + Msg("new collection added from nested directory") + maps.Copy(collection, newCollection) + } + return nil + }) + } else { + contents, err = os.ReadFile(path) + if err != nil { + return collection, fmt.Errorf("failed to read file at path '%s': %v", path, err) + } + + // skip empty files + if len(contents) == 0 { + return collection, fmt.Errorf("file is empty") + } + + log.Debug(). + Str("path", path). + Msg("file added to collection") + + // add loaded data to collection of all data + collection[path] = contents + } + } else { + return nil, fmt.Errorf("failed to stat file: %v", err) + } + return collection, nil +} + func parseProfile(contents []byte, dataFormat format.DataFormat) (*makeshift.Profile, error) { var ( data *makeshift.Profile diff --git a/pkg/models.go b/pkg/models.go index 1b45444..ff2e823 100644 --- a/pkg/models.go +++ b/pkg/models.go @@ -29,10 +29,18 @@ type Hook struct { Plugin Plugin } +func (h *Hook) Init() error { + return h.Plugin.Init() +} + func (h *Hook) Run() error { return h.Plugin.Run(h.Data, h.Args) } +func (h *Hook) Cleanup() error { + return h.Plugin.Cleanup() +} + func PluginToMap(p Plugin) map[string]any { return map[string]any{ "name": p.Name(), diff --git a/pkg/service/routes.go b/pkg/service/routes.go index b5dd890..d63201d 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -190,7 +190,9 @@ func (s *Service) Upload() http.HandlerFunc { ) // show what we're uploading - log.Debug().Str("path", path).Msg("Service.Upload()") + log.Debug(). + Str("path", path). + Msg("Service.Upload()") // take the provided path and store the file contents dirpath = filepath.Dir(path) @@ -296,7 +298,7 @@ func (s *Service) loadProfiles(profileIDs []string, store storage.KVStore, errs profile *makeshift.Profile err error ) - if i > DEFAULT_PROFILES_MAX_COUNT { + if i > s.ProfilesMaxCount { log.Warn().Msg("max profiles count reached...stopping") return errs } @@ -329,7 +331,7 @@ func (s *Service) loadPlugins(pluginNames []string, store storage.KVStore, args plugin makeshift.Plugin err error ) - if i > DEFAULT_PLUGINS_MAX_COUNT { + if i > s.PluginsMaxCount { log.Warn().Msg("max plugins count reached or exceeded...stopping") return hooks, errs } diff --git a/pkg/service/service.go b/pkg/service/service.go index 362a08c..7197876 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -19,9 +19,10 @@ import ( ) type Service struct { - Addr string - RootPath string `yaml:"root,omitempty"` - Environment map[string]string + Addr string + RootPath string `yaml:"root,omitempty"` + CACertFile string `yaml:"cacert,omitempty"` + CACertKeyfile string `yaml:"keyfile,omitempty"` // max counts PluginsMaxCount int @@ -32,13 +33,8 @@ type Service struct { // New creates a new Service instance with default values func New() *Service { return &Service{ - Addr: ":5050", - RootPath: "./", - Environment: map[string]string{ - "MAKESHIFT_HOST": "", - "MAKESHIFT_ROOT": "", - "ACCESS_TOKEN": "", - }, + Addr: ":5050", + RootPath: "./", PluginsMaxCount: DEFAULT_PLUGINS_MAX_COUNT, ProfilesMaxCount: DEFAULT_PROFILES_MAX_COUNT, Timeout: DEFAULT_TIMEOUT_IN_SECS, @@ -121,7 +117,11 @@ func (s *Service) Serve() error { // always available public routes go here router.HandleFunc("/status", s.GetStatus) - return http.ListenAndServe(s.Addr, router) + if s.CACertFile != "" && s.CACertKeyfile != "" { + return http.ListenAndServeTLS(s.Addr, s.CACertFile, s.CACertKeyfile, router) + } else { + return http.ListenAndServe(s.Addr, router) + } } func (s *Service) requireAuth() bool { From 277de43a0212e8efc901d0e2d2d390bad63842f9 Mon Sep 17 00:00:00 2001 From: David Allen Date: Sun, 31 Aug 2025 22:04:02 -0600 Subject: [PATCH 89/89] feat: added Init() and Cleanup() in hooks --- internal/archive/archive.go | 8 ++++++++ pkg/service/routes.go | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index eaf6412..4fadf22 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -133,10 +133,18 @@ func addToArchive(tw *tar.Writer, filename string, hooks []makeshift.Hook) error } hook.Data.Set("file", contents) + err = hook.Init() + if err != nil { + return err + } err = hook.Run() if err != nil { return err } + err = hook.Cleanup() + if err != nil { + return err + } // create temporary file to use to add to archive hook = hooks[len(hooks)-1] diff --git a/pkg/service/routes.go b/pkg/service/routes.go index d63201d..7d371c9 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -142,6 +142,14 @@ func (s *Service) Download() http.HandlerFunc { "version": hook.Plugin.Version(), }, }).Send() + err = hook.Init() + if err != nil { + log.Error(). + Err(err). + Str("plugin", hook.Plugin.Name()). + Msg("failed to initialize plugin") + continue + } err = hook.Run() if err != nil { log.Error(). @@ -150,6 +158,14 @@ func (s *Service) Download() http.HandlerFunc { Msg("failed to run plugin") continue } + err = hook.Cleanup() + if err != nil { + log.Error(). + Err(err). + Str("plugin", hook.Plugin.Name()). + Msg("failed to cleanup plugin") + continue + } } // take the contents from the last hook and update files