diff --git a/README.md b/README.md index b6b6a6b..2ce09ce 100644 --- a/README.md +++ b/README.md @@ -239,5 +239,9 @@ Profiles can be created using JSON and only require an `id` with optional `data` There are some features still missing that will be added later. -1. Optionally build plugins directly into the main driver -2. Protected routes that require authentication +1. Running `makeshift` locally with profiles and plugins +2. Plugin to add user data for one-time use without creating a profile +3. Optionally build plugins directly into the main driver +4. Protected routes that require authentication +5. Configuration file for persistent runs +6. `Dockerfile` and `docker-compose.yml` files to build containers \ No newline at end of file diff --git a/cmd/download.go b/cmd/download.go index c19a783..6c50c3f 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -1,7 +1,6 @@ package cmd import ( - "encoding/base64" "fmt" "net/http" "net/url" @@ -16,7 +15,10 @@ import ( "github.com/spf13/cobra" ) -var pluginKWArgs kwargs.KWArgs = kwargs.KWArgs{} +var ( + pluginArgs []string + pluginKWArgs kwargs.KWArgs = kwargs.KWArgs{} +) var downloadCmd = cobra.Command{ Use: "download", Example: ` @@ -50,7 +52,6 @@ var downloadCmd = cobra.Command{ configPath, _ = cmd.Flags().GetString("config") cacertPath, _ = cmd.Flags().GetString("cacert") pluginNames, _ = cmd.Flags().GetStringSlice("plugins") - pluginArgs, _ = cmd.Flags().GetStringSlice("plugin-args") profileIDs, _ = cmd.Flags().GetStringSlice("profiles") extract, _ = cmd.Flags().GetBool("extract") removeArchive, _ = cmd.Flags().GetBool("remove-archive") @@ -74,7 +75,7 @@ var downloadCmd = cobra.Command{ query += "&args=" + url.QueryEscape(strings.Join(pluginArgs, ",")) } if len(pluginKWArgs) > 0 { - query += "&kwargs=" + base64.RawURLEncoding.EncodeToString(pluginKWArgs.Bytes()) + query += "&kwargs=" + url.QueryEscape(pluginKWArgs.String()) } log.Debug(). @@ -85,8 +86,6 @@ var downloadCmd = cobra.Command{ Str("output", outputPath). Strs("profiles", profileIDs). Strs("plugins", pluginNames). - Strs("args", pluginArgs). - Any("kwargs", pluginKWArgs). Send() if cacertPath != "" { diff --git a/cmd/run.go b/cmd/run.go index e738e26..846e560 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -44,7 +44,6 @@ var runCmd = &cobra.Command{ keyfile, _ = cmd.Flags().GetString("keyfile") timeout, _ = cmd.Flags().GetInt("timeout") pluginNames, _ = cmd.Flags().GetStringSlice("plugins") - pluginArgs, _ = cmd.Flags().GetStringSlice("plugin-args") profileIDs, _ = cmd.Flags().GetStringSlice("profiles") extract, _ = cmd.Flags().GetBool("extract") removeArchive, _ = cmd.Flags().GetBool("remove-archive") diff --git a/cmd/serve.go b/cmd/serve.go index fba854b..b723a02 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -94,10 +94,8 @@ func init() { serveCmd.Flags().String("host", "localhost:5050", "Set the configurator server host (can be set with MAKESHIFT_HOST)") serveCmd.Flags().String("root", "./", "Set the root path to serve files (can be set with MAKESHIFT_ROOT)") serveCmd.Flags().IntP("timeout", "t", 60, "Set the timeout in seconds for requests (can be set with MAKESHIFT_TIMEOUT)") - serveCmd.Flags().String("cacert", "", "Set the CA certificate path to load (can be set with MAKESHIFT_CACERT, only used if set with '--keyfile' flag)") - serveCmd.Flags().String("keyfile", "", "Set the CA key file to use (can be set with MAKESHIFT_KEYFILE, only used if set with '--cacert' flag)") - serveCmd.Flags().String("keyurl", "", "Set the JWKS remote host for JWT verification") - serveCmd.Flags().StringSlice("protect-routes", []string{}, "Set the routes to require authentication (uses default routes if not set with '--keyurl' flag)") + serveCmd.Flags().String("cacert", "", "Set the CA certificate path to load (can be set with MAKESHIFT_CACERT)") + serveCmd.Flags().String("keyfile", "", "Set the CA key file to use (can be set with MAKESHIFT_KEYFILE)") serveCmd.MarkFlagsRequiredTogether("cacert", "keyfile") diff --git a/internal/format/format.go b/internal/format/format.go index 11cfa56..bdbfea0 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "path/filepath" - "strings" "gopkg.in/yaml.v3" ) @@ -93,11 +92,11 @@ func Unmarshal(data []byte, v any, inFormat DataFormat) error { // both the standard default format (JSON) and any command line // change to that provided by options. func DataFormatFromFileExt(path string, defaultFmt DataFormat) DataFormat { - switch strings.TrimLeft(strings.ToLower(filepath.Ext(path)), ".") { - case JSON.String(): + switch filepath.Ext(path) { + case ".json", ".JSON": // The file is a JSON file return JSON - case YAML.String(), "yml": + case ".yaml", ".yml", ".YAML", ".YML": // The file is a YAML file return YAML } diff --git a/internal/kwargs/kwargs.go b/internal/kwargs/kwargs.go index 0c8911f..1da80bd 100644 --- a/internal/kwargs/kwargs.go +++ b/internal/kwargs/kwargs.go @@ -5,28 +5,15 @@ import ( "fmt" "git.towk2.me/towk/makeshift/internal/format" - "github.com/rs/zerolog/log" ) const RESERVED_KEY = "kwargs" type KWArgs map[string]any -func New() KWArgs { - return KWArgs{} -} - func (kw KWArgs) String() string { - return string(kw.Bytes()) -} - -func (kw KWArgs) Bytes() []byte { - b, err := json.Marshal(kw) - if err != nil { - log.Error().Err(err).Msg("failed to marshal kwargs") - return []byte("{}") - } - return b + b, _ := json.Marshal(kw) + return string(b) } func (kw *KWArgs) Set(v string /* should be JSON object*/) error { diff --git a/pkg/plugins/jinja2/jinja2.go b/pkg/plugins/jinja2/jinja2.go index 6282d7c..daf6b20 100644 --- a/pkg/plugins/jinja2/jinja2.go +++ b/pkg/plugins/jinja2/jinja2.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" - "git.towk2.me/towk/makeshift/internal/kwargs" makeshift "git.towk2.me/towk/makeshift/pkg" "git.towk2.me/towk/makeshift/pkg/storage" "github.com/nikolalohinski/gonja/v2" @@ -43,11 +42,10 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { mappings struct { Data map[string]any `json:"data"` } - userdata *kwargs.KWArgs context *exec.Context template *exec.Template profiles any // makeshift.ProfileMap - contents any // []byte + input any // []byte output bytes.Buffer err error ) @@ -58,26 +56,18 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { Int("arg_count", len(args)). Msg("(jinja2) Run()") - // get profile data used as variable `{{ makeshift.profiles }}` profiles, err = store.Get("profiles") if err != nil { return fmt.Errorf("(jinja2) failed to get profiles: %v", err) } - // get userdata used as variable `{{ makeshift.userdata }}` - userdata, err = store.GetKWArgs() - if err != nil { - return fmt.Errorf("(jinja2) failed to get key-word arguments: %v", err) - } - - // get file contents used for templating - contents, err = store.Get("file") + input, err = store.Get("file") if err != nil { return fmt.Errorf("(jinja2) failed to get input data: %v", err) } // get the templates provided as args to the plugin - template, err = gonja.FromBytes(contents.([]byte)) + template, err = gonja.FromBytes(input.([]byte)) if err != nil { return fmt.Errorf("(jinja2) failed to get template from args: %v", err) } @@ -93,7 +83,6 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { } } - // get mappings from provided profiles `{{ makeshift.plugin.*}}` var ps = make(map[string]any) for profileID, profile := range profiles.(makeshift.ProfileMap) { ps[profileID] = map[string]any{ @@ -107,7 +96,6 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { mappings.Data = map[string]any{ "makeshift": map[string]any{ "profiles": ps, - "userdata": userdata, "plugin": map[string]any{ "name": p.Name(), "version": p.Version(), diff --git a/pkg/plugins/mapper/mapper.go b/pkg/plugins/mapper/mapper.go new file mode 100644 index 0000000..66b4a5d --- /dev/null +++ b/pkg/plugins/mapper/mapper.go @@ -0,0 +1,39 @@ +package main + +import ( + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/storage" +) + +type Mapper struct{} + +func (p *Mapper) Name() string { return "mapper" } +func (p *Mapper) Version() string { return "v0.0.1-alpha" } +func (p *Mapper) Description() string { return "Directly maps data to store" } +func (p *Mapper) Metadata() makeshift.Metadata { + return makeshift.Metadata{ + "author": map[string]any{ + "name": "David J. Allen", + "email": "davidallendj@gmail.com", + "links": []string{ + "https://github.com/davidallendj", + "https://git.towk2.me/towk", + }, + }, + } +} + +func (p *Mapper) Init() error { + // nothing to initialize + return nil +} + +func (p *Mapper) Run(data storage.KVStore, args []string) error { + return nil +} + +func (p *Mapper) Clean() error { + return nil +} + +var Makeshift Mapper diff --git a/pkg/plugins/pyjinja2/pyjinja2.go b/pkg/plugins/pyjinja2/pyjinja2.go index f625c26..673f1a1 100644 --- a/pkg/plugins/pyjinja2/pyjinja2.go +++ b/pkg/plugins/pyjinja2/pyjinja2.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" - "git.towk2.me/towk/makeshift/internal/kwargs" makeshift "git.towk2.me/towk/makeshift/pkg" "git.towk2.me/towk/makeshift/pkg/storage" jinja2 "github.com/kluctl/kluctl/lib/go-jinja2" @@ -43,7 +42,6 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { mappings struct { Data map[string]any `json:"data"` } - userdata *kwargs.KWArgs profiles any // makeshift.ProfileMap input any // []byte output string @@ -56,19 +54,11 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { Int("arg_count", len(args)). Msg("(pyjinja2) Run()") - // get profile data used as variable `{{ makeshift.profiles }}` profiles, err = store.Get("profiles") if err != nil { return fmt.Errorf("(pyjinja2) failed to get profiles: %v", err) } - // get userdata used as variable `{{ makeshift.userdata }}` - userdata, err = store.GetKWArgs() - if err != nil { - return fmt.Errorf("(pyjinja2) failed to get key-word arguments: %v", err) - } - - // get file contents used for templating input, err = store.Get("file") if err != nil { return fmt.Errorf("(pyjinja2) failed to get input data: %v", err) @@ -85,7 +75,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { } } - // get mappings from provided profiles `{{ makeshift.plugin.*}}` + // get mappings from provided profiles var ps = make(map[string]any) for profileID, profile := range profiles.(makeshift.ProfileMap) { ps[profileID] = map[string]any{ @@ -99,7 +89,6 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { mappings.Data = map[string]any{ "makeshift": map[string]any{ "profiles": ps, - "userdata": userdata, "plugin": map[string]any{ "name": p.Name(), "version": p.Version(), diff --git a/pkg/plugins/user/user.go b/pkg/plugins/user/user.go new file mode 100644 index 0000000..ff59640 --- /dev/null +++ b/pkg/plugins/user/user.go @@ -0,0 +1,36 @@ +package main + +import ( + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/storage" +) + +type User struct{} + +func (p *User) Name() string { return "user" } +func (p *User) Version() string { return "v0.0.1-alpha" } +func (p *User) Description() string { return "Get user information" } +func (p *User) Metadata() makeshift.Metadata { + return makeshift.Metadata{ + "author": map[string]any{ + "name": "David J. Allen", + "email": "davidallendj@gmail.com", + "links": []string{ + "https://github.com/davidallendj", + "https://git.towk2.me/towk", + }, + }, + } +} + +func (p *User) Init() error { + return nil +} + +func (p *User) Run(store storage.KVStore, args []string) error { + return nil +} + +func (p *User) Cleanup() error { + return nil +} diff --git a/pkg/service/routes.go b/pkg/service/routes.go index faa0236..e4fdb04 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -1,7 +1,6 @@ package service import ( - "encoding/base64" "encoding/json" "fmt" "io" @@ -17,63 +16,44 @@ import ( makeshift "git.towk2.me/towk/makeshift/pkg" "git.towk2.me/towk/makeshift/pkg/storage" "git.towk2.me/towk/makeshift/pkg/util" + "github.com/go-chi/chi/v5" "github.com/rs/zerolog/log" ) func (s *Service) Download() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download") - pluginArgs = strings.Split(r.URL.Query().Get("args"), ",") - pluginNames = strings.Split(r.URL.Query().Get("plugins"), ",") - profileIDs = strings.Split(r.URL.Query().Get("profiles"), ",") + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download") + pluginKWArgs = chi.URLParam(r, "kwargs") + pluginArgs = strings.Split(r.URL.Query().Get("args"), ",") + pluginNames = strings.Split(r.URL.Query().Get("plugins"), ",") + profileIDs = strings.Split(r.URL.Query().Get("profiles"), ",") - kw *kwargs.KWArgs = new(kwargs.KWArgs) + kw *kwargs.KWArgs fileInfo os.FileInfo out *os.File store *storage.MemoryStorage = new(storage.MemoryStorage) hooks []makeshift.Hook contents []byte - decoded []byte errs []error err error ) // parse the KWArgs from request - decoded, err = base64.RawURLEncoding.DecodeString(r.URL.Query().Get("kwargs")) - if err != nil { - s.writeErrorResponse(w, err.Error(), http.StatusBadRequest) - return - } + kw.Set(pluginKWArgs) + + // initialize storage + store.Init() + store.SetKWArgs(kw) log.Debug(). Str("path", path). Str("client_host", r.Host). Strs("plugins", pluginNames). Strs("profiles", profileIDs). - Strs("args", pluginArgs). - Str("kwargs", string(decoded)). Any("query", r.URL.Query()). Msg("Service.Download()") - err = kw.Set(string(decoded)) - if err != nil { - s.writeErrorResponse(w, err.Error(), http.StatusBadRequest) - return - } - - // initialize storage - err = store.Init() - if err != nil { - s.writeErrorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - err = store.SetKWArgs(kw) - if err != nil { - s.writeErrorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - // prepare profiles errs = s.LoadProfiles(profileIDs, store, errs) if len(errs) > 0 { diff --git a/pkg/storage/memory.go b/pkg/storage/memory.go index 76874c1..a3b1abb 100644 --- a/pkg/storage/memory.go +++ b/pkg/storage/memory.go @@ -20,15 +20,11 @@ func (ms *MemoryStorage) Cleanup() error { } func (ms *MemoryStorage) SetKWArgs(kw *kwargs.KWArgs) error { - ms.Data[kwargs.RESERVED_KEY] = kw - return nil + return ms.Set(kwargs.RESERVED_KEY, kw) } func (ms *MemoryStorage) GetKWArgs() (*kwargs.KWArgs, error) { kw, err := ms.Get(kwargs.RESERVED_KEY) - if err != nil { - return nil, err - } return kw.(*kwargs.KWArgs), err } @@ -41,7 +37,7 @@ func (ms *MemoryStorage) Get(k string) (any, error) { } func (ms *MemoryStorage) Set(k string, v any) error { - if k == kwargs.RESERVED_KEY { + if k == "kwargs" { return fmt.Errorf("cannot set reserved key '%s' (use SetKWArgs() instead)", k) } ms.Data[k] = v diff --git a/pkg/storage/memory_test.go b/pkg/storage/memory_test.go deleted file mode 100644 index 82be054..0000000 --- a/pkg/storage/memory_test.go +++ /dev/null @@ -1 +0,0 @@ -package storage