diff --git a/README.md b/README.md index 2ce09ce..b6b6a6b 100644 --- a/README.md +++ b/README.md @@ -239,9 +239,5 @@ Profiles can be created using JSON and only require an `id` with optional `data` There are some features still missing that will be added later. -1. Running `makeshift` locally with profiles and plugins -2. Plugin to add user data for one-time use without creating a profile -3. Optionally build plugins directly into the main driver -4. Protected routes that require authentication -5. Configuration file for persistent runs -6. `Dockerfile` and `docker-compose.yml` files to build containers \ No newline at end of file +1. Optionally build plugins directly into the main driver +2. Protected routes that require authentication diff --git a/cmd/download.go b/cmd/download.go index 6c50c3f..c19a783 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -1,6 +1,7 @@ package cmd import ( + "encoding/base64" "fmt" "net/http" "net/url" @@ -15,10 +16,7 @@ import ( "github.com/spf13/cobra" ) -var ( - pluginArgs []string - pluginKWArgs kwargs.KWArgs = kwargs.KWArgs{} -) +var pluginKWArgs kwargs.KWArgs = kwargs.KWArgs{} var downloadCmd = cobra.Command{ Use: "download", Example: ` @@ -52,6 +50,7 @@ var downloadCmd = cobra.Command{ configPath, _ = cmd.Flags().GetString("config") cacertPath, _ = cmd.Flags().GetString("cacert") pluginNames, _ = cmd.Flags().GetStringSlice("plugins") + pluginArgs, _ = cmd.Flags().GetStringSlice("plugin-args") profileIDs, _ = cmd.Flags().GetStringSlice("profiles") extract, _ = cmd.Flags().GetBool("extract") removeArchive, _ = cmd.Flags().GetBool("remove-archive") @@ -75,7 +74,7 @@ var downloadCmd = cobra.Command{ query += "&args=" + url.QueryEscape(strings.Join(pluginArgs, ",")) } if len(pluginKWArgs) > 0 { - query += "&kwargs=" + url.QueryEscape(pluginKWArgs.String()) + query += "&kwargs=" + base64.RawURLEncoding.EncodeToString(pluginKWArgs.Bytes()) } log.Debug(). @@ -86,6 +85,8 @@ var downloadCmd = cobra.Command{ Str("output", outputPath). Strs("profiles", profileIDs). Strs("plugins", pluginNames). + Strs("args", pluginArgs). + Any("kwargs", pluginKWArgs). Send() if cacertPath != "" { diff --git a/cmd/run.go b/cmd/run.go index 846e560..e738e26 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -44,6 +44,7 @@ var runCmd = &cobra.Command{ keyfile, _ = cmd.Flags().GetString("keyfile") timeout, _ = cmd.Flags().GetInt("timeout") pluginNames, _ = cmd.Flags().GetStringSlice("plugins") + pluginArgs, _ = cmd.Flags().GetStringSlice("plugin-args") profileIDs, _ = cmd.Flags().GetStringSlice("profiles") extract, _ = cmd.Flags().GetBool("extract") removeArchive, _ = cmd.Flags().GetBool("remove-archive") diff --git a/cmd/serve.go b/cmd/serve.go index b723a02..fba854b 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -94,8 +94,10 @@ func init() { serveCmd.Flags().String("host", "localhost:5050", "Set the configurator server host (can be set with MAKESHIFT_HOST)") serveCmd.Flags().String("root", "./", "Set the root path to serve files (can be set with MAKESHIFT_ROOT)") serveCmd.Flags().IntP("timeout", "t", 60, "Set the timeout in seconds for requests (can be set with MAKESHIFT_TIMEOUT)") - serveCmd.Flags().String("cacert", "", "Set the CA certificate path to load (can be set with MAKESHIFT_CACERT)") - serveCmd.Flags().String("keyfile", "", "Set the CA key file to use (can be set with MAKESHIFT_KEYFILE)") + serveCmd.Flags().String("cacert", "", "Set the CA certificate path to load (can be set with MAKESHIFT_CACERT, only used if set with '--keyfile' flag)") + serveCmd.Flags().String("keyfile", "", "Set the CA key file to use (can be set with MAKESHIFT_KEYFILE, only used if set with '--cacert' flag)") + serveCmd.Flags().String("keyurl", "", "Set the JWKS remote host for JWT verification") + serveCmd.Flags().StringSlice("protect-routes", []string{}, "Set the routes to require authentication (uses default routes if not set with '--keyurl' flag)") serveCmd.MarkFlagsRequiredTogether("cacert", "keyfile") diff --git a/internal/format/format.go b/internal/format/format.go index bdbfea0..11cfa56 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "path/filepath" + "strings" "gopkg.in/yaml.v3" ) @@ -92,11 +93,11 @@ func Unmarshal(data []byte, v any, inFormat DataFormat) error { // both the standard default format (JSON) and any command line // change to that provided by options. func DataFormatFromFileExt(path string, defaultFmt DataFormat) DataFormat { - switch filepath.Ext(path) { - case ".json", ".JSON": + switch strings.TrimLeft(strings.ToLower(filepath.Ext(path)), ".") { + case JSON.String(): // The file is a JSON file return JSON - case ".yaml", ".yml", ".YAML", ".YML": + case YAML.String(), "yml": // The file is a YAML file return YAML } diff --git a/internal/kwargs/kwargs.go b/internal/kwargs/kwargs.go index 1da80bd..0c8911f 100644 --- a/internal/kwargs/kwargs.go +++ b/internal/kwargs/kwargs.go @@ -5,15 +5,28 @@ import ( "fmt" "git.towk2.me/towk/makeshift/internal/format" + "github.com/rs/zerolog/log" ) const RESERVED_KEY = "kwargs" type KWArgs map[string]any +func New() KWArgs { + return KWArgs{} +} + func (kw KWArgs) String() string { - b, _ := json.Marshal(kw) - return string(b) + return string(kw.Bytes()) +} + +func (kw KWArgs) Bytes() []byte { + b, err := json.Marshal(kw) + if err != nil { + log.Error().Err(err).Msg("failed to marshal kwargs") + return []byte("{}") + } + return b } func (kw *KWArgs) Set(v string /* should be JSON object*/) error { diff --git a/pkg/plugins/jinja2/jinja2.go b/pkg/plugins/jinja2/jinja2.go index daf6b20..6282d7c 100644 --- a/pkg/plugins/jinja2/jinja2.go +++ b/pkg/plugins/jinja2/jinja2.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" + "git.towk2.me/towk/makeshift/internal/kwargs" makeshift "git.towk2.me/towk/makeshift/pkg" "git.towk2.me/towk/makeshift/pkg/storage" "github.com/nikolalohinski/gonja/v2" @@ -42,10 +43,11 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { mappings struct { Data map[string]any `json:"data"` } + userdata *kwargs.KWArgs context *exec.Context template *exec.Template profiles any // makeshift.ProfileMap - input any // []byte + contents any // []byte output bytes.Buffer err error ) @@ -56,18 +58,26 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { Int("arg_count", len(args)). Msg("(jinja2) Run()") + // get profile data used as variable `{{ makeshift.profiles }}` profiles, err = store.Get("profiles") if err != nil { return fmt.Errorf("(jinja2) failed to get profiles: %v", err) } - input, err = store.Get("file") + // get userdata used as variable `{{ makeshift.userdata }}` + userdata, err = store.GetKWArgs() + if err != nil { + return fmt.Errorf("(jinja2) failed to get key-word arguments: %v", err) + } + + // get file contents used for templating + contents, err = store.Get("file") if err != nil { return fmt.Errorf("(jinja2) failed to get input data: %v", err) } // get the templates provided as args to the plugin - template, err = gonja.FromBytes(input.([]byte)) + template, err = gonja.FromBytes(contents.([]byte)) if err != nil { return fmt.Errorf("(jinja2) failed to get template from args: %v", err) } @@ -83,6 +93,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { } } + // get mappings from provided profiles `{{ makeshift.plugin.*}}` var ps = make(map[string]any) for profileID, profile := range profiles.(makeshift.ProfileMap) { ps[profileID] = map[string]any{ @@ -96,6 +107,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { mappings.Data = map[string]any{ "makeshift": map[string]any{ "profiles": ps, + "userdata": userdata, "plugin": map[string]any{ "name": p.Name(), "version": p.Version(), diff --git a/pkg/plugins/mapper/mapper.go b/pkg/plugins/mapper/mapper.go deleted file mode 100644 index 66b4a5d..0000000 --- a/pkg/plugins/mapper/mapper.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - makeshift "git.towk2.me/towk/makeshift/pkg" - "git.towk2.me/towk/makeshift/pkg/storage" -) - -type Mapper struct{} - -func (p *Mapper) Name() string { return "mapper" } -func (p *Mapper) Version() string { return "v0.0.1-alpha" } -func (p *Mapper) Description() string { return "Directly maps data to store" } -func (p *Mapper) Metadata() makeshift.Metadata { - return makeshift.Metadata{ - "author": map[string]any{ - "name": "David J. Allen", - "email": "davidallendj@gmail.com", - "links": []string{ - "https://github.com/davidallendj", - "https://git.towk2.me/towk", - }, - }, - } -} - -func (p *Mapper) Init() error { - // nothing to initialize - return nil -} - -func (p *Mapper) Run(data storage.KVStore, args []string) error { - return nil -} - -func (p *Mapper) Clean() error { - return nil -} - -var Makeshift Mapper diff --git a/pkg/plugins/pyjinja2/pyjinja2.go b/pkg/plugins/pyjinja2/pyjinja2.go index 673f1a1..f625c26 100644 --- a/pkg/plugins/pyjinja2/pyjinja2.go +++ b/pkg/plugins/pyjinja2/pyjinja2.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" + "git.towk2.me/towk/makeshift/internal/kwargs" makeshift "git.towk2.me/towk/makeshift/pkg" "git.towk2.me/towk/makeshift/pkg/storage" jinja2 "github.com/kluctl/kluctl/lib/go-jinja2" @@ -42,6 +43,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { mappings struct { Data map[string]any `json:"data"` } + userdata *kwargs.KWArgs profiles any // makeshift.ProfileMap input any // []byte output string @@ -54,11 +56,19 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { Int("arg_count", len(args)). Msg("(pyjinja2) Run()") + // get profile data used as variable `{{ makeshift.profiles }}` profiles, err = store.Get("profiles") if err != nil { return fmt.Errorf("(pyjinja2) failed to get profiles: %v", err) } + // get userdata used as variable `{{ makeshift.userdata }}` + userdata, err = store.GetKWArgs() + if err != nil { + return fmt.Errorf("(pyjinja2) failed to get key-word arguments: %v", err) + } + + // get file contents used for templating input, err = store.Get("file") if err != nil { return fmt.Errorf("(pyjinja2) failed to get input data: %v", err) @@ -75,7 +85,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { } } - // get mappings from provided profiles + // get mappings from provided profiles `{{ makeshift.plugin.*}}` var ps = make(map[string]any) for profileID, profile := range profiles.(makeshift.ProfileMap) { ps[profileID] = map[string]any{ @@ -89,6 +99,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error { mappings.Data = map[string]any{ "makeshift": map[string]any{ "profiles": ps, + "userdata": userdata, "plugin": map[string]any{ "name": p.Name(), "version": p.Version(), diff --git a/pkg/plugins/user/user.go b/pkg/plugins/user/user.go deleted file mode 100644 index ff59640..0000000 --- a/pkg/plugins/user/user.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import ( - makeshift "git.towk2.me/towk/makeshift/pkg" - "git.towk2.me/towk/makeshift/pkg/storage" -) - -type User struct{} - -func (p *User) Name() string { return "user" } -func (p *User) Version() string { return "v0.0.1-alpha" } -func (p *User) Description() string { return "Get user information" } -func (p *User) Metadata() makeshift.Metadata { - return makeshift.Metadata{ - "author": map[string]any{ - "name": "David J. Allen", - "email": "davidallendj@gmail.com", - "links": []string{ - "https://github.com/davidallendj", - "https://git.towk2.me/towk", - }, - }, - } -} - -func (p *User) Init() error { - return nil -} - -func (p *User) Run(store storage.KVStore, args []string) error { - return nil -} - -func (p *User) Cleanup() error { - return nil -} diff --git a/pkg/service/routes.go b/pkg/service/routes.go index e4fdb04..faa0236 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -1,6 +1,7 @@ package service import ( + "encoding/base64" "encoding/json" "fmt" "io" @@ -16,44 +17,63 @@ import ( makeshift "git.towk2.me/towk/makeshift/pkg" "git.towk2.me/towk/makeshift/pkg/storage" "git.towk2.me/towk/makeshift/pkg/util" - "github.com/go-chi/chi/v5" "github.com/rs/zerolog/log" ) func (s *Service) Download() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( - path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download") - pluginKWArgs = chi.URLParam(r, "kwargs") - pluginArgs = strings.Split(r.URL.Query().Get("args"), ",") - pluginNames = strings.Split(r.URL.Query().Get("plugins"), ",") - profileIDs = strings.Split(r.URL.Query().Get("profiles"), ",") + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download") + pluginArgs = strings.Split(r.URL.Query().Get("args"), ",") + pluginNames = strings.Split(r.URL.Query().Get("plugins"), ",") + profileIDs = strings.Split(r.URL.Query().Get("profiles"), ",") - kw *kwargs.KWArgs + kw *kwargs.KWArgs = new(kwargs.KWArgs) fileInfo os.FileInfo out *os.File store *storage.MemoryStorage = new(storage.MemoryStorage) hooks []makeshift.Hook contents []byte + decoded []byte errs []error err error ) // parse the KWArgs from request - kw.Set(pluginKWArgs) - - // initialize storage - store.Init() - store.SetKWArgs(kw) + decoded, err = base64.RawURLEncoding.DecodeString(r.URL.Query().Get("kwargs")) + if err != nil { + s.writeErrorResponse(w, err.Error(), http.StatusBadRequest) + return + } log.Debug(). Str("path", path). Str("client_host", r.Host). Strs("plugins", pluginNames). Strs("profiles", profileIDs). + Strs("args", pluginArgs). + Str("kwargs", string(decoded)). Any("query", r.URL.Query()). Msg("Service.Download()") + err = kw.Set(string(decoded)) + if err != nil { + s.writeErrorResponse(w, err.Error(), http.StatusBadRequest) + return + } + + // initialize storage + err = store.Init() + if err != nil { + s.writeErrorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + err = store.SetKWArgs(kw) + if err != nil { + s.writeErrorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + // prepare profiles errs = s.LoadProfiles(profileIDs, store, errs) if len(errs) > 0 { diff --git a/pkg/storage/memory.go b/pkg/storage/memory.go index a3b1abb..76874c1 100644 --- a/pkg/storage/memory.go +++ b/pkg/storage/memory.go @@ -20,11 +20,15 @@ func (ms *MemoryStorage) Cleanup() error { } func (ms *MemoryStorage) SetKWArgs(kw *kwargs.KWArgs) error { - return ms.Set(kwargs.RESERVED_KEY, kw) + ms.Data[kwargs.RESERVED_KEY] = kw + return nil } func (ms *MemoryStorage) GetKWArgs() (*kwargs.KWArgs, error) { kw, err := ms.Get(kwargs.RESERVED_KEY) + if err != nil { + return nil, err + } return kw.(*kwargs.KWArgs), err } @@ -37,7 +41,7 @@ func (ms *MemoryStorage) Get(k string) (any, error) { } func (ms *MemoryStorage) Set(k string, v any) error { - if k == "kwargs" { + if k == kwargs.RESERVED_KEY { return fmt.Errorf("cannot set reserved key '%s' (use SetKWArgs() instead)", k) } ms.Data[k] = v diff --git a/pkg/storage/memory_test.go b/pkg/storage/memory_test.go new file mode 100644 index 0000000..82be054 --- /dev/null +++ b/pkg/storage/memory_test.go @@ -0,0 +1 @@ +package storage