refactor: minor changes and fixes

README.md

@@ -239,9 +239,5 @@ Profiles can be created using JSON and only require an `id` with optional `data`
 
 There are some features still missing that will be added later.
 
-1. Running `makeshift` locally with profiles and plugins
+1. Optionally build plugins directly into the main driver
-2. Plugin to add user data for one-time use without creating a profile
+2. Protected routes that require authentication
-3. Optionally build plugins directly into the main driver
-4. Protected routes that require authentication
-5. Configuration file for persistent runs
-6. `Dockerfile` and `docker-compose.yml` files to build containers

cmd/download.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -15,10 +16,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var (
+var pluginKWArgs kwargs.KWArgs = kwargs.KWArgs{}
-	pluginArgs   []string
-	pluginKWArgs kwargs.KWArgs = kwargs.KWArgs{}
-)
 var downloadCmd = cobra.Command{
 	Use: "download",
 	Example: `
@@ -52,6 +50,7 @@ var downloadCmd = cobra.Command{
 			configPath, _    = cmd.Flags().GetString("config")
 			cacertPath, _    = cmd.Flags().GetString("cacert")
 			pluginNames, _   = cmd.Flags().GetStringSlice("plugins")
+			pluginArgs, _    = cmd.Flags().GetStringSlice("plugin-args")
 			profileIDs, _    = cmd.Flags().GetStringSlice("profiles")
 			extract, _       = cmd.Flags().GetBool("extract")
 			removeArchive, _ = cmd.Flags().GetBool("remove-archive")
@@ -75,7 +74,7 @@ var downloadCmd = cobra.Command{
 			query += "&args=" + url.QueryEscape(strings.Join(pluginArgs, ","))
 		}
 		if len(pluginKWArgs) > 0 {
-			query += "&kwargs=" + url.QueryEscape(pluginKWArgs.String())
+			query += "&kwargs=" + base64.RawURLEncoding.EncodeToString(pluginKWArgs.Bytes())
 		}
 
 		log.Debug().
@@ -86,6 +85,8 @@ var downloadCmd = cobra.Command{
 			Str("output", outputPath).
 			Strs("profiles", profileIDs).
 			Strs("plugins", pluginNames).
+			Strs("args", pluginArgs).
+			Any("kwargs", pluginKWArgs).
 			Send()
 
 		if cacertPath != "" {

cmd/run.go

@@ -44,6 +44,7 @@ var runCmd = &cobra.Command{
 			keyfile, _       = cmd.Flags().GetString("keyfile")
 			timeout, _       = cmd.Flags().GetInt("timeout")
 			pluginNames, _   = cmd.Flags().GetStringSlice("plugins")
+			pluginArgs, _    = cmd.Flags().GetStringSlice("plugin-args")
 			profileIDs, _    = cmd.Flags().GetStringSlice("profiles")
 			extract, _       = cmd.Flags().GetBool("extract")
 			removeArchive, _ = cmd.Flags().GetBool("remove-archive")

cmd/serve.go

@@ -94,8 +94,10 @@ func init() {
 	serveCmd.Flags().String("host", "localhost:5050", "Set the configurator server host (can be set with MAKESHIFT_HOST)")
 	serveCmd.Flags().String("root", "./", "Set the root path to serve files (can be set with MAKESHIFT_ROOT)")
 	serveCmd.Flags().IntP("timeout", "t", 60, "Set the timeout in seconds for requests (can be set with MAKESHIFT_TIMEOUT)")
-	serveCmd.Flags().String("cacert", "", "Set the CA certificate path to load (can be set with MAKESHIFT_CACERT)")
+	serveCmd.Flags().String("cacert", "", "Set the CA certificate path to load (can be set with MAKESHIFT_CACERT, only used if set with '--keyfile' flag)")
-	serveCmd.Flags().String("keyfile", "", "Set the CA key file to use (can be set with MAKESHIFT_KEYFILE)")
+	serveCmd.Flags().String("keyfile", "", "Set the CA key file to use (can be set with MAKESHIFT_KEYFILE, only used if set with '--cacert' flag)")
+	serveCmd.Flags().String("keyurl", "", "Set the JWKS remote host for JWT verification")
+	serveCmd.Flags().StringSlice("protect-routes", []string{}, "Set the routes to require authentication (uses default routes if not set with '--keyurl' flag)")
 
 	serveCmd.MarkFlagsRequiredTogether("cacert", "keyfile")
 

internal/format/format.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 )
@@ -92,11 +93,11 @@ func Unmarshal(data []byte, v any, inFormat DataFormat) error {
 // both the standard default format (JSON) and any command line
 // change to that provided by options.
 func DataFormatFromFileExt(path string, defaultFmt DataFormat) DataFormat {
-	switch filepath.Ext(path) {
+	switch strings.TrimLeft(strings.ToLower(filepath.Ext(path)), ".") {
-	case ".json", ".JSON":
+	case JSON.String():
 		// The file is a JSON file
 		return JSON
-	case ".yaml", ".yml", ".YAML", ".YML":
+	case YAML.String(), "yml":
 		// The file is a YAML file
 		return YAML
 	}

internal/kwargs/kwargs.go

@@ -5,15 +5,28 @@ import (
 	"fmt"
 
 	"git.towk2.me/towk/makeshift/internal/format"
+	"github.com/rs/zerolog/log"
 )
 
 const RESERVED_KEY = "kwargs"
 
 type KWArgs map[string]any
 
+func New() KWArgs {
+	return KWArgs{}
+}
+
 func (kw KWArgs) String() string {
-	b, _ := json.Marshal(kw)
+	return string(kw.Bytes())
-	return string(b)
+}
+
+func (kw KWArgs) Bytes() []byte {
+	b, err := json.Marshal(kw)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to marshal kwargs")
+		return []byte("{}")
+	}
+	return b
 }
 
 func (kw *KWArgs) Set(v string /* should be JSON object*/) error {

pkg/plugins/jinja2/jinja2.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"git.towk2.me/towk/makeshift/internal/kwargs"
 	makeshift "git.towk2.me/towk/makeshift/pkg"
 	"git.towk2.me/towk/makeshift/pkg/storage"
 	"github.com/nikolalohinski/gonja/v2"
@@ -42,10 +43,11 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error {
 		mappings struct {
 			Data map[string]any `json:"data"`
 		}
+		userdata *kwargs.KWArgs
 		context  *exec.Context
 		template *exec.Template
 		profiles any // makeshift.ProfileMap
-		input    any // []byte
+		contents any // []byte
 		output   bytes.Buffer
 		err      error
 	)
@@ -56,18 +58,26 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error {
 		Int("arg_count", len(args)).
 		Msg("(jinja2) Run()")
 
+	// get profile data used as variable `{{ makeshift.profiles }}`
 	profiles, err = store.Get("profiles")
 	if err != nil {
 		return fmt.Errorf("(jinja2) failed to get profiles: %v", err)
 	}
 
-	input, err = store.Get("file")
+	// get userdata used as variable `{{ makeshift.userdata }}`
+	userdata, err = store.GetKWArgs()
+	if err != nil {
+		return fmt.Errorf("(jinja2) failed to get key-word arguments: %v", err)
+	}
+
+	// get file contents used for templating
+	contents, err = store.Get("file")
 	if err != nil {
 		return fmt.Errorf("(jinja2) failed to get input data: %v", err)
 	}
 
 	// get the templates provided as args to the plugin
-	template, err = gonja.FromBytes(input.([]byte))
+	template, err = gonja.FromBytes(contents.([]byte))
 	if err != nil {
 		return fmt.Errorf("(jinja2) failed to get template from args: %v", err)
 	}
@@ -83,6 +93,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error {
 		}
 	}
 
+	// get mappings from provided profiles `{{ makeshift.plugin.*}}`
 	var ps = make(map[string]any)
 	for profileID, profile := range profiles.(makeshift.ProfileMap) {
 		ps[profileID] = map[string]any{
@@ -96,6 +107,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error {
 	mappings.Data = map[string]any{
 		"makeshift": map[string]any{
 			"profiles": ps,
+			"userdata": userdata,
 			"plugin": map[string]any{
 				"name":        p.Name(),
 				"version":     p.Version(),

pkg/plugins/mapper/mapper.go

@@ -1,39 +0,0 @@
-package main
-
-import (
-	makeshift "git.towk2.me/towk/makeshift/pkg"
-	"git.towk2.me/towk/makeshift/pkg/storage"
-)
-
-type Mapper struct{}
-
-func (p *Mapper) Name() string        { return "mapper" }
-func (p *Mapper) Version() string     { return "v0.0.1-alpha" }
-func (p *Mapper) Description() string { return "Directly maps data to store" }
-func (p *Mapper) Metadata() makeshift.Metadata {
-	return makeshift.Metadata{
-		"author": map[string]any{
-			"name":  "David J. Allen",
-			"email": "davidallendj@gmail.com",
-			"links": []string{
-				"https://github.com/davidallendj",
-				"https://git.towk2.me/towk",
-			},
-		},
-	}
-}
-
-func (p *Mapper) Init() error {
-	// nothing to initialize
-	return nil
-}
-
-func (p *Mapper) Run(data storage.KVStore, args []string) error {
-	return nil
-}
-
-func (p *Mapper) Clean() error {
-	return nil
-}
-
-var Makeshift Mapper

pkg/plugins/pyjinja2/pyjinja2.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"git.towk2.me/towk/makeshift/internal/kwargs"
 	makeshift "git.towk2.me/towk/makeshift/pkg"
 	"git.towk2.me/towk/makeshift/pkg/storage"
 	jinja2 "github.com/kluctl/kluctl/lib/go-jinja2"
@@ -42,6 +43,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error {
 		mappings struct {
 			Data map[string]any `json:"data"`
 		}
+		userdata *kwargs.KWArgs
 		profiles any // makeshift.ProfileMap
 		input    any // []byte
 		output   string
@@ -54,11 +56,19 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error {
 		Int("arg_count", len(args)).
 		Msg("(pyjinja2) Run()")
 
+	// get profile data used as variable `{{ makeshift.profiles }}`
 	profiles, err = store.Get("profiles")
 	if err != nil {
 		return fmt.Errorf("(pyjinja2) failed to get profiles: %v", err)
 	}
 
+	// get userdata used as variable `{{ makeshift.userdata }}`
+	userdata, err = store.GetKWArgs()
+	if err != nil {
+		return fmt.Errorf("(pyjinja2) failed to get key-word arguments: %v", err)
+	}
+
+	// get file contents used for templating
 	input, err = store.Get("file")
 	if err != nil {
 		return fmt.Errorf("(pyjinja2) failed to get input data: %v", err)
@@ -75,7 +85,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error {
 		}
 	}
 
-	// get mappings from provided profiles
+	// get mappings from provided profiles `{{ makeshift.plugin.*}}`
 	var ps = make(map[string]any)
 	for profileID, profile := range profiles.(makeshift.ProfileMap) {
 		ps[profileID] = map[string]any{
@@ -89,6 +99,7 @@ func (p *Jinja2) Run(store storage.KVStore, args []string) error {
 	mappings.Data = map[string]any{
 		"makeshift": map[string]any{
 			"profiles": ps,
+			"userdata": userdata,
 			"plugin": map[string]any{
 				"name":        p.Name(),
 				"version":     p.Version(),

pkg/plugins/user/user.go

@@ -1,36 +0,0 @@
-package main
-
-import (
-	makeshift "git.towk2.me/towk/makeshift/pkg"
-	"git.towk2.me/towk/makeshift/pkg/storage"
-)
-
-type User struct{}
-
-func (p *User) Name() string        { return "user" }
-func (p *User) Version() string     { return "v0.0.1-alpha" }
-func (p *User) Description() string { return "Get user information" }
-func (p *User) Metadata() makeshift.Metadata {
-	return makeshift.Metadata{
-		"author": map[string]any{
-			"name":  "David J. Allen",
-			"email": "davidallendj@gmail.com",
-			"links": []string{
-				"https://github.com/davidallendj",
-				"https://git.towk2.me/towk",
-			},
-		},
-	}
-}
-
-func (p *User) Init() error {
-	return nil
-}
-
-func (p *User) Run(store storage.KVStore, args []string) error {
-	return nil
-}
-
-func (p *User) Cleanup() error {
-	return nil
-}

pkg/service/routes.go

@@ -1,6 +1,7 @@
 package service
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -16,7 +17,6 @@ import (
 	makeshift "git.towk2.me/towk/makeshift/pkg"
 	"git.towk2.me/towk/makeshift/pkg/storage"
 	"git.towk2.me/towk/makeshift/pkg/util"
-	"github.com/go-chi/chi/v5"
 	"github.com/rs/zerolog/log"
 )
 
@@ -24,36 +24,56 @@ func (s *Service) Download() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var (
 			path        = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download")
-			pluginKWArgs = chi.URLParam(r, "kwargs")
 			pluginArgs  = strings.Split(r.URL.Query().Get("args"), ",")
 			pluginNames = strings.Split(r.URL.Query().Get("plugins"), ",")
 			profileIDs  = strings.Split(r.URL.Query().Get("profiles"), ",")
 
-			kw       *kwargs.KWArgs
+			kw       *kwargs.KWArgs = new(kwargs.KWArgs)
 			fileInfo os.FileInfo
 			out      *os.File
 			store    *storage.MemoryStorage = new(storage.MemoryStorage)
 			hooks    []makeshift.Hook
 			contents []byte
+			decoded  []byte
 			errs     []error
 			err      error
 		)
 
 		// parse the KWArgs from request
-		kw.Set(pluginKWArgs)
+		decoded, err = base64.RawURLEncoding.DecodeString(r.URL.Query().Get("kwargs"))
-
+		if err != nil {
-		// initialize storage
+			s.writeErrorResponse(w, err.Error(), http.StatusBadRequest)
-		store.Init()
+			return
-		store.SetKWArgs(kw)
+		}
 
 		log.Debug().
 			Str("path", path).
 			Str("client_host", r.Host).
 			Strs("plugins", pluginNames).
 			Strs("profiles", profileIDs).
+			Strs("args", pluginArgs).
+			Str("kwargs", string(decoded)).
 			Any("query", r.URL.Query()).
 			Msg("Service.Download()")
 
+		err = kw.Set(string(decoded))
+		if err != nil {
+			s.writeErrorResponse(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		// initialize storage
+		err = store.Init()
+		if err != nil {
+			s.writeErrorResponse(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		err = store.SetKWArgs(kw)
+		if err != nil {
+			s.writeErrorResponse(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
 		// prepare profiles
 		errs = s.LoadProfiles(profileIDs, store, errs)
 		if len(errs) > 0 {

pkg/storage/memory.go

@@ -20,11 +20,15 @@ func (ms *MemoryStorage) Cleanup() error {
 }
 
 func (ms *MemoryStorage) SetKWArgs(kw *kwargs.KWArgs) error {
-	return ms.Set(kwargs.RESERVED_KEY, kw)
+	ms.Data[kwargs.RESERVED_KEY] = kw
+	return nil
 }
 
 func (ms *MemoryStorage) GetKWArgs() (*kwargs.KWArgs, error) {
 	kw, err := ms.Get(kwargs.RESERVED_KEY)
+	if err != nil {
+		return nil, err
+	}
 	return kw.(*kwargs.KWArgs), err
 }
 
@@ -37,7 +41,7 @@ func (ms *MemoryStorage) Get(k string) (any, error) {
 }
 
 func (ms *MemoryStorage) Set(k string, v any) error {
-	if k == "kwargs" {
+	if k == kwargs.RESERVED_KEY {
 		return fmt.Errorf("cannot set reserved key '%s' (use SetKWArgs() instead)", k)
 	}
 	ms.Data[k] = v

pkg/storage/memory_test.go

@@ -0,0 +1 @@
+package storage