feat: added format package for files

cmd/download.go

@@ -30,6 +30,7 @@ var downloadCmd = cobra.Command{
   curl $MAKESHIFT_HOST/download/test?plugins=smd,jinja2&profile=test
 
   # download directory and extract it's contents automatically
+  # then, remove the downloaded archive
   makeshift download -xr
 `,
 	Short: "Download and modify files with plugins",

cmd/upload.go

@@ -1,24 +1,234 @@
 package cmd
 
-import "github.com/spf13/cobra"
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"git.towk2.me/towk/makeshift/internal/format"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var (
+	inputFormat format.DataFormat = format.JSON
+	dataArgs    []string
+)
 
 var uploadCmd = &cobra.Command{
 	Use: "upload",
-	Run: func(cmd *cobra.Command, args []string) {
+	Example: `
+  # upload a single file
+  makeshift upload -d @compute-base.yaml -t file
 
+  # upload a single file with contents without specify type
+  makeshift upload -d '{"name": "John Smith", "email": "john.smith@example.com"}'
+  
+  # upload a directory
+  makeshift upload -d @setup/  -t directory
+
+  # upload an archive (extracted and saved on server)
+  makeshift upload -d @setup.tar.gz -t archive
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		// make one request be host positional argument (restricted to 1 for now)
+		var inputData []map[string]any
+		temp := append(handleArgs(args), processDataArgs(dataArgs)...)
+		for _, data := range temp {
+			if data != nil {
+				inputData = append(inputData, data)
+			}
+		}
 	},
 }
 
 var uploadProfileCmd = &cobra.Command{
 	Use: "profile",
+	Example: `
+  # upload a new profile
+  makeshift upload profile -d @compute.json 
+
+  # upload a new profile with a specific name (used for lookups)
+  makeshift upload profile -d @kubernetes.json -n k8s
+`,
+	Args:  cobra.ExactArgs(1),
+	Short: "Upload a new profile",
+	Run: func(cmd *cobra.Command, args []string) {
+
+		// make one request be host positional argument (restricted to 1 for now)
+		var inputData []map[string]any
+		temp := append(handleArgs(args), processDataArgs(dataArgs)...)
+		for _, data := range temp {
+			if data != nil {
+				inputData = append(inputData, data)
+			}
+		}
+
+	},
 }
 
 var uploadPluginCmd = &cobra.Command{
 	Use: "plugin",
+	Example: `
+  # upload a new plugin
+  makeshift upload plugin -d @slurm.so
+
+  # upload a new plugin with a specific name (used for lookups)
+  makeshift upload plugin -d @cobbler.so -n merge
+`,
+	Args:  cobra.ExactArgs(1),
+	Short: "Upload a new plugin",
+	Run: func(cmd *cobra.Command, args []string) {
+		// make one request be host positional argument (restricted to 1 for now)
+		var inputData []map[string]any
+		temp := append(handleArgs(args), processDataArgs(dataArgs)...)
+		for _, data := range temp {
+			if data != nil {
+				inputData = append(inputData, data)
+			}
+		}
+	},
 }
 
 func init() {
+	uploadProfileCmd.Flags().VarP(&inputFormat, "format", "F", "Set the input format for profile")
 
 	uploadCmd.AddCommand(uploadProfileCmd, uploadPluginCmd)
 	rootCmd.AddCommand(uploadCmd)
 }
+
+// processDataArgs takes a slice of strings that check for the @ symbol and loads
+// the contents from the file specified in place (which replaces the path).
+//
+// NOTE: The purpose is to make the input arguments uniform for our request. This
+// function is meant to handle data passed with the `-d/--data` flag and positional
+// args from the CLI.
+func processDataArgs(args []string) []map[string]any {
+	// JSON representation
+	type (
+		JSONObject = map[string]any
+		JSONArray  = []JSONObject
+	)
+
+	// load data either from file or directly from args
+	var collection = make(JSONArray, len(args))
+	for i, arg := range args {
+		// if arg is empty string, then skip and continue
+		if len(arg) > 0 {
+			// determine if we're reading from file to load contents
+			if strings.HasPrefix(arg, "@") {
+				var (
+					path     string = strings.TrimLeft(arg, "@")
+					contents []byte
+					data     JSONArray
+					err      error
+				)
+				contents, err = os.ReadFile(path)
+				if err != nil {
+					log.Error().Err(err).Str("path", path).Msg("failed to read file")
+					continue
+				}
+
+				// skip empty files
+				if len(contents) == 0 {
+					log.Warn().Str("path", path).Msg("file is empty")
+					continue
+				}
+
+				// convert/validate input data
+				data, err = parseInput(contents, format.DataFormatFromFileExt(path, inputFormat))
+				if err != nil {
+					log.Error().Err(err).Str("path", path).Msg("failed to validate input from file")
+				}
+
+				// add loaded data to collection of all data
+				collection = append(collection, data...)
+			} else {
+				// input should be a valid JSON
+				var (
+					data  JSONArray
+					input = []byte(arg)
+					err   error
+				)
+				if !json.Valid(input) {
+					log.Error().Msgf("argument %d not a valid JSON", i)
+					continue
+				}
+				err = json.Unmarshal(input, &data)
+				if err != nil {
+					log.Error().Err(err).Msgf("failed to unmarshal input for argument %d", i)
+				}
+				return data
+			}
+		}
+	}
+	return collection
+}
+
+func handleArgs(args []string) []map[string]any {
+	// JSON representation
+	type (
+		JSONObject = map[string]any
+		JSONArray  = []JSONObject
+	)
+	// no file to load, so we just use the joined args (since each one is a new line)
+	// and then stop
+	var (
+		collection JSONArray
+		data       []byte
+		err        error
+	)
+
+	if len(dataArgs) > 0 {
+		return nil
+	}
+	data, err = ReadStdin()
+	if err != nil {
+		log.Error().Err(err).Msg("failed to read from standard input")
+		return nil
+	}
+	if len(data) == 0 {
+		log.Warn().Msg("no data found from standard input")
+		return nil
+	}
+	fmt.Println(string(data))
+	collection, err = parseInput([]byte(data), inputFormat)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to validate input from arg")
+	}
+	return collection
+}
+
+func parseInput(contents []byte, dataFormat format.DataFormat) ([]map[string]any, error) {
+	var (
+		data []map[string]any
+		err  error
+	)
+
+	// convert/validate JSON input format
+	err = format.Unmarshal(contents, &data, dataFormat)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal data: %v", err)
+	}
+	return data, nil
+}
+
+// ReadStdin reads all of standard input and returns the bytes. If an error
+// occurs during scanning, it is returned.
+func ReadStdin() ([]byte, error) {
+	var b []byte
+	input := bufio.NewScanner(os.Stdin)
+	for input.Scan() {
+		b = append(b, input.Bytes()...)
+		b = append(b, byte('\n'))
+		if len(b) == 0 {
+			break
+		}
+	}
+	if err := input.Err(); err != nil {
+		return b, fmt.Errorf("failed to read stdin: %w", err)
+	}
+	return b, nil
+}

go.mod

@@ -13,7 +13,9 @@ require (
 	github.com/rs/zerolog v1.34.0
 	github.com/spf13/cobra v1.8.0
 	github.com/tidwall/sjson v1.2.5
+	go.yaml.in/yaml/v3 v3.0.4
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -22,6 +24,7 @@ require (
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/httprc v1.0.6 // indirect

go.sum

@@ -6,6 +6,7 @@ github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIH
 github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -31,6 +32,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
 github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
 github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
@@ -59,10 +64,13 @@ github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus
 github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
 github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
 github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
 github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
@@ -92,6 +100,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
 go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
 golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
@@ -109,6 +119,8 @@ golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
 golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
 golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/format/format.go

@@ -0,0 +1,104 @@
+package format
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+
+	"gopkg.in/yaml.v3"
+)
+
+type DataFormat string
+
+const (
+	List        DataFormat = "list"
+	JSON        DataFormat = "json"
+	FORMAT_YAML DataFormat = "yaml"
+)
+
+func (df DataFormat) String() string {
+	return string(df)
+}
+
+func (df *DataFormat) Set(v string) error {
+	switch DataFormat(v) {
+	case List, JSON, FORMAT_YAML:
+		*df = DataFormat(v)
+		return nil
+	default:
+		return fmt.Errorf("must be one of %v", []DataFormat{
+			List, JSON, FORMAT_YAML,
+		})
+	}
+}
+
+func (df DataFormat) Type() string {
+	return "DataFormat"
+}
+
+// MarshalData marshals arbitrary data into a byte slice formatted as outFormat.
+// If a marshalling error occurs or outFormat is unknown, an error is returned.
+//
+// Supported values are: json, list, yaml
+func Marshal(data interface{}, outFormat DataFormat) ([]byte, error) {
+	switch outFormat {
+	case JSON:
+		if bytes, err := json.MarshalIndent(data, "", "  "); err != nil {
+			return nil, fmt.Errorf("failed to marshal data into JSON: %w", err)
+		} else {
+			return bytes, nil
+		}
+	case FORMAT_YAML:
+		if bytes, err := yaml.Marshal(data); err != nil {
+			return nil, fmt.Errorf("failed to marshal data into YAML: %w", err)
+		} else {
+			return bytes, nil
+		}
+	case List:
+		return nil, fmt.Errorf("this data format cannot be marshaled")
+	default:
+		return nil, fmt.Errorf("unknown data format: %s", outFormat)
+	}
+}
+
+// UnmarshalData unmarshals a byte slice formatted as inFormat into an interface
+// v. If an unmarshalling error occurs or inFormat is unknown, an error is
+// returned.
+//
+// Supported values are: json, list, yaml
+func Unmarshal(data []byte, v interface{}, inFormat DataFormat) error {
+	switch inFormat {
+	case JSON:
+		if err := json.Unmarshal(data, v); err != nil {
+			return fmt.Errorf("failed to unmarshal data into JSON: %w", err)
+		}
+	case FORMAT_YAML:
+		if err := yaml.Unmarshal(data, v); err != nil {
+			return fmt.Errorf("failed to unmarshal data into YAML: %w", err)
+		}
+	case List:
+		return fmt.Errorf("this data format cannot be unmarshaled")
+	default:
+		return fmt.Errorf("unknown data format: %s", inFormat)
+	}
+
+	return nil
+}
+
+// DataFormatFromFileExt determines the type of the contents
+// (JSON or YAML) based on the filname extension. The default
+// format is passed in, so if it doesn't match one of the cases,
+// that's what we will use. The defaultFmt value takes into account
+// both the standard default format (JSON) and any command line
+// change to that provided by options.
+func DataFormatFromFileExt(path string, defaultFmt DataFormat) DataFormat {
+	switch filepath.Ext(path) {
+	case ".json", ".JSON":
+		// The file is a JSON file
+		return JSON
+	case ".yaml", ".yml", ".YAML", ".YML":
+		// The file is a YAML file
+		return FORMAT_YAML
+	}
+	return defaultFmt
+}

pkg/service/routes.go

@@ -185,6 +185,18 @@ func (s *Service) Upload() http.HandlerFunc {
 	}
 }
 
+func (s *Service) UploadPlugin() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+	}
+}
+
+func (s *Service) UploadProfile() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+	}
+}
+
 func (s *Service) List() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var (

pkg/service/service.go

@@ -100,8 +100,8 @@ func (s *Service) Serve() error {
 		// router.Handle("/download/*", http.StripPrefix("/download/", http.FileServer(http.Dir(s.PathForData()))))
 		router.Get("/download/*", s.Download())
 		router.Post("/upload/", s.Upload())
-		router.Post("/upload/plugin", s.Upload())
-		router.Post("/upload/profile", s.Upload())
+		router.Post("/upload/plugin", s.UploadPlugin())
+		router.Post("/upload/profile", s.UploadProfile())
 		router.Get("/list/*", s.List())
 
 		// profiles