diff --git a/cmd/download.go b/cmd/download.go index e36ee86..10c860f 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -30,6 +30,7 @@ var downloadCmd = cobra.Command{ curl $MAKESHIFT_HOST/download/test?plugins=smd,jinja2&profile=test # download directory and extract it's contents automatically + # then, remove the downloaded archive makeshift download -xr `, Short: "Download and modify files with plugins", diff --git a/cmd/upload.go b/cmd/upload.go index 39aa49f..12b58b1 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -1,24 +1,234 @@ package cmd -import "github.com/spf13/cobra" +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strings" + + "git.towk2.me/towk/makeshift/internal/format" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var ( + inputFormat format.DataFormat = format.JSON + dataArgs []string +) var uploadCmd = &cobra.Command{ Use: "upload", - Run: func(cmd *cobra.Command, args []string) { + Example: ` + # upload a single file + makeshift upload -d @compute-base.yaml -t file + # upload a single file with contents without specify type + makeshift upload -d '{"name": "John Smith", "email": "john.smith@example.com"}' + + # upload a directory + makeshift upload -d @setup/ -t directory + + # upload an archive (extracted and saved on server) + makeshift upload -d @setup.tar.gz -t archive +`, + Run: func(cmd *cobra.Command, args []string) { + // make one request be host positional argument (restricted to 1 for now) + var inputData []map[string]any + temp := append(handleArgs(args), processDataArgs(dataArgs)...) + for _, data := range temp { + if data != nil { + inputData = append(inputData, data) + } + } }, } var uploadProfileCmd = &cobra.Command{ Use: "profile", + Example: ` + # upload a new profile + makeshift upload profile -d @compute.json + + # upload a new profile with a specific name (used for lookups) + makeshift upload profile -d @kubernetes.json -n k8s +`, + Args: cobra.ExactArgs(1), + Short: "Upload a new profile", + Run: func(cmd *cobra.Command, args []string) { + + // make one request be host positional argument (restricted to 1 for now) + var inputData []map[string]any + temp := append(handleArgs(args), processDataArgs(dataArgs)...) + for _, data := range temp { + if data != nil { + inputData = append(inputData, data) + } + } + + }, } var uploadPluginCmd = &cobra.Command{ Use: "plugin", + Example: ` + # upload a new plugin + makeshift upload plugin -d @slurm.so + + # upload a new plugin with a specific name (used for lookups) + makeshift upload plugin -d @cobbler.so -n merge +`, + Args: cobra.ExactArgs(1), + Short: "Upload a new plugin", + Run: func(cmd *cobra.Command, args []string) { + // make one request be host positional argument (restricted to 1 for now) + var inputData []map[string]any + temp := append(handleArgs(args), processDataArgs(dataArgs)...) + for _, data := range temp { + if data != nil { + inputData = append(inputData, data) + } + } + }, } func init() { + uploadProfileCmd.Flags().VarP(&inputFormat, "format", "F", "Set the input format for profile") uploadCmd.AddCommand(uploadProfileCmd, uploadPluginCmd) rootCmd.AddCommand(uploadCmd) } + +// processDataArgs takes a slice of strings that check for the @ symbol and loads +// the contents from the file specified in place (which replaces the path). +// +// NOTE: The purpose is to make the input arguments uniform for our request. This +// function is meant to handle data passed with the `-d/--data` flag and positional +// args from the CLI. +func processDataArgs(args []string) []map[string]any { + // JSON representation + type ( + JSONObject = map[string]any + JSONArray = []JSONObject + ) + + // load data either from file or directly from args + var collection = make(JSONArray, len(args)) + for i, arg := range args { + // if arg is empty string, then skip and continue + if len(arg) > 0 { + // determine if we're reading from file to load contents + if strings.HasPrefix(arg, "@") { + var ( + path string = strings.TrimLeft(arg, "@") + contents []byte + data JSONArray + err error + ) + contents, err = os.ReadFile(path) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("failed to read file") + continue + } + + // skip empty files + if len(contents) == 0 { + log.Warn().Str("path", path).Msg("file is empty") + continue + } + + // convert/validate input data + data, err = parseInput(contents, format.DataFormatFromFileExt(path, inputFormat)) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("failed to validate input from file") + } + + // add loaded data to collection of all data + collection = append(collection, data...) + } else { + // input should be a valid JSON + var ( + data JSONArray + input = []byte(arg) + err error + ) + if !json.Valid(input) { + log.Error().Msgf("argument %d not a valid JSON", i) + continue + } + err = json.Unmarshal(input, &data) + if err != nil { + log.Error().Err(err).Msgf("failed to unmarshal input for argument %d", i) + } + return data + } + } + } + return collection +} + +func handleArgs(args []string) []map[string]any { + // JSON representation + type ( + JSONObject = map[string]any + JSONArray = []JSONObject + ) + // no file to load, so we just use the joined args (since each one is a new line) + // and then stop + var ( + collection JSONArray + data []byte + err error + ) + + if len(dataArgs) > 0 { + return nil + } + data, err = ReadStdin() + if err != nil { + log.Error().Err(err).Msg("failed to read from standard input") + return nil + } + if len(data) == 0 { + log.Warn().Msg("no data found from standard input") + return nil + } + fmt.Println(string(data)) + collection, err = parseInput([]byte(data), inputFormat) + if err != nil { + log.Error().Err(err).Msg("failed to validate input from arg") + } + return collection +} + +func parseInput(contents []byte, dataFormat format.DataFormat) ([]map[string]any, error) { + var ( + data []map[string]any + err error + ) + + // convert/validate JSON input format + err = format.Unmarshal(contents, &data, dataFormat) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + return data, nil +} + +// ReadStdin reads all of standard input and returns the bytes. If an error +// occurs during scanning, it is returned. +func ReadStdin() ([]byte, error) { + var b []byte + input := bufio.NewScanner(os.Stdin) + for input.Scan() { + b = append(b, input.Bytes()...) + b = append(b, byte('\n')) + if len(b) == 0 { + break + } + } + if err := input.Err(); err != nil { + return b, fmt.Errorf("failed to read stdin: %w", err) + } + return b, nil +} diff --git a/go.mod b/go.mod index 4090b3f..4b47295 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,9 @@ require ( github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.8.0 github.com/tidwall/sjson v1.2.5 + go.yaml.in/yaml/v3 v3.0.4 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -22,6 +24,7 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect diff --git a/go.sum b/go.sum index 3c1e2f4..91a82d0 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIH github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -31,6 +32,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= @@ -59,10 +64,13 @@ github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -92,6 +100,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= @@ -109,6 +119,8 @@ golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/format/format.go b/internal/format/format.go new file mode 100644 index 0000000..d38aa3e --- /dev/null +++ b/internal/format/format.go @@ -0,0 +1,104 @@ +package format + +import ( + "encoding/json" + "fmt" + "path/filepath" + + "gopkg.in/yaml.v3" +) + +type DataFormat string + +const ( + List DataFormat = "list" + JSON DataFormat = "json" + FORMAT_YAML DataFormat = "yaml" +) + +func (df DataFormat) String() string { + return string(df) +} + +func (df *DataFormat) Set(v string) error { + switch DataFormat(v) { + case List, JSON, FORMAT_YAML: + *df = DataFormat(v) + return nil + default: + return fmt.Errorf("must be one of %v", []DataFormat{ + List, JSON, FORMAT_YAML, + }) + } +} + +func (df DataFormat) Type() string { + return "DataFormat" +} + +// MarshalData marshals arbitrary data into a byte slice formatted as outFormat. +// If a marshalling error occurs or outFormat is unknown, an error is returned. +// +// Supported values are: json, list, yaml +func Marshal(data interface{}, outFormat DataFormat) ([]byte, error) { + switch outFormat { + case JSON: + if bytes, err := json.MarshalIndent(data, "", " "); err != nil { + return nil, fmt.Errorf("failed to marshal data into JSON: %w", err) + } else { + return bytes, nil + } + case FORMAT_YAML: + if bytes, err := yaml.Marshal(data); err != nil { + return nil, fmt.Errorf("failed to marshal data into YAML: %w", err) + } else { + return bytes, nil + } + case List: + return nil, fmt.Errorf("this data format cannot be marshaled") + default: + return nil, fmt.Errorf("unknown data format: %s", outFormat) + } +} + +// UnmarshalData unmarshals a byte slice formatted as inFormat into an interface +// v. If an unmarshalling error occurs or inFormat is unknown, an error is +// returned. +// +// Supported values are: json, list, yaml +func Unmarshal(data []byte, v interface{}, inFormat DataFormat) error { + switch inFormat { + case JSON: + if err := json.Unmarshal(data, v); err != nil { + return fmt.Errorf("failed to unmarshal data into JSON: %w", err) + } + case FORMAT_YAML: + if err := yaml.Unmarshal(data, v); err != nil { + return fmt.Errorf("failed to unmarshal data into YAML: %w", err) + } + case List: + return fmt.Errorf("this data format cannot be unmarshaled") + default: + return fmt.Errorf("unknown data format: %s", inFormat) + } + + return nil +} + +// DataFormatFromFileExt determines the type of the contents +// (JSON or YAML) based on the filname extension. The default +// format is passed in, so if it doesn't match one of the cases, +// that's what we will use. The defaultFmt value takes into account +// both the standard default format (JSON) and any command line +// change to that provided by options. +func DataFormatFromFileExt(path string, defaultFmt DataFormat) DataFormat { + switch filepath.Ext(path) { + case ".json", ".JSON": + // The file is a JSON file + return JSON + case ".yaml", ".yml", ".YAML", ".YML": + // The file is a YAML file + return FORMAT_YAML + } + return defaultFmt +} diff --git a/pkg/service/routes.go b/pkg/service/routes.go index 3d2db0b..42e7338 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -185,6 +185,18 @@ func (s *Service) Upload() http.HandlerFunc { } } +func (s *Service) UploadPlugin() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + } +} + +func (s *Service) UploadProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + } +} + func (s *Service) List() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( diff --git a/pkg/service/service.go b/pkg/service/service.go index d6c642f..c69d68f 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -100,8 +100,8 @@ func (s *Service) Serve() error { // router.Handle("/download/*", http.StripPrefix("/download/", http.FileServer(http.Dir(s.PathForData())))) router.Get("/download/*", s.Download()) router.Post("/upload/", s.Upload()) - router.Post("/upload/plugin", s.Upload()) - router.Post("/upload/profile", s.Upload()) + router.Post("/upload/plugin", s.UploadPlugin()) + router.Post("/upload/profile", s.UploadProfile()) router.Get("/list/*", s.List()) // profiles