From 835b678e75f7122302198653e3894a92c20d17f0 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 21 Apr 2025 08:47:52 -0600 Subject: [PATCH] feat: initial implementation of command split --- cmd/collect.go | 22 ++++++---- cmd/root.go | 3 +- cmd/send.go | 98 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- pkg/client/client.go | 52 +++++++++++------------ pkg/client/default.go | 8 ++-- pkg/collect.go | 49 +++++++++++++++------- pkg/crawler/main.go | 10 ++--- 8 files changed, 181 insertions(+), 63 deletions(-) create mode 100644 cmd/send.go diff --git a/cmd/collect.go b/cmd/collect.go index ef3390c..bbd6e50 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -74,6 +74,7 @@ var CollectCmd = &cobra.Command{ SecretsFile: secretsFile, Username: username, Password: password, + Format: format, } // show all of the 'collect' parameters being set from CLI if verbose @@ -122,15 +123,17 @@ var CollectCmd = &cobra.Command{ func init() { currentUser, _ = user.Current() - CollectCmd.PersistentFlags().StringVar(&host, "host", "", "Set the URI to the SMD root endpoint") - CollectCmd.PersistentFlags().StringVarP(&username, "username", "u", "", "Set the master BMC username") - CollectCmd.PersistentFlags().StringVarP(&password, "password", "p", "", "Set the master BMC password") - CollectCmd.PersistentFlags().StringVar(&secretsFile, "secrets-file", "", "Set path to the node secrets file") - CollectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the default scheme used to query when not included in URI") - CollectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "Set the protocol used to query") - CollectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "Set the path to store collection data") - CollectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "Set flag to force update data sent to SMD") - CollectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "Set the path to CA cert file. (defaults to system CAs when blank)") + CollectCmd.Flags().StringVar(&host, "host", "", "Set the URI to the SMD root endpoint") + CollectCmd.Flags().StringVarP(&username, "username", "u", "", "Set the master BMC username") + CollectCmd.Flags().StringVarP(&password, "password", "p", "", "Set the master BMC password") + CollectCmd.Flags().StringVar(&secretsFile, "secrets-file", "", "Set path to the node secrets file") + CollectCmd.Flags().StringVar(&scheme, "scheme", "https", "Set the default scheme used to query when not included in URI") + CollectCmd.Flags().StringVar(&protocol, "protocol", "tcp", "Set the protocol used to query") + CollectCmd.Flags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "Set the path to store collection data") + CollectCmd.Flags().BoolVar(&forceUpdate, "force-update", false, "Set flag to force update data sent to SMD") + CollectCmd.Flags().StringVar(&cacertPath, "cacert", "", "Set the path to CA cert file. (defaults to system CAs when blank)") + CollectCmd.Flags().StringVarP(&format, "format", "F", "hive", "Set the output format (json|yaml)") + CollectCmd.Flags().BoolVar(&useHive, "use-hive", true, "Set the output format") // set flags to only be used together CollectCmd.MarkFlagsRequiredTogether("username", "password") @@ -143,6 +146,7 @@ func init() { checkBindFlagError(viper.BindPFlag("collect.force-update", CollectCmd.Flags().Lookup("force-update"))) checkBindFlagError(viper.BindPFlag("collect.cacert", CollectCmd.Flags().Lookup("cacert"))) checkBindFlagError(viper.BindPFlags(CollectCmd.Flags())) + checkBindFlagError(viper.BindPFlag("collect.use-hive", CollectCmd.Flags().Lookup("use-hive"))) rootCmd.AddCommand(CollectCmd) } diff --git a/cmd/root.go b/cmd/root.go index 2fde1eb..d073c18 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -45,6 +45,7 @@ var ( debug bool forceUpdate bool insecure bool + useHive bool ) // The `root` command doesn't do anything on it's own except display @@ -79,7 +80,7 @@ func init() { rootCmd.PersistentFlags().IntVarP(&timeout, "timeout", "t", 5, "Set the timeout for requests") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "Set the config file path") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Set to enable/disable verbose output") - rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Set to enable/disable debug messages") + rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Set to enable/disable debug messages") rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "Set the access token") rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", currentUser.Username), "Set the scanning result cache path") diff --git a/cmd/send.go b/cmd/send.go new file mode 100644 index 0000000..e7897be --- /dev/null +++ b/cmd/send.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "crypto/x509" + "encoding/json" + "net/http" + "os" + "strings" + + urlx "github.com/OpenCHAMI/magellan/internal/url" + "github.com/OpenCHAMI/magellan/pkg/auth" + "github.com/OpenCHAMI/magellan/pkg/client" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var sendData []string + +var sendCmd = &cobra.Command{ + Use: "send [host]", + Example: ` // send data from collect output + magellan send -d @collected-1.json -d @collected-2.json https://smd.openchami.cluster + magellan send -d '{...}' -d @collected-1.json https://api.exampe.com + `, + Short: "Send collected node information to specified host.", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + // concatenate all of the data from `-d` flag to send + var ( + smdClient = &client.SmdClient{Client: &http.Client{}} + inputData = []byte(strings.Join(sendData, "\n")) + ) + + // try to load access token either from env var, file, or config if var not set + if accessToken == "" { + var err error + accessToken, err = auth.LoadAccessToken(tokenPath) + if err != nil && verbose { + log.Warn().Err(err).Msgf("could not load access token") + } + } + + // try and load cert if argument is passed + if cacertPath != "" { + cacert, err := os.ReadFile(cacertPath) + if err != nil { + log.Warn().Err(err).Msg("failed to read cert path") + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + // smdClient.WithCertPool(certPool) + // client.WithCertPool(smdClient, certPool) + } + + // create and set headers for request + headers := client.HTTPHeader{} + headers.Authorization(accessToken) + headers.ContentType("application/json") + + // unmarshal into map + data := map[string]any{} + err := json.Unmarshal(inputData, &data) + if err != nil { + log.Error().Err(err).Msg("failed to unmarshal data to make request") + } + + for _, host := range args { + host, err := urlx.Sanitize(host) + if err != nil { + log.Error().Err(err).Msg("failed to sanitize host") + } + + smdClient.URI = host + err = smdClient.Add(inputData, headers) + if err != nil { + + // try updating instead + if forceUpdate { + smdClient.Xname = data["ID"].(string) + err = smdClient.Update(inputData, headers) + if err != nil { + log.Error().Err(err).Msgf("failed to forcibly update Redfish endpoint") + } + } else { + log.Error().Err(err).Msgf("failed to add Redfish endpoint") + } + } + } + }, +} + +func init() { + sendCmd.Flags().StringSliceVarP(&sendData, "data", "d", []string{}, "Set the data to send to specified host.") + sendCmd.Flags().BoolVarP(&forceUpdate, "force-update", "f", false, "Set flag to force update data sent to SMD.") + sendCmd.Flags().StringVar(&cacertPath, "cacert", "", "Set the path to CA cert file. (defaults to system CAs when blank)") + + rootCmd.AddCommand(sendCmd) +} diff --git a/go.mod b/go.mod index 01bcf2b..74137dd 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( require ( github.com/rs/zerolog v1.33.0 golang.org/x/crypto v0.32.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -56,5 +57,4 @@ require ( golang.org/x/text v0.21.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/client/client.go b/pkg/client/client.go index eda049b..6b455c6 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -9,8 +9,6 @@ import ( "net/http" "os" "time" - - "github.com/rs/zerolog/log" ) type Option[T Client] func(client *T) @@ -38,47 +36,47 @@ func NewClient[T Client](opts ...func(T)) T { return *client } -func WithCertPool[T Client](certPool *x509.CertPool) func(T) { +func WithCertPool(client Client, certPool *x509.CertPool) error { // make sure we have a valid cert pool if certPool == nil { - return func(client T) {} + return fmt.Errorf("invalid cert pool") } - return func(client T) { - // make sure that we can access the internal client - if client.GetInternalClient() == nil { - log.Warn().Any("client", client.GetInternalClient()).Msg("invalid internal HTTP client ()") - return - } - client.GetInternalClient().Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: certPool, - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - Dial: (&net.Dialer{ - Timeout: 120 * time.Second, - KeepAlive: 120 * time.Second, - }).Dial, - TLSHandshakeTimeout: 120 * time.Second, - ResponseHeaderTimeout: 120 * time.Second, - } + + // make sure that we can access the internal client + internalClient := client.GetInternalClient() + if internalClient == nil { + return fmt.Errorf("invalid HTTP client") } + internalClient.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + return nil } -func WithSecureTLS[T Client](certPath string) func(T) { +func WithSecureTLS(client Client, certPath string) error { cacert, err := os.ReadFile(certPath) if err != nil { - return func(client T) {} + return fmt.Errorf("failed to read certificate from path '%s': %v", certPath, err) } certPool := x509.NewCertPool() certPool.AppendCertsFromPEM(cacert) - return WithCertPool[T](certPool) + return WithCertPool(client, certPool) } // Post() is a simplified wrapper function that packages all of the // that marshals a mapper into a JSON-formatted byte array, and then performs // a request to the specified URL. -func (c *MagellanClient) Post(url string, data map[string]any, header HTTPHeader) (*http.Response, HTTPBody, error) { +func (c *DefaultClient) Post(url string, data map[string]any, header HTTPHeader) (*http.Response, HTTPBody, error) { // serialize data into byte array body, err := json.Marshal(data) if err != nil { diff --git a/pkg/client/default.go b/pkg/client/default.go index 2830921..d1f433e 100644 --- a/pkg/client/default.go +++ b/pkg/client/default.go @@ -5,11 +5,11 @@ import ( "net/http" ) -type MagellanClient struct { +type DefaultClient struct { *http.Client } -func (c *MagellanClient) Name() string { +func (c *DefaultClient) Name() string { return "default" } @@ -18,7 +18,7 @@ func (c *MagellanClient) Name() string { // the first argument with no data processing or manipulation. The function sends // the data to a set callback URL (which may be changed to use a configurable value // instead). -func (c *MagellanClient) Add(data HTTPBody, headers HTTPHeader) error { +func (c *DefaultClient) Add(data HTTPBody, headers HTTPHeader) error { if data == nil { return fmt.Errorf("no data found") } @@ -35,7 +35,7 @@ func (c *MagellanClient) Add(data HTTPBody, headers HTTPHeader) error { return err } -func (c *MagellanClient) Update(data HTTPBody, headers HTTPHeader) error { +func (c *DefaultClient) Update(data HTTPBody, headers HTTPHeader) error { if data == nil { return fmt.Errorf("no data found") } diff --git a/pkg/collect.go b/pkg/collect.go index ec980ba..375aeca 100644 --- a/pkg/collect.go +++ b/pkg/collect.go @@ -18,6 +18,7 @@ import ( "github.com/OpenCHAMI/magellan/pkg/client" "github.com/OpenCHAMI/magellan/pkg/crawler" "github.com/OpenCHAMI/magellan/pkg/secrets" + "gopkg.in/yaml.v3" "github.com/rs/zerolog/log" @@ -39,6 +40,7 @@ type CollectParams struct { CaCertPath string // set the cert path with the 'cacert' flag Verbose bool // set whether to include verbose output with 'verbose' flag OutputPath string // set the path to save output with 'output' flag + Format string // set the output format ForceUpdate bool // set whether to force updating SMD with 'force-update' flag AccessToken string // set the access token to include in request with 'access-token' flag SecretsFile string // set the path to secrets file @@ -191,9 +193,18 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, localStore s headers.Authorization(params.AccessToken) headers.ContentType("application/json") - body, err := json.MarshalIndent(data, "", " ") - if err != nil { - log.Error().Err(err).Msgf("failed to marshal output to JSON") + var body []byte + switch params.Format { + case "json": + body, err = json.MarshalIndent(data, "", " ") + if err != nil { + log.Error().Err(err).Msgf("failed to marshal output to JSON") + } + case "yaml": + body, err = yaml.Marshal(data) + if err != nil { + log.Error().Err(err).Msgf("failed to marshal output to YAML") + } } if params.Verbose { @@ -203,22 +214,28 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, localStore s // add data output to collections collection = append(collection, data) - // write JSON data to file if output path is set using hive partitioning strategy + // write data to file if output path is set using set format if outputPath != "" { - var ( - finalPath = fmt.Sprintf("./%s/%s/%d.json", outputPath, data["ID"], time.Now().Unix()) - finalDir = filepath.Dir(finalPath) - ) - // if it doesn't, make the directory and write file - err = os.MkdirAll(finalDir, 0o777) - if err == nil { // no error - err = os.WriteFile(path.Clean(finalPath), body, os.ModePerm) - if err != nil { - log.Error().Err(err).Msgf("failed to write collect output to file") + switch params.Format { + case "hive": + var ( + finalPath = fmt.Sprintf("./%s/%s/%d.json", outputPath, data["ID"], time.Now().Unix()) + finalDir = filepath.Dir(finalPath) + ) + // if it doesn't, make the directory and write file + err = os.MkdirAll(finalDir, 0o777) + if err == nil { // no error + err = os.WriteFile(path.Clean(finalPath), body, os.ModePerm) + if err != nil { + log.Error().Err(err).Msgf("failed to write collect output to file") + } + } else { // error is set + log.Error().Err(err).Msg("failed to make directory for collect output") } + case "json": + case "yaml": - } else { // error is set - log.Error().Err(err).Msg("failed to make directory for collect output") + default: } } diff --git a/pkg/crawler/main.go b/pkg/crawler/main.go index 2eb9932..11e1eab 100644 --- a/pkg/crawler/main.go +++ b/pkg/crawler/main.go @@ -126,17 +126,17 @@ func CrawlBMCForSystems(config CrawlerConfig) ([]InventoryDetail, error) { // Obtain the ServiceRoot rf_service := client.GetService() - log.Info().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion) + log.Debug().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion) // Nodes are sometimes only found under Chassis, but they should be found under Systems. rf_chassis, err := rf_service.Chassis() if err == nil { - log.Info().Msgf("found %d chassis in ServiceRoot", len(rf_chassis)) + log.Debug().Msgf("found %d chassis in ServiceRoot", len(rf_chassis)) for _, chassis := range rf_chassis { rf_chassis_systems, err := chassis.ComputerSystems() if err == nil { rf_systems = append(rf_systems, rf_chassis_systems...) - log.Info().Msgf("found %d systems in chassis %s", len(rf_chassis_systems), chassis.ID) + log.Debug().Msgf("found %d systems in chassis %s", len(rf_chassis_systems), chassis.ID) } } } @@ -144,7 +144,7 @@ func CrawlBMCForSystems(config CrawlerConfig) ([]InventoryDetail, error) { if err != nil { log.Error().Err(err).Msg("failed to get systems from ServiceRoot") } - log.Info().Msgf("found %d systems in ServiceRoot", len(rf_root_systems)) + log.Debug().Msgf("found %d systems in ServiceRoot", len(rf_root_systems)) rf_systems = append(rf_systems, rf_root_systems...) return walkSystems(rf_systems, nil, config.URI) } @@ -202,7 +202,7 @@ func CrawlBMCForManagers(config CrawlerConfig) ([]Manager, error) { // Obtain the ServiceRoot rf_service := client.GetService() - log.Info().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion) + log.Debug().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion) rf_managers, err := rf_service.Managers() if err != nil {