Split the collect Command For Customization (#93)

* feat: initial implementation of command split * feat: update collect and new send cmd * chore: cleanup unused code * chore: refactored getting username * chore: more refactoring and cleanup * feat: update send cmd implementation * chore: changed/updated example config * chore: made cmd more consistent and added formatting * refactor: removed --host flag from scan * chore: cleaned up and fixed issue with client * chore: cleaned up CLI flags in collect cmd * feat: updated crawl to include managers and output YAML optionally * refactor: updated and improved send implementation * refactor: minor improvements * refactor: added util func to check for empty slices * fix: issue with reading from stdin * refactor: added scheme trimming function for URIs * refactor: changed host arg back to positional * refactor: removed unused vars and added --output-dir flag * fix: make -f for secrets persistent * refactor: removed --host flag and request in collect * refactor: changed --output flag to --output-file * fix: updated flags for collect * fix: typo in crawler error * fix: dir being created when outputDir not set * fix: reading stdin and data args * fix: made output using -v and -o consistent * readme: added info about command split * updated changelog adding missing version entries * chore: updated example to use host as positional arg * fix: issue with reading --data arg * fix: remove unused import from collect pkg Signed-off-by: Devon Bautista <devonb@lanl.gov> --------- Signed-off-by: David Allen <16520934+davidallendj@users.noreply.github.com> Signed-off-by: Devon Bautista <devonb@lanl.gov> Co-authored-by: Devon Bautista <devonb@lanl.gov>
2025-12-20 03:27:03 -07:00 · 2025-05-29 13:15:46 -06:00 · 2025-05-29 13:15:46 -06:00 · 04e1fb26c9
commit 04e1fb26c9
parent fba4a89a0e
19 changed files with 736 additions and 223 deletions
--- a/pkg/client/client.go
+++ b/pkg/client/client.go
@ -9,8 +9,6 @@ import (
 	"net/http"
 	"os"
 	"time"
-
-	"github.com/rs/zerolog/log"
 )

 type Option[T Client] func(client *T)
@ -29,56 +27,51 @@ type Client interface {
 	Update(data HTTPBody, headers HTTPHeader) error
 }

-// NewClient() creates a new client
-func NewClient[T Client](opts ...func(T)) T {
-	client := new(T)
-	for _, opt := range opts {
-		opt(*client)
-	}
-	return *client
-}
-
-func WithCertPool[T Client](certPool *x509.CertPool) func(T) {
-	// make sure we have a valid cert pool
-	if certPool == nil {
-		return func(client T) {}
-	}
-	return func(client T) {
-		// make sure that we can access the internal client
-		if client.GetInternalClient() == nil {
-			log.Warn().Any("client", client.GetInternalClient()).Msg("invalid internal HTTP client ()")
-			return
-		}
-		client.GetInternalClient().Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{
-				RootCAs:            certPool,
-				InsecureSkipVerify: true,
-			},
-			DisableKeepAlives: true,
-			Dial: (&net.Dialer{
-				Timeout:   120 * time.Second,
-				KeepAlive: 120 * time.Second,
-			}).Dial,
-			TLSHandshakeTimeout:   120 * time.Second,
-			ResponseHeaderTimeout: 120 * time.Second,
-		}
-	}
-}
-
-func WithSecureTLS[T Client](certPath string) func(T) {
-	cacert, err := os.ReadFile(certPath)
+func LoadCertificateFromPath(client Client, path string) error {
+	cacert, err := os.ReadFile(path)
 	if err != nil {
-		return func(client T) {}
+		return fmt.Errorf("failed to read certificate at path: %s", path)
 	}
 	certPool := x509.NewCertPool()
 	certPool.AppendCertsFromPEM(cacert)
-	return WithCertPool[T](certPool)
+	err = LoadCertificateFromPool(client, certPool)
+	if err != nil {
+		return fmt.Errorf("could not initialize certificate from pool: %v", err)
+	}
+	return nil
+}
+
+func LoadCertificateFromPool(client Client, certPool *x509.CertPool) error {
+	// make sure we have a valid cert pool
+	if certPool == nil {
+		return fmt.Errorf("invalid cert pool")
+	}
+
+	// make sure that we can access the internal client
+	internalClient := client.GetInternalClient()
+	if internalClient == nil {
+		return fmt.Errorf("invalid HTTP client")
+	}
+	internalClient.Transport = &http.Transport{
+		TLSClientConfig: &tls.Config{
+			RootCAs:            certPool,
+			InsecureSkipVerify: false,
+		},
+		DisableKeepAlives: true,
+		Dial: (&net.Dialer{
+			Timeout:   120 * time.Second,
+			KeepAlive: 120 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout:   120 * time.Second,
+		ResponseHeaderTimeout: 120 * time.Second,
+	}
+	return nil
 }

 // Post() is a simplified wrapper function that packages all of the
 // that marshals a mapper into a JSON-formatted byte array, and then performs
 // a request to the specified URL.
-func (c *MagellanClient) Post(url string, data map[string]any, header HTTPHeader) (*http.Response, HTTPBody, error) {
+func (c *DefaultClient) Post(url string, data map[string]any, header HTTPHeader) (*http.Response, HTTPBody, error) {
 	// serialize data into byte array
 	body, err := json.Marshal(data)
 	if err != nil {
--- a/pkg/client/default.go
+++ b/pkg/client/default.go
@ -5,11 +5,11 @@ import (
 	"net/http"
 )

-type MagellanClient struct {
+type DefaultClient struct {
 	*http.Client
 }

-func (c *MagellanClient) Name() string {
+func (c *DefaultClient) Name() string {
 	return "default"
 }

@ -18,7 +18,7 @@ func (c *MagellanClient) Name() string {
 // the first argument with no data processing or manipulation. The function sends
 // the data to a set callback URL (which may be changed to use a configurable value
 // instead).
-func (c *MagellanClient) Add(data HTTPBody, headers HTTPHeader) error {
+func (c *DefaultClient) Add(data HTTPBody, headers HTTPHeader) error {
 	if data == nil {
 		return fmt.Errorf("no data found")
 	}
@ -35,7 +35,7 @@ func (c *MagellanClient) Add(data HTTPBody, headers HTTPHeader) error {
 	return err
 }

-func (c *MagellanClient) Update(data HTTPBody, headers HTTPHeader) error {
+func (c *DefaultClient) Update(data HTTPBody, headers HTTPHeader) error {
 	if data == nil {
 		return fmt.Errorf("no data found")
 	}
--- a/pkg/client/smd.go
+++ b/pkg/client/smd.go
@ -4,6 +4,7 @@ package client
 //	https://github.com/OpenCHAMI/hms-smd/blob/master/docs/examples.adoc
 //	https://github.com/OpenCHAMI/hms-smd
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"

@ -16,6 +17,12 @@ type SmdClient struct {
 	Xname string
 }

+func NewSmdClient() *SmdClient {
+	return &SmdClient{
+		Client: &http.Client{},
+	}
+}
+
 func (c *SmdClient) Init() {
 	c.Client = &http.Client{}
 }
@ -44,7 +51,7 @@ func (c *SmdClient) Add(data HTTPBody, headers HTTPHeader) error {
 	url := c.RootEndpoint("/Inventory/RedfishEndpoints")
 	res, body, err := MakeRequest(c.Client, url, http.MethodPost, data, headers)
 	if res != nil {
-		statusOk := res.StatusCode >= 200 && res.StatusCode < 300
+		statusOk := res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusMultipleChoices
 		if !statusOk {
 			if len(body) > 0 {
 				return fmt.Errorf("%d: %s", res.StatusCode, string(body))
@ -77,3 +84,17 @@ func (c *SmdClient) Update(data HTTPBody, headers HTTPHeader) error {
 	}
 	return err
 }
+
+func (c *SmdClient) SetXnameFromJSON(contents []byte, key string) error {
+	var (
+		data map[string]any
+		err  error
+	)
+
+	err = json.Unmarshal(contents, &data)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal xname: %v", err)
+	}
+	c.Xname = data[key].(string)
+	return nil
+}
--- a/pkg/collect.go
+++ b/pkg/collect.go
@ -2,12 +2,9 @@
 package magellan

 import (
-	"crypto/tls"
-	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"net"
-	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@ -15,10 +12,12 @@ import (
 	"sync"
 	"time"

+	"github.com/OpenCHAMI/magellan/internal/util"
 	"github.com/OpenCHAMI/magellan/pkg/bmc"
 	"github.com/OpenCHAMI/magellan/pkg/client"
 	"github.com/OpenCHAMI/magellan/pkg/crawler"
 	"github.com/OpenCHAMI/magellan/pkg/secrets"
+	"gopkg.in/yaml.v3"

 	"github.com/rs/zerolog/log"

@ -38,6 +37,8 @@ type CollectParams struct {
 	CaCertPath  string              // set the cert path with the 'cacert' flag
 	Verbose     bool                // set whether to include verbose output with 'verbose' flag
 	OutputPath  string              // set the path to save output with 'output' flag
+	OutputDir   string              // set the directory path to save output with `output-dir` flag
+	Format      string              // set the output format
 	ForceUpdate bool                // set whether to force updating SMD with 'force-update' flag
 	AccessToken string              // set the access token to include in request with 'access-token' flag
 	SecretStore secrets.SecretStore // set BMC credentials
@ -66,34 +67,9 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin
 		found      = make([]string, 0, len(*assets))
 		done       = make(chan struct{}, params.Concurrency+1)
 		chanAssets = make(chan RemoteAsset, params.Concurrency+1)
-		outputPath = path.Clean(params.OutputPath)
-		smdClient  = &client.SmdClient{Client: &http.Client{}}
 	)

 	// set the client's params from CLI
-	// NOTE: temporary solution until client.NewClient() is fixed
-	smdClient.URI = params.URI
-	if params.CaCertPath != "" {
-		cacert, err := os.ReadFile(params.CaCertPath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read CA cert path: %w", err)
-		}
-		certPool := x509.NewCertPool()
-		certPool.AppendCertsFromPEM(cacert)
-		smdClient.Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{
-				RootCAs:            certPool,
-				InsecureSkipVerify: true,
-			},
-			DisableKeepAlives: true,
-			Dial: (&net.Dialer{
-				Timeout:   120 * time.Second,
-				KeepAlive: 120 * time.Second,
-			}).Dial,
-			TLSHandshakeTimeout:   120 * time.Second,
-			ResponseHeaderTimeout: 120 * time.Second,
-		}
-	}
 	wg.Add(params.Concurrency)
 	for i := 0; i < params.Concurrency; i++ {
 		go func() {
@ -141,7 +117,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin
 				}

 				// we didn't find anything so do not proceed
-				if len(systems) == 0 && len(managers) == 0 {
+				if util.IsEmpty(systems) && util.IsEmpty(managers) {
 					continue
 				}

@ -167,6 +143,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin

 				// optionally, add the MACAddr property if we find a matching IP
 				// from the correct ethernet interface
+
 				host := sr.Host
 				str_protocol := "https://"
 				if strings.Contains(host, str_protocol) {
@ -185,59 +162,9 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin
 				headers.Authorization(params.AccessToken)
 				headers.ContentType("application/json")

-				body, err := json.MarshalIndent(data, "", "    ")
-				if err != nil {
-					log.Error().Err(err).Msgf("failed to marshal output to JSON")
-				}
-
-				if params.Verbose {
-					fmt.Printf("%v\n", string(body))
-				}
-
 				// add data output to collections
 				collection = append(collection, data)

-				// write JSON data to file if output path is set using hive partitioning strategy
-				if outputPath != "" {
-					var (
-						finalPath = fmt.Sprintf("./%s/%s/%d.json", outputPath, data["ID"], time.Now().Unix())
-						finalDir  = filepath.Dir(finalPath)
-					)
-					// if it doesn't, make the directory and write file
-					err = os.MkdirAll(finalDir, 0o777)
-					if err == nil { // no error
-						err = os.WriteFile(path.Clean(finalPath), body, os.ModePerm)
-						if err != nil {
-							log.Error().Err(err).Msgf("failed to write collect output to file")
-						}
-
-					} else { // error is set
-						log.Error().Err(err).Msg("failed to make directory for collect output")
-					}
-				}
-
-				// add all endpoints to SMD ONLY if a host is provided
-				if smdClient.URI != "" {
-					err = smdClient.Add(body, headers)
-					if err != nil {
-
-						// try updating instead
-						if params.ForceUpdate {
-							smdClient.Xname = data["ID"].(string)
-							err = smdClient.Update(body, headers)
-							if err != nil {
-								log.Error().Err(err).Msgf("failed to forcibly update Redfish endpoint")
-							}
-						} else {
-							log.Error().Err(err).Msgf("failed to add Redfish endpoint")
-						}
-					}
-				} else {
-					if params.Verbose {
-						log.Warn().Msg("no request made (host argument is empty)")
-					}
-				}
-
 				// got host information, so add to list of already probed hosts
 				found = append(found, sr.Host)
 			}
@ -269,6 +196,64 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin
 	wg.Wait()
 	close(done)

+	var (
+		output []byte
+		err    error
+	)
+
+	// format our output to write to file or standard out
+	switch params.Format {
+	case "json":
+		output, err = json.MarshalIndent(collection, "", "    ")
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to marshal output to JSON")
+		}
+	case "yaml":
+		output, err = yaml.Marshal(collection)
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to marshal output to YAML")
+		}
+	}
+
+	// print the final combined output at the end to write to file
+	if params.Verbose {
+		fmt.Printf("%v\n", string(output))
+	}
+
+	// write data to file in preset directory if output path is set using set format
+	if params.OutputDir != "" {
+		for _, data := range collection {
+			var (
+				finalPath = fmt.Sprintf("./%s/%s/%d.%s", path.Clean(params.OutputDir), data["ID"], time.Now().Unix(), params.Format)
+				finalDir  = filepath.Dir(finalPath)
+			)
+			// if it doesn't, make the directory and write file
+			err = os.MkdirAll(finalDir, 0o777)
+			if err == nil { // no error
+				err = os.WriteFile(path.Clean(finalPath), output, os.ModePerm)
+				if err != nil {
+					log.Error().Err(err).Msgf("failed to write collect output to file")
+				}
+			} else { // error is set
+				log.Error().Err(err).Msg("failed to make directory for collect output")
+			}
+		}
+	}
+
+	// write data to only to the path set (no preset directory structure)
+	if params.OutputPath != "" {
+		// if it doesn't, make the directory and write file
+		err = os.MkdirAll(filepath.Dir(params.OutputPath), 0o777)
+		if err == nil { // no error
+			err = os.WriteFile(path.Clean(params.OutputPath), output, os.ModePerm)
+			if err != nil {
+				log.Error().Err(err).Msgf("failed to write collect output to file")
+			}
+		} else { // error is set
+			log.Error().Err(err).Msg("failed to make directory for collect output")
+		}
+	}
+
 	return collection, nil
 }

@ -345,6 +330,7 @@ func FindMACAddressWithIP(config crawler.CrawlerConfig, targetIP net.IP) (string
 			continue
 		}
 	}
+
 	// no matches found, so return an empty string
 	return "", fmt.Errorf("no ethernet interfaces found with IP address")
 }
--- a/pkg/crawler/main.go
+++ b/pkg/crawler/main.go
@ -122,17 +122,17 @@ func CrawlBMCForSystems(config CrawlerConfig) ([]InventoryDetail, error) {

 	// Obtain the ServiceRoot
 	rf_service := client.GetService()
-	log.Info().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion)
+	log.Debug().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion)

 	// Nodes are sometimes only found under Chassis, but they should be found under Systems.
 	rf_chassis, err := rf_service.Chassis()
 	if err == nil {
-		log.Info().Msgf("found %d chassis in ServiceRoot", len(rf_chassis))
+		log.Debug().Msgf("found %d chassis in ServiceRoot", len(rf_chassis))
 		for _, chassis := range rf_chassis {
 			rf_chassis_systems, err := chassis.ComputerSystems()
 			if err == nil {
 				rf_systems = append(rf_systems, rf_chassis_systems...)
-				log.Info().Msgf("found %d systems in chassis %s", len(rf_chassis_systems), chassis.ID)
+				log.Debug().Msgf("found %d systems in chassis %s", len(rf_chassis_systems), chassis.ID)
 			}
 		}
 	}
@ -140,7 +140,7 @@ func CrawlBMCForSystems(config CrawlerConfig) ([]InventoryDetail, error) {
 	if err != nil {
 		log.Error().Err(err).Msg("failed to get systems from ServiceRoot")
 	}
-	log.Info().Msgf("found %d systems in ServiceRoot", len(rf_root_systems))
+	log.Debug().Msgf("found %d systems in ServiceRoot", len(rf_root_systems))
 	rf_systems = append(rf_systems, rf_root_systems...)
 	return walkSystems(rf_systems, nil, config.URI)
 }
@ -198,7 +198,7 @@ func CrawlBMCForManagers(config CrawlerConfig) ([]Manager, error) {

 	// Obtain the ServiceRoot
 	rf_service := client.GetService()
-	log.Info().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion)
+	log.Debug().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion)

 	rf_managers, err := rf_service.Managers()
 	if err != nil {
@ -375,7 +375,7 @@ func loadBMCCreds(config CrawlerConfig) (bmc.BMCCredentials, error) {
 		return bmc.BMCCredentials{}, fmt.Errorf("credential store is invalid")
 	}
 	if creds := util.GetBMCCredentials(config.CredentialStore, config.URI); creds == (bmc.BMCCredentials{}) {
-		return creds, fmt.Errorf("%s: credentials blank for BNC", config.URI)
+		return creds, fmt.Errorf("%s: credentials blank for BMC", config.URI)
 	} else {
 		return creds, nil
 	}