Merge branch 'main' into cache-cmd

internal/cache/sqlite/sqlite.go

@@ -106,11 +106,9 @@ func DeleteScannedAssets(path string, assets ...magellan.RemoteAsset) error {
 
 func GetScannedAssets(path string) ([]magellan.RemoteAsset, error) {
 	// check if path exists first to prevent creating the database
-	exists, err := util.PathExists(path)
+	_, exists := util.PathExists(path)
 	if !exists {
 		return nil, fmt.Errorf("no file found")
-	} else if err != nil {
-		return nil, err
 	}
 
 	// now check if the file is the SQLite database

internal/collect.go

@@ -10,18 +10,20 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"strings"
+	"path/filepath"
 	"sync"
 	"time"
 
 	"github.com/OpenCHAMI/magellan/pkg/client"
 	"github.com/OpenCHAMI/magellan/pkg/crawler"
 
-	"github.com/OpenCHAMI/magellan/internal/util"
 	"github.com/rs/zerolog/log"
 
 	"github.com/Cray-HPE/hms-xname/xnames"
 	_ "github.com/mattn/go-sqlite3"
-	_ "github.com/stmcginnis/gofish"
+	"github.com/stmcginnis/gofish"
+	"github.com/stmcginnis/gofish/redfish"
 	"golang.org/x/exp/slices"
 )
 
@@ -41,12 +43,13 @@ type CollectParams struct {
 }
 
 // This is the main function used to collect information from the BMC nodes via Redfish.
+// The results of the collect are stored in a cache specified with the `--cache` flag.
 // The function expects a list of hosts found using the `ScanForAssets()` function.
 //
 // Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency
-// property value between 1 and 255.
+// property value between 1 and 10000.
 func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error {
-	// check for available probe states
+	// check for available remote assets found from scan
 	if assets == nil {
 		return fmt.Errorf("no assets found")
 	}
@@ -109,14 +112,23 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error {
 				offset += 1
 
 				// crawl BMC node to fetch inventory data via Redfish
-				systems, err := crawler.CrawlBMC(crawler.CrawlerConfig{
-					URI:      fmt.Sprintf("%s:%d", sr.Host, sr.Port),
-					Username: params.Username,
-					Password: params.Password,
-					Insecure: true,
-				})
+				var (
+					systems  []crawler.InventoryDetail
+					managers []crawler.Manager
+					config   = crawler.CrawlerConfig{
+						URI:      fmt.Sprintf("%s:%d", sr.Host, sr.Port),
+						Username: params.Username,
+						Password: params.Password,
+						Insecure: true,
+					}
+				)
+				systems, err := crawler.CrawlBMCForSystems(config)
 				if err != nil {
-					log.Error().Err(err).Msgf("failed to crawl BMC")
+					log.Error().Err(err).Msg("failed to crawl BMC for systems")
+				}
+				managers, err = crawler.CrawlBMCForManagers(config)
+				if err != nil {
+					log.Error().Err(err).Msg("failed to crawl BMC for managers")
 				}
 
 				// data to be sent to smd
@@ -129,9 +141,20 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error {
 					"MACRequired":        true,
 					"RediscoverOnUpdate": false,
 					"Systems":            systems,
+					"Managers":           managers,
 					"SchemaVersion":      1,
 				}
 
+				// optionally, add the MACAddr property if we find a matching IP
+				// from the correct ethernet interface
+				mac, err := FindMACAddressWithIP(config, net.ParseIP(sr.Host))
+				if err != nil {
+					log.Warn().Err(err).Msgf("failed to find MAC address with IP '%s'", sr.Host)
+				}
+				if mac != "" {
+					data["MACAddr"] = mac
+				}
+
 				// create and set headers for request
 				headers := client.HTTPHeader{}
 				headers.Authorization(params.AccessToken)
@@ -148,25 +171,20 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error {
 
 				// write JSON data to file if output path is set using hive partitioning strategy
 				if outputPath != "" {
-					// make directory if it does exists
-					exists, err := util.PathExists(outputPath)
-					if err == nil && !exists {
-						err = os.MkdirAll(outputPath, 0o644)
+					var (
+						finalPath = fmt.Sprintf("./%s/%s/%d.json", outputPath, data["ID"], time.Now().Unix())
+						finalDir  = filepath.Dir(finalPath)
+					)
+					// if it doesn't, make the directory and write file
+					err = os.MkdirAll(finalDir, 0o777)
+					if err == nil { // no error
+						err = os.WriteFile(path.Clean(finalPath), body, os.ModePerm)
 						if err != nil {
-							log.Error().Err(err).Msg("failed to make directory for output")
-						} else {
-							// make the output directory to store files
-							outputPath, err := util.MakeOutputDirectory(outputPath, false)
-							if err != nil {
-								log.Error().Err(err).Msg("failed to make output directory")
-							} else {
-								// write the output to the final path
-								err = os.WriteFile(path.Clean(fmt.Sprintf("%s/%s/%d.json", params.URI, outputPath, time.Now().Unix())), body, os.ModePerm)
-								if err != nil {
-									log.Error().Err(err).Msgf("failed to write data to file")
-								}
-							}
+							log.Error().Err(err).Msgf("failed to write collect output to file")
 						}
+
+					} else { // error is set
+						log.Error().Err(err).Msg("failed to make directory for collect output")
 					}
 				}
 
@@ -225,3 +243,75 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error {
 
 	return nil
 }
+
+// FindMACAddressWithIP() returns the MAC address of an ethernet interface with
+// a matching IPv4Address. Returns an empty string and error if there are no matches
+// found.
+func FindMACAddressWithIP(config crawler.CrawlerConfig, targetIP net.IP) (string, error) {
+	// get the managers to find the BMC MAC address compared with IP
+	//
+	// NOTE: Since we don't have a RedfishEndpoint type abstraction in
+	// magellan and the crawler crawls for systems information, it
+	// may just make more sense to get the managers directly via
+	// gofish (at least for now). If there's a need for grabbing more
+	// manager information in the future, we can move the logic into
+	// the crawler.
+	client, err := gofish.Connect(gofish.ClientConfig{
+		Endpoint:  config.URI,
+		Username:  config.Username,
+		Password:  config.Password,
+		Insecure:  config.Insecure,
+		BasicAuth: true,
+	})
+	if err != nil {
+		if strings.HasPrefix(err.Error(), "404:") {
+			err = fmt.Errorf("no ServiceRoot found.  This is probably not a BMC: %s", config.URI)
+		}
+		if strings.HasPrefix(err.Error(), "401:") {
+			err = fmt.Errorf("authentication failed.  Check your username and password: %s", config.URI)
+		}
+		event := log.Error()
+		event.Err(err)
+		event.Msg("failed to connect to BMC")
+		return "", err
+	}
+	defer client.Logout()
+
+	var (
+		rf_service  = client.GetService()
+		rf_managers []*redfish.Manager
+	)
+	rf_managers, err = rf_service.Managers()
+	if err != nil {
+		return "", fmt.Errorf("failed to get managers: %v", err)
+	}
+
+	// find the manager with the same IP address of the BMC to get
+	// it's MAC address from its EthernetInterface
+	for _, manager := range rf_managers {
+		eths, err := manager.EthernetInterfaces()
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to get ethernet interfaces from manager '%s'", manager.Name)
+			continue
+		}
+		for _, eth := range eths {
+			// compare the ethernet interface IP with argument
+			for _, ip := range eth.IPv4Addresses {
+				if ip.Address == targetIP.String() {
+					// we found matching IP address so return the ethernet interface MAC
+					return eth.MACAddress, nil
+				}
+			}
+			// do the same thing as above, but with static IP addresses
+			for _, ip := range eth.IPv4StaticAddresses {
+				if ip.Address == targetIP.String() {
+					return eth.MACAddress, nil
+				}
+			}
+			// no matches found, so go to next ethernet interface
+			continue
+		}
+	}
+	// no matches found, so return an empty string
+	return "", fmt.Errorf("no ethernet interfaces found with IP address")
+}

internal/util/path.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -13,15 +14,9 @@ import (
 //
 // Returns whether the path exists and no error if successful,
 // otherwise, it returns false with an error.
-func PathExists(path string) (bool, error) {
-	_, err := os.Stat(path)
-	if err == nil {
-		return true, nil
-	}
-	if os.IsNotExist(err) {
-		return false, nil
-	}
-	return false, err
+func PathExists(path string) (fs.FileInfo, bool) {
+	fi, err := os.Stat(path)
+	return fi, !os.IsNotExist(err)
 }
 
 // SplitPathForViper() is an utility function to split a path into 3 parts:
@@ -51,17 +46,14 @@ func MakeOutputDirectory(path string, overwrite bool) (string, error) {
 	final := path + "/" + dirname
 
 	// check if path is valid and directory
-	pathExists, err := PathExists(final)
-	if err != nil {
-		return "", fmt.Errorf("failed to check for existing path: %v", err)
-	}
+	_, pathExists := PathExists(final)
 	if pathExists && !overwrite {
 		// make sure it is directory with 0o644 permissions
 		return "", fmt.Errorf("found existing path: %v", final)
 	}
 
 	// create directory with data + time
-	err = os.MkdirAll(final, 0766)
+	err := os.MkdirAll(final, 0766)
 	if err != nil {
 		return "", fmt.Errorf("failed to make directory: %v", err)
 	}

internal/util/util.go

@@ -0,0 +1,27 @@
+package util
+
+import (
+	"fmt"
+	"time"
+)
+
+// CheckUntil regularly check a predicate until it's true or time out is reached.
+func CheckUntil(interval time.Duration, timeout time.Duration, predicate func() (bool, error)) error {
+	timeoutCh := time.After(timeout)
+
+	for {
+		select {
+		case <-time.After(interval):
+			predTrue, err := predicate()
+			if predTrue {
+				return nil
+			}
+
+			if err != nil {
+				return err
+			}
+		case <-timeoutCh:
+			return fmt.Errorf("timeout of %ds reached", int64(timeout/time.Second))
+		}
+	}
+}

internal/version/version.go

@@ -0,0 +1,60 @@
+package version
+
+import (
+	"fmt"
+)
+
+// GitCommit stores the latest Git commit hash.
+// Set via -ldflags "-X github.com/OpenCHAMI/magellan/internal/version.GitCommit=$(git rev-parse HEAD)"
+var GitCommit string
+
+// BuildTime stores the build timestamp in UTC.
+// Set via -ldflags "-X github.com/OpenCHAMI/magellan/internal/version.BuildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+var BuildTime string
+
+// Version indicates the version of the binary, such as a release number or semantic version.
+// Set via -ldflags "-X github.com/OpenCHAMI/magellan/internal/version.Version=v1.0.0"
+var Version string
+
+// GitBranch holds the name of the Git branch from which the build was created.
+// Set via -ldflags "-X github.com/OpenCHAMI/magellan/internal/version.GitBranch=$(git rev-parse --abbrev-ref HEAD)"
+var GitBranch string
+
+// GitTag represents the most recent Git tag at build time, if any.
+// Set via -ldflags "-X github.com/OpenCHAMI/magellan/internal/version.GitTag=$(git describe --tags --abbrev=0)"
+var GitTag string
+
+// GitState indicates whether the working directory was "clean" or "dirty" (i.e., with uncommitted changes).
+// Set via -ldflags "-X github.com/OpenCHAMI/magellan/internal/version.GitState=$(if git diff-index --quiet HEAD --; then echo 'clean'; else echo 'dirty'; fi)"
+var GitState string
+
+// BuildHost stores the hostname of the machine where the binary was built.
+// Set via -ldflags "-X github.com/OpenCHAMI/magellan/internal/version.BuildHost=$(hostname)"
+var BuildHost string
+
+// GoVersion captures the Go version used to build the binary.
+// Typically, this can be obtained automatically with runtime.Version(), but you can set it manually.
+// Set via -ldflags "-X github.com/OpenCHAMI/magellan/internal/version.GoVersion=$(go version | awk '{print $3}')"
+var GoVersion string
+
+// BuildUser is the username of the person or system that initiated the build process.
+// Set via -ldflags "-X github.com/OpenCHAMI/magellan/internal/version.BuildUser=$(whoami)"
+var BuildUser string
+
+// PrintVersionInfo outputs all versioning information for troubleshooting or version checks.
+func PrintVersionInfo() {
+	fmt.Printf("Version: %s\n", Version)
+	fmt.Printf("Git Commit: %s\n", GitCommit)
+	fmt.Printf("Build Time: %s\n", BuildTime)
+	fmt.Printf("Git Branch: %s\n", GitBranch)
+	fmt.Printf("Git Tag: %s\n", GitTag)
+	fmt.Printf("Git State: %s\n", GitState)
+	fmt.Printf("Build Host: %s\n", BuildHost)
+	fmt.Printf("Go Version: %s\n", GoVersion)
+	fmt.Printf("Build User: %s\n", BuildUser)
+}
+
+func VersionInfo() string {
+	return fmt.Sprintf("Version: %s, Git Commit: %s, Build Time: %s, Git Branch: %s, Git Tag: %s, Git State: %s, Build Host: %s, Go Version: %s, Build User: %s",
+		Version, GitCommit, BuildTime, GitBranch, GitTag, GitState, BuildHost, GoVersion, BuildUser)
+}