mirror of
https://github.com/davidallendj/magellan.git
synced 2025-12-20 11:37:01 -07:00
Removed unused code, rename vars, and changed output to use hive partitioning strategy
This commit is contained in:
David Allen
parent
aefce13f57
commit
606a7b47cc
1 changed files with 64 additions and 367 deletions
|
|
@ -2,27 +2,22 @@
|
|||
package magellan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/OpenCHAMI/magellan/internal/log"
|
||||
"github.com/OpenCHAMI/magellan/pkg/client"
|
||||
"github.com/OpenCHAMI/magellan/pkg/crawler"
|
||||
|
||||
"github.com/OpenCHAMI/magellan/internal/api/smd"
|
||||
"github.com/OpenCHAMI/magellan/internal/util"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/Cray-HPE/hms-xname/xnames"
|
||||
bmclib "github.com/bmc-toolbox/bmclib/v2"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"github.com/stmcginnis/gofish"
|
||||
_ "github.com/stmcginnis/gofish"
|
||||
"github.com/stmcginnis/gofish/redfish"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
|
|
@ -39,7 +34,6 @@ const (
|
|||
type QueryParams struct {
|
||||
Host string // set by the 'host' flag
|
||||
Port int // set by the 'port' flag
|
||||
Protocol string // set by the 'protocol' flag
|
||||
Username string // set the BMC username with the 'username' flag
|
||||
Password string // set the BMC password with the 'password' flag
|
||||
Concurrency int // set the of concurrent jobs with the 'concurrency' flag
|
||||
|
|
@ -56,44 +50,38 @@ type QueryParams struct {
|
|||
//
|
||||
// Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency
|
||||
// property value between 1 and 255.
|
||||
func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) error {
|
||||
func CollectInventory(scannedResults *[]ScannedResult, params *QueryParams) error {
|
||||
// check for available probe states
|
||||
if probeStates == nil {
|
||||
if scannedResults == nil {
|
||||
return fmt.Errorf("no probe states found")
|
||||
}
|
||||
if len(*probeStates) <= 0 {
|
||||
if len(*scannedResults) <= 0 {
|
||||
return fmt.Errorf("no probe states found")
|
||||
}
|
||||
|
||||
// make the output directory to store files
|
||||
outputPath := path.Clean(q.OutputPath)
|
||||
outputPath, err := util.MakeOutputDirectory(outputPath)
|
||||
if err != nil {
|
||||
l.Log.Errorf("failed to make output directory: %v", err)
|
||||
}
|
||||
|
||||
// collect bmc information asynchronously
|
||||
var (
|
||||
offset = 0
|
||||
wg sync.WaitGroup
|
||||
found = make([]string, 0, len(*probeStates))
|
||||
done = make(chan struct{}, q.Concurrency+1)
|
||||
chanProbeState = make(chan ScannedResult, q.Concurrency+1)
|
||||
client = smd.NewClient(
|
||||
smd.WithSecureTLS(q.CaCertPath),
|
||||
offset = 0
|
||||
wg sync.WaitGroup
|
||||
found = make([]string, 0, len(*scannedResults))
|
||||
done = make(chan struct{}, params.Concurrency+1)
|
||||
chanScannedResult = make(chan ScannedResult, params.Concurrency+1)
|
||||
outputPath = path.Clean(params.OutputPath)
|
||||
smdClient = client.NewClient(
|
||||
client.WithSecureTLS(params.CaCertPath),
|
||||
)
|
||||
)
|
||||
wg.Add(q.Concurrency)
|
||||
for i := 0; i < q.Concurrency; i++ {
|
||||
wg.Add(params.Concurrency)
|
||||
for i := 0; i < params.Concurrency; i++ {
|
||||
go func() {
|
||||
for {
|
||||
ps, ok := <-chanProbeState
|
||||
sr, ok := <-chanScannedResult
|
||||
if !ok {
|
||||
wg.Done()
|
||||
return
|
||||
}
|
||||
q.Host = ps.Host
|
||||
q.Port = ps.Port
|
||||
params.Host = sr.Host
|
||||
params.Port = sr.Port
|
||||
|
||||
// generate custom xnames for bmcs
|
||||
node := xnames.Node{
|
||||
|
|
@ -104,107 +92,91 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err
|
|||
}
|
||||
offset += 1
|
||||
|
||||
gofishClient, err := connectGofish(q)
|
||||
// TODO: use pkg/crawler to request inventory data via Redfish
|
||||
systems, err := crawler.CrawlBMC(crawler.CrawlerConfig{
|
||||
URI: fmt.Sprintf("https://%s:%d", sr.Host, sr.Port),
|
||||
Username: params.Username,
|
||||
Password: params.Password,
|
||||
Insecure: true,
|
||||
})
|
||||
if err != nil {
|
||||
l.Log.Errorf("failed to connect to BMC (%v:%v): %v", q.Host, q.Port, err)
|
||||
log.Error().Err(err).Msgf("failed to crawl BMC")
|
||||
}
|
||||
defer gofishClient.Logout()
|
||||
|
||||
// data to be sent to smd
|
||||
data := map[string]any{
|
||||
"ID": fmt.Sprintf("%v", node.String()[:len(node.String())-2]),
|
||||
"Type": "",
|
||||
"Name": "",
|
||||
"FQDN": ps.Host,
|
||||
"User": q.Username,
|
||||
// "Password": q.Pass,
|
||||
"ID": fmt.Sprintf("%v", node.String()[:len(node.String())-2]),
|
||||
"Type": "",
|
||||
"Name": "",
|
||||
"FQDN": sr.Host,
|
||||
"User": params.Username,
|
||||
"MACRequired": true,
|
||||
"RediscoverOnUpdate": false,
|
||||
"Systems": systems,
|
||||
}
|
||||
|
||||
// chassis
|
||||
if gofishClient != nil {
|
||||
chassis, err := CollectChassis(gofishClient, q)
|
||||
if err != nil {
|
||||
l.Log.Errorf("failed to collect chassis: %v", err)
|
||||
continue
|
||||
}
|
||||
data["Chassis"] = chassis
|
||||
|
||||
// systems
|
||||
systems, err := CollectSystems(gofishClient, q)
|
||||
if err != nil {
|
||||
l.Log.Errorf("failed to collect systems: %v", err)
|
||||
}
|
||||
data["Systems"] = systems
|
||||
|
||||
// add other fields from systems
|
||||
if len(systems) > 0 {
|
||||
system := systems[0]["Data"].(*redfish.ComputerSystem)
|
||||
if system == nil {
|
||||
l.Log.Errorf("invalid system data (data is nil)")
|
||||
} else {
|
||||
data["Name"] = system.Name
|
||||
}
|
||||
}
|
||||
} else {
|
||||
l.Log.Errorf("invalid client (client is nil)")
|
||||
continue
|
||||
}
|
||||
|
||||
headers := make(map[string]string)
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
// use access token in authorization header if we have it
|
||||
if q.AccessToken != "" {
|
||||
headers["Authorization"] = "Bearer " + q.AccessToken
|
||||
}
|
||||
// create and set headers for request
|
||||
headers := util.HTTPHeader{}
|
||||
headers.Authorization(params.AccessToken)
|
||||
headers.ContentType("application/json")
|
||||
|
||||
body, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
l.Log.Errorf("failed to marshal output to JSON: %v", err)
|
||||
log.Error().Err(err).Msgf("failed to marshal output to JSON")
|
||||
}
|
||||
|
||||
if q.Verbose {
|
||||
if params.Verbose {
|
||||
fmt.Printf("%v\n", string(body))
|
||||
}
|
||||
|
||||
// write JSON data to file if output path is set
|
||||
// write JSON data to file if output path is set using hive partitioning strategy
|
||||
if outputPath != "" {
|
||||
err = os.WriteFile(path.Clean(outputPath+"/"+q.Host+".json"), body, os.ModePerm)
|
||||
err = os.MkdirAll(outputPath, os.ModeDir)
|
||||
if err != nil {
|
||||
l.Log.Errorf("failed to write data to file: %v", err)
|
||||
log.Error().Err(err).Msg("failed to make directory for output")
|
||||
} else {
|
||||
// make the output directory to store files
|
||||
outputPath, err := util.MakeOutputDirectory(outputPath, false)
|
||||
if err != nil {
|
||||
log.Error().Msgf("failed to make output directory: %v", err)
|
||||
} else {
|
||||
// write the output to the final path
|
||||
err = os.WriteFile(path.Clean(fmt.Sprintf("%s/%s/%d.json", params.Host, outputPath, time.Now().Unix())), body, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("failed to write data to file")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// add all endpoints to smd
|
||||
err = client.AddRedfishEndpoint(body, headers)
|
||||
err = smdClient.AddRedfishEndpoint(data, headers)
|
||||
if err != nil {
|
||||
l.Log.Error(err)
|
||||
log.Error().Err(err).Msgf("failed to add Redfish endpoint")
|
||||
|
||||
// try updating instead
|
||||
if q.ForceUpdate {
|
||||
err = client.UpdateRedfishEndpoint(data["ID"].(string), body, headers)
|
||||
if params.ForceUpdate {
|
||||
err = smdClient.UpdateRedfishEndpoint(data["ID"].(string), body, headers)
|
||||
if err != nil {
|
||||
l.Log.Error(err)
|
||||
log.Error().Err(err).Msgf("failed to update Redfish endpoint")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// got host information, so add to list of already probed hosts
|
||||
found = append(found, ps.Host)
|
||||
found = append(found, sr.Host)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// use the found results to query bmc information
|
||||
for _, ps := range *probeStates {
|
||||
for _, ps := range *scannedResults {
|
||||
// skip if found info from host
|
||||
foundHost := slices.Index(found, ps.Host)
|
||||
if !ps.State || foundHost >= 0 {
|
||||
continue
|
||||
}
|
||||
chanProbeState <- ps
|
||||
chanScannedResult <- ps
|
||||
}
|
||||
|
||||
// handle goroutine paths
|
||||
|
|
@ -218,288 +190,13 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err
|
|||
}
|
||||
}()
|
||||
|
||||
close(chanProbeState)
|
||||
close(chanScannedResult)
|
||||
wg.Wait()
|
||||
close(done)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CollectEthernetInterfaces() collects all of the ethernet interfaces found
|
||||
// from all systems from under the "/redfish/v1/Systems" endpoint.
|
||||
//
|
||||
// TODO: This function needs to be refactored entirely...if not deleted
|
||||
// in favor of using crawler.CrawlBM() instead.
|
||||
func CollectEthernetInterfaces(c *gofish.APIClient, q *QueryParams, systemID string) ([]byte, error) {
|
||||
// TODO: add more endpoints to test for ethernet interfaces
|
||||
// /redfish/v1/Chassis/{ChassisID}/NetworkAdapters/{NetworkAdapterId}/NetworkDeviceFunctions/{NetworkDeviceFunctionId}/EthernetInterfaces/{EthernetInterfaceId}
|
||||
// /redfish/v1/Managers/{ManagerId}/EthernetInterfaces/{EthernetInterfaceId}
|
||||
// /redfish/v1/Systems/{ComputerSystemId}/EthernetInterfaces/{EthernetInterfaceId}
|
||||
// /redfish/v1/Systems/{ComputerSystemId}/OperatingSystem/Containers/EthernetInterfaces/{EthernetInterfaceId}
|
||||
systems, err := c.Service.Systems()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get systems: (%v:%v): %v", q.Host, q.Port, err)
|
||||
}
|
||||
|
||||
var (
|
||||
interfaces []*redfish.EthernetInterface
|
||||
errList []error
|
||||
)
|
||||
|
||||
// get all of the ethernet interfaces in our systems
|
||||
for _, system := range systems {
|
||||
system.EthernetInterfaces()
|
||||
eth, err := redfish.ListReferencedEthernetInterfaces(c, "/redfish/v1/Systems/"+system.ID+"/EthernetInterfaces")
|
||||
if err != nil {
|
||||
errList = append(errList, err)
|
||||
}
|
||||
|
||||
interfaces = append(interfaces, eth...)
|
||||
}
|
||||
|
||||
// print any report errors
|
||||
err = util.FormatErrorList(errList)
|
||||
if util.HasErrors(errList) {
|
||||
return nil, fmt.Errorf("failed to get ethernet interfaces with %d error(s): \n%v", len(errList), err)
|
||||
}
|
||||
|
||||
data := map[string]any{"EthernetInterfaces": interfaces}
|
||||
b, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal JSON: %v", err)
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// CollectChassis() fetches all chassis related information from each node specified
|
||||
// via the Redfish API. Like the other collect functions, this function uses the gofish
|
||||
// library to make requests to each node. Additionally, all of the network adapters found
|
||||
// are added to the output as well.
|
||||
//
|
||||
// Returns a map that represents a Chassis object with NetworkAdapters.
|
||||
func CollectChassis(c *gofish.APIClient, q *QueryParams) ([]map[string]any, error) {
|
||||
rfChassis, err := c.Service.Chassis()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query chassis (%v:%v): %v", q.Host, q.Port, err)
|
||||
}
|
||||
|
||||
var chassis []map[string]any
|
||||
for _, ch := range rfChassis {
|
||||
networkAdapters, err := ch.NetworkAdapters()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get network adapters: %v", err)
|
||||
}
|
||||
|
||||
chassis = append(chassis, map[string]any{
|
||||
"Data": ch,
|
||||
"NetworkAdapters": networkAdapters,
|
||||
})
|
||||
}
|
||||
|
||||
return chassis, nil
|
||||
}
|
||||
|
||||
// CollectSystems pulls system information from each BMC node via Redfish using the
|
||||
// `gofish` library.
|
||||
//
|
||||
// The process of collecting this info is as follows:
|
||||
// 1. check if system has ethernet interfaces
|
||||
// 1.a. if yes, create system data and ethernet interfaces JSON
|
||||
// 1.b. if no, try to get data using manager instead
|
||||
// 2. check if manager has "ManagerForServices" and "EthernetInterfaces" properties
|
||||
// 2.a. if yes, query both properties to use in next step
|
||||
// 2.b. for each service, query its data and add the ethernet interfaces
|
||||
// 2.c. add the system to list of systems to marshal and return
|
||||
func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]map[string]any, error) {
|
||||
rfSystems, err := c.Service.Systems()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get systems (%v:%v): %v", q.Host, q.Port, err)
|
||||
}
|
||||
|
||||
var systems []map[string]any
|
||||
for _, system := range rfSystems {
|
||||
eths, err := system.EthernetInterfaces()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get system ethernet interfaces: %v", err)
|
||||
}
|
||||
|
||||
// try and get ethernet interfaces through manager if empty
|
||||
if len(eths) <= 0 {
|
||||
if q.Verbose {
|
||||
fmt.Printf("no system ethernet interfaces found...trying to get from managers interface\n")
|
||||
}
|
||||
|
||||
managedBy, err := system.ManagedBy()
|
||||
if err == nil {
|
||||
for _, managerLink := range system.ManagedBy {
|
||||
// try getting ethernet interface from all managers until one is found
|
||||
eths, err = redfish.ListReferencedEthernetInterfaces(c, managerLink+"/EthernetInterfaces")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get system manager ethernet interfaces: %v", err)
|
||||
}
|
||||
if len(eths) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// add network interfaces to system
|
||||
rfNetworkInterfaces, err := system.NetworkInterfaces()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get system network interfaces: %v", err)
|
||||
}
|
||||
|
||||
// get the network adapter ID for each network interface
|
||||
var networkInterfaces []map[string]any
|
||||
for _, rfNetworkInterface := range rfNetworkInterfaces {
|
||||
networkAdapter, err := rfNetworkInterface.NetworkAdapter()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get network adapter: %v", err)
|
||||
}
|
||||
|
||||
networkInterfaces = append(networkInterfaces, map[string]any{
|
||||
"Data": rfNetworkInterface,
|
||||
"NetworkAdapterId": networkAdapter.ID,
|
||||
})
|
||||
}
|
||||
|
||||
// add system to collection of systems
|
||||
systems = append(systems, map[string]any{
|
||||
"Data": system,
|
||||
"EthernetInterfaces": eths,
|
||||
"NetworkInterfaces": networkInterfaces,
|
||||
})
|
||||
}
|
||||
|
||||
return systems, nil
|
||||
}
|
||||
|
||||
// TODO: MAYBE DELETE???
|
||||
func CollectProcessors(q *QueryParams) ([]byte, error) {
|
||||
url := baseRedfishUrl(q) + "/Systems"
|
||||
res, body, err := util.MakeRequest(nil, url, "GET", nil, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("something went wrong: %v", err)
|
||||
} else if res == nil {
|
||||
return nil, fmt.Errorf("no response returned (url: %s)", url)
|
||||
} else if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("returned status code %d", res.StatusCode)
|
||||
}
|
||||
|
||||
// convert to not get base64 string
|
||||
var procs map[string]json.RawMessage
|
||||
var members []map[string]any
|
||||
err = json.Unmarshal(body, &procs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal processors: %v", err)
|
||||
}
|
||||
err = json.Unmarshal(procs["Members"], &members)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal processor members: %v", err)
|
||||
}
|
||||
|
||||
// request data about each processor member on node
|
||||
for _, member := range members {
|
||||
var oid = member["@odata.id"].(string)
|
||||
var infoUrl = url + oid
|
||||
res, _, err := util.MakeRequest(nil, infoUrl, "GET", nil, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("something went wrong: %v", err)
|
||||
} else if res == nil {
|
||||
return nil, fmt.Errorf("no response returned (url: %s)", url)
|
||||
} else if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("returned status code %d", res.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
data := map[string]any{"Processors": procs}
|
||||
b, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal JSON: %v", err)
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func connectGofish(q *QueryParams) (*gofish.APIClient, error) {
|
||||
config, err := makeGofishConfig(q)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to make gofish config: %v", err)
|
||||
}
|
||||
c, err := gofish.Connect(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to redfish endpoint: %v", err)
|
||||
}
|
||||
if c != nil {
|
||||
c.Service.ProtocolFeaturesSupported = gofish.ProtocolFeaturesSupported{
|
||||
ExpandQuery: gofish.Expand{
|
||||
ExpandAll: true,
|
||||
Links: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
return c, err
|
||||
}
|
||||
|
||||
func makeGofishConfig(q *QueryParams) (gofish.ClientConfig, error) {
|
||||
var (
|
||||
client = &http.Client{
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
url = baseRedfishUrl(q)
|
||||
)
|
||||
return gofish.ClientConfig{
|
||||
Endpoint: url,
|
||||
Username: q.Username,
|
||||
Password: q.Password,
|
||||
Insecure: true,
|
||||
TLSHandshakeTimeout: q.Timeout,
|
||||
HTTPClient: client,
|
||||
// MaxConcurrentRequests: int64(q.Threads), // NOTE: this was added in latest version of gofish
|
||||
}, nil
|
||||
}
|
||||
|
||||
func makeRequest[T any](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) {
|
||||
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout))
|
||||
client.Registry.FilterForCompatible(ctx)
|
||||
err := client.Open(ctx)
|
||||
if err != nil {
|
||||
ctxCancel()
|
||||
return nil, fmt.Errorf("failed to open client: %v", err)
|
||||
}
|
||||
|
||||
defer client.Close(ctx)
|
||||
|
||||
response, err := fn(ctx)
|
||||
if err != nil {
|
||||
ctxCancel()
|
||||
return nil, fmt.Errorf("failed to get response: %v", err)
|
||||
}
|
||||
|
||||
ctxCancel()
|
||||
return makeJson(response)
|
||||
}
|
||||
|
||||
func makeJson(object any) ([]byte, error) {
|
||||
b, err := json.MarshalIndent(object, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal JSON: %v", err)
|
||||
}
|
||||
return []byte(b), nil
|
||||
}
|
||||
|
||||
func baseRedfishUrl(q *QueryParams) string {
|
||||
url := fmt.Sprintf("%s://", q.Protocol)
|
||||
if q.Username != "" && q.Password != "" {
|
||||
url += fmt.Sprintf("%s:%s@", q.Username, q.Password)
|
||||
}
|
||||
return fmt.Sprintf("%s%s:%d", url, q.Host, q.Port)
|
||||
return fmt.Sprintf("%s:%d", q.Host, q.Port)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue