diff --git a/bin/magellan.sh b/bin/magellan.sh index a422071..99f1ce6 100644 --- a/bin/magellan.sh +++ b/bin/magellan.sh @@ -4,7 +4,7 @@ function build(){ } function scan() { - ./magellan scan --subnet 172.16.0.0 --db.path data/assets.db --port 623 + ./magellan scan --subnet 172.16.0.0 --db.path data/assets.db --port 443,623,5000 } function list(){ @@ -12,6 +12,6 @@ function list(){ } function collect() { - ./magellan collect --db.path data/assets.db --driver ipmi --timeout 5 --user admin --pass password + ./magellan collect --db.path data/assets.db --driver redfish --timeout 30 --user admin --pass password } diff --git a/cmd/collect.go b/cmd/collect.go index 58f5b99..2a7d0d4 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -1,12 +1,11 @@ package cmd import ( - "davidallendj/magellan/api/smd" magellan "davidallendj/magellan/internal" + "davidallendj/magellan/internal/api/smd" "davidallendj/magellan/internal/db/sqlite" - "fmt" - "github.com/Cray-HPE/hms-xname/xnames" + "github.com/cznic/mathutil" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -26,93 +25,127 @@ var collectCmd = &cobra.Command{ l.Log.Errorf("could not get states: %v", err) } - // generate custom xnames for bmcs - node := xnames.Node{ - Cabinet: 1000, - Chassis: 1, - ComputeModule: 7, - NodeBMC: 1, - Node: 0, + q := &magellan.QueryParams{ + User: user, + Pass: pass, + Drivers: drivers, + Timeout: timeout, + Threads: threads, + Verbose: verbose, + WithSecureTLS: withSecureTLS, } - // use the found results to query bmc information - users := [][]byte{} - for _, ps := range probeStates { - if !ps.State { - continue - } - logrus.Infof("querying %v\n", ps) - q := magellan.QueryParams{ - Host: ps.Host, - Port: ps.Port, - User: user, - Pass: pass, - Drivers: drivers, - Timeout: timeout, - Verbose: true, - WithSecureTLS: withSecureTLS, - } + // scan and store probe data in dbPath + if threads <= 0 { + threads = mathutil.Clamp(len(probeStates), 1, 255) + } + magellan.CollectInfo(&probeStates, l, q) - client, err := magellan.NewClient(l, &q) - if err != nil { - l.Log.Errorf("could not make client: %v", err) - return - } + // generate custom xnames for bmcs + // node := xnames.Node{ + // Cabinet: 1000, + // Chassis: 1, + // ComputeModule: 7, + // NodeBMC: 1, + // Node: 0, + // } - // metadata - _, err = magellan.QueryMetadata(client, l, &q) - if err != nil { - l.Log.Errorf("could not query metadata: %v\n", err) - } + // // use the found results to query bmc information + // // users := [][]byte{} + // probedHosts := []string{} + // for _, ps := range probeStates { - // inventories - inventory, err := magellan.QueryInventory(client, l, &q) - // inventory, err := magellan.QueryInventoryV2(q.Host, q.Port, q.User, q.Pass) - if err != nil { - l.Log.Errorf("could not query inventory: %v\n", err) - } + // // skip if found info from host + // foundHost := slices.Index(probedHosts, ps.Host) + // if !ps.State || foundHost >= 0{ + // continue + // } - node.NodeBMC += 1 + // logrus.Printf("querying %v:%v (%v)\n", ps.Host, ps.Port, ps.Protocol) + - data := make(map[string]any) - data["ID"] = fmt.Sprintf("%v", node) - data["FQDN"] = ps.Host - data["RediscoverOnUpdate"] = false + // client, err := magellan.NewClient(l, q) + // if err != nil { + // l.Log.Errorf("could not make client: %v", err) + // return + // } - headers := make(map[string]string) - headers["Content-Type"] = "application/json" + // // metadata + // // _, err = magellan.QueryMetadata(client, l, &q) + // // if err != nil { + // // l.Log.Errorf("could not query metadata: %v\n", err) + // // } - // add all endpoints to smd - err = smd.AddRedfishEndpoint(inventory, headers) - if err != nil { - logrus.Errorf("could not add redfish endpoint: %v", err) - } + // // inventories + // inventory, err := magellan.QueryInventory(client, l, q) + // if err != nil { + // l.Log.Errorf("could not query inventory: %v\n", err) + // continue + // } - // confirm the inventories were added - err = smd.GetRedfishEndpoints() - if err != nil { - logrus.Errorf("could not get redfish endpoints: %v\n", err) - } + // // chassis + // _, err = magellan.QueryChassis(client, l, q) + // if err != nil { + // l.Log.Errorf("could not query chassis: %v\n", err) + // continue + // } + + // // got host information, so add to list of already probed hosts + // probedHosts = append(probedHosts, ps.Host) + + // node.NodeBMC += 1 + + // headers := make(map[string]string) + // headers["Content-Type"] = "application/json" + + // data := make(map[string]any) + // data["ID"] = fmt.Sprintf("%v", node) + // data["Type"] = "" + // data["Name"] = "" + // data["FQDN"] = ps.Host + // data["RediscoverOnUpdate"] = false + // data["Inventory"] = inventory + + + // b, err := json.MarshalIndent(data, "", " ") + // if err != nil { + // l.Log.Errorf("could not marshal JSON: %v\n", err) + // continue + // } + + // // add all endpoints to smd + // err = smd.AddRedfishEndpoint(b, headers) + // if err != nil { + // logrus.Errorf("could not add redfish endpoint: %v", err) + // continue + // } + + // // confirm the inventories were added + // err = smd.GetRedfishEndpoints() + // if err != nil { + // logrus.Errorf("could not get redfish endpoints: %v\n", err) + // continue + // } // users - user, err := magellan.QueryUsers(client, l, &q) - if err != nil { - l.Log.Errorf("could not query users: %v\n", err) - } - users = append(users, user) + // user, err := magellan.QueryUsers(client, l, &q) + // if err != nil { + // l.Log.Errorf("could not query users: %v\n", err) + // } + // users = append(users, user) - // bios - _, err = magellan.QueryBios(client, l, &q) - if err != nil { - l.Log.Errorf("could not query bios: %v\n", err) - } + // // bios + // _, err = magellan.QueryBios(client, l, &q) + // if err != nil { + // l.Log.Errorf("could not query bios: %v\n", err) + // } - _, err = magellan.QueryPowerState(client, l, &q) - if err != nil { - l.Log.Errorf("could not query power state: %v\n", err) - } + // _, err = magellan.QueryPowerState(client, l, &q) + // if err != nil { + // l.Log.Errorf("could not query power state: %v\n", err) + // } - } + // } }, } diff --git a/cmd/root.go b/cmd/root.go index b4d20e8..ea661a7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,6 +20,7 @@ var ( drivers []string preferredDriver string ipmitoolPath string + verbose bool ) // TODO: discover bmc's on network (dora) @@ -49,7 +50,6 @@ func Execute() { func init() { rootCmd.PersistentFlags().IntVar(&threads, "threads", -1, "set the number of threads") rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 10, "set the timeout") - + rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", true, "set verbose flag") rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", "/tmp/magellan.db", "set the probe storage path") - } diff --git a/cmd/scan.go b/cmd/scan.go index e86835d..a5b1a37 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -42,7 +42,9 @@ var scanCmd = &cobra.Command{ threads = mathutil.Clamp(len(hostsToScan), 1, 255) } probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, threads, timeout) - fmt.Printf("probe states: %v\n", probeStates) + for _, r := range probeStates { + fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol) + } sqlite.InsertProbeResults(dbpath, &probeStates) }, } diff --git a/api/dora/dora.go b/internal/api/dora/dora.go similarity index 100% rename from api/dora/dora.go rename to internal/api/dora/dora.go diff --git a/api/smd/smd.go b/internal/api/smd/smd.go similarity index 97% rename from api/smd/smd.go rename to internal/api/smd/smd.go index 4d612af..492375d 100644 --- a/api/smd/smd.go +++ b/internal/api/smd/smd.go @@ -4,7 +4,7 @@ package smd // https://github.com/Cray-HPE/hms-smd/blob/master/docs/examples.adoc // https://github.com/alexlovelltroy/hms-smd import ( - "davidallendj/magellan/api" + "davidallendj/magellan/internal/api" "fmt" // hms "github.com/alexlovelltroy/hms-smd" ) diff --git a/api/util.go b/internal/api/util.go similarity index 100% rename from api/util.go rename to internal/api/util.go diff --git a/internal/magellan.go b/internal/collect.go similarity index 59% rename from internal/magellan.go rename to internal/collect.go index f19f691..7cb663f 100644 --- a/internal/magellan.go +++ b/internal/collect.go @@ -3,21 +3,28 @@ package magellan import ( "context" "crypto/x509" + "davidallendj/magellan/internal/api/smd" "encoding/json" "fmt" "os" + "sync" "time" + "github.com/Cray-HPE/hms-xname/xnames" bmclib "github.com/bmc-toolbox/bmclib/v2" "github.com/jacobweinstock/registrar" _ "github.com/mattn/go-sqlite3" + "github.com/sirupsen/logrus" + "github.com/stmcginnis/gofish" _ "github.com/stmcginnis/gofish" + "github.com/stmcginnis/gofish/redfish" + "golang.org/x/exp/slices" ) const ( IPMI_PORT = 623 SSH_PORT = 22 - TLS_PORT = 443 + HTTPS_PORT = 443 REDFISH_PORT = 5000 ) @@ -35,6 +42,7 @@ type QueryParams struct { User string Pass string Drivers []string + Threads int Preferred string Timeout int WithSecureTLS bool @@ -56,10 +64,11 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) { // init client clientOpts := []bmclib.Option{ - // bmclib.WithSecureTLS(), + // bmclib.WithSecureTLS(nil), // bmclib.WithHTTPClient(&httpClient), // bmclib.WithLogger(), // bmclib.WithRedfishHTTPClient(&httpClient), + bmclib.WithDellRedfishUseBasicAuth(true), bmclib.WithRedfishPort(fmt.Sprint(q.Port)), bmclib.WithRedfishUseBasicAuth(true), bmclib.WithIpmitoolPort(fmt.Sprint(IPMI_PORT)), @@ -93,7 +102,6 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) { } else { url += q.Host } - client := bmclib.NewClient(url, q.User, q.Pass, clientOpts...) ds := registrar.Drivers{} for _, driver := range q.Drivers { @@ -104,6 +112,147 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) { return client, nil } +func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error { + if probeStates == nil { + return fmt.Errorf("no probe states found") + } + if len(*probeStates) <= 0 { + return fmt.Errorf("no probe states found") + } + + // generate custom xnames for bmcs + node := xnames.Node{ + Cabinet: 1000, + Chassis: 1, + ComputeModule: 7, + NodeBMC: 1, + Node: 0, + } + + found := make([]string, 0, len(*probeStates)) + done := make(chan struct{}, q.Threads+1) + chanProbeState := make(chan BMCProbeResult, q.Threads+1) + + // + var wg sync.WaitGroup + wg.Add(q.Threads) + for i := 0; i < q.Threads; i++ { + go func() { + for { + ps, ok := <- chanProbeState + if !ok { + wg.Done() + return + } + q.Host = ps.Host + q.Port = ps.Port + + logrus.Printf("querying %v:%v (%v)\n", ps.Host, ps.Port, ps.Protocol) + + client, err := NewClient(l, q) + if err != nil { + l.Log.Errorf("could not make client: %v", err) + continue + } + + // metadata + // _, err = magellan.QueryMetadata(client, l, &q) + // if err != nil { + // l.Log.Errorf("could not query metadata: %v\n", err) + // } + + // inventories + inventory, err := QueryInventory(client, l, q) + if err != nil { + l.Log.Errorf("could not query inventory: %v", err) + } + + // chassis + _, err = QueryChassis(client, l, q) + if err != nil { + l.Log.Errorf("could not query chassis: %v", err) + } + + node.NodeBMC += 1 + + headers := make(map[string]string) + headers["Content-Type"] = "application/json" + + data := make(map[string]any) + data["ID"] = fmt.Sprintf("%v", node) + data["Type"] = "" + data["Name"] = "" + data["FQDN"] = ps.Host + data["RediscoverOnUpdate"] = false + data["Inventory"] = inventory + + b, err := json.MarshalIndent(data, "", " ") + if err != nil { + l.Log.Errorf("could not marshal JSON: %v", err) + } + + // add all endpoints to smd + err = smd.AddRedfishEndpoint(b, headers) + if err != nil { + l.Log.Errorf("could not add redfish endpoint: %v", err) + } + + // confirm the inventories were added + err = smd.GetRedfishEndpoints() + if err != nil { + l.Log.Errorf("could not get redfish endpoints: %v", err) + } + + // users + // user, err := magellan.QueryUsers(client, l, &q) + // if err != nil { + // l.Log.Errorf("could not query users: %v\n", err) + // } + // users = append(users, user) + + // bios + // _, err = magellan.QueryBios(client, l, &q) + // if err != nil { + // l.Log.Errorf("could not query bios: %v\n", err) + // } + + // _, err = magellan.QueryPowerState(client, l, &q) + // if err != nil { + // l.Log.Errorf("could not query power state: %v\n", err) + // } + + // got host information, so add to list of already probed hosts + found = append(found, ps.Host) + } + }() + } + + // use the found results to query bmc information + for _, ps := range *probeStates { + // skip if found info from host + foundHost := slices.Index(found, ps.Host) + if !ps.State || foundHost >= 0{ + continue + } + chanProbeState <- ps + } + + go func() { + select { + case <-done: + wg.Done() + break + default: + time.Sleep(1000) + } + }() + + close(chanProbeState) + wg.Wait() + close(done) + return nil +} + func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { // client, err := NewClient(l, q) @@ -113,7 +262,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er err := client.Open(ctx) if err != nil { ctxCancel() - return nil, fmt.Errorf("could not open BMC client: %v", err) + return nil, fmt.Errorf("could not connect to bmc: %v", err) } defer client.Close(ctx) @@ -135,7 +284,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er fmt.Printf("metadata: %v\n", string(b)) } ctxCancel() - return []byte(b), nil + return b, nil } func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { @@ -168,7 +317,7 @@ func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, e fmt.Printf("inventory: %v\n", string(b)) } ctxCancel() - return []byte(b), nil + return b, nil } func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { @@ -198,7 +347,7 @@ func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, fmt.Printf("power state: %v\n", string(b)) } ctxCancel() - return []byte(b), nil + return b, nil } @@ -215,7 +364,7 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error err := client.Open(ctx) if err != nil { ctxCancel() - return nil, fmt.Errorf("could not open BMC client: %v", err) + return nil, fmt.Errorf("could not connect to bmc: %v", err) } defer client.Close(ctx) @@ -238,7 +387,7 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error if q.Verbose { fmt.Printf("users: %v\n", string(b)) } - return []byte(b), nil + return b, nil } func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { @@ -253,6 +402,46 @@ func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) return b, err } +func QueryEthernetInterfaces(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { + config := gofish.ClientConfig{ + + } + c, err := gofish.Connect(config) + if err != nil { + + } + + redfish.ListReferencedEthernetInterfaces(c, "") + return []byte{}, nil +} + +func QueryChassis(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { + config := gofish.ClientConfig { + Endpoint: "https://" + q.Host, + Username: q.User, + Password: q.Pass, + Insecure: q.WithSecureTLS, + } + c, err := gofish.Connect(config) + if err != nil { + return nil, fmt.Errorf("could not connect to bmc: %v", err) + } + chassis, err := c.Service.Chassis() + if err != nil { + return nil, fmt.Errorf("could not query chassis: %v", err) + } + + b, err := json.MarshalIndent(chassis, "", " ") + if err != nil { + return nil, fmt.Errorf("could not marshal JSON: %v", err) + } + + if q.Verbose { + fmt.Printf("chassis: %v\n", string(b)) + } + return b, nil +} + func makeRequest[T interface{}](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout)) client.Registry.FilterForCompatible(ctx) diff --git a/internal/db/sqlite/sqlite.go b/internal/db/sqlite/sqlite.go index a5ec3fb..d7c3aeb 100644 --- a/internal/db/sqlite/sqlite.go +++ b/internal/db/sqlite/sqlite.go @@ -16,10 +16,11 @@ func InsertProbeResults(path string, states *[]magellan.BMCProbeResult) error { // create database if it doesn't already exist schema := ` CREATE TABLE IF NOT EXISTS magellan_scanned_ports ( - host TEXT PRIMARY KEY NOT NULL, - port INTEGER, + host TEXT NOT NULL, + port INTEGER NOT NULL, protocol TEXT, - state INTEGER + state INTEGER, + PRIMARY KEY (host, port) ); ` db, err := sqlx.Open("sqlite3", path) @@ -52,7 +53,7 @@ func GetProbeResults(path string) ([]magellan.BMCProbeResult, error) { } results := []magellan.BMCProbeResult{} - err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC") + err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC, port ASC;") if err != nil { return nil, fmt.Errorf("could not retrieve probes: %v", err) } diff --git a/internal/scan.go b/internal/scan.go index d2e66ca..c373c40 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -52,10 +52,11 @@ func GenerateHosts(subnet string, begin uint8, end uint8) []string { } func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCProbeResult { - states := make([]BMCProbeResult, 0, len(hosts)) - done := make(chan struct{}, threads+1) - chanHost := make(chan string, threads+1) + results := make([]BMCProbeResult, 0, len(hosts)) + done := make(chan struct{}, threads+1) + chanHost := make(chan string, threads+1) // chanPort := make(chan int, threads+1) + var wg sync.WaitGroup wg.Add(threads) for i := 0; i < threads; i++ { @@ -67,7 +68,7 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCP return } s := rawConnect(host, ports, timeout, true) - states = append(states, s...) + results = append(results, s...) } }() } @@ -87,9 +88,9 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCP close(chanHost) wg.Wait() close(done) - return states + return results } func GetDefaultPorts() []int { - return []int{SSH_PORT, TLS_PORT, IPMI_PORT, REDFISH_PORT} + return []int{SSH_PORT, HTTPS_PORT, IPMI_PORT, REDFISH_PORT} } \ No newline at end of file