Add goroutine to info collect

Signed-off-by: David J. Allen <allend@lanl.gov>

bin/magellan.sh

@@ -4,7 +4,7 @@ function build(){
 }
 
 function scan() {
-    ./magellan scan --subnet 172.16.0.0 --db.path data/assets.db --port 623
+    ./magellan scan --subnet 172.16.0.0 --db.path data/assets.db --port 443,623,5000
 }
 
 function list(){
@@ -12,6 +12,6 @@ function list(){
 }
 
 function collect() {
-    ./magellan collect --db.path data/assets.db --driver ipmi --timeout 5 --user admin --pass password
+    ./magellan collect --db.path data/assets.db --driver redfish --timeout 30 --user admin --pass password
 }
 

cmd/collect.go

@@ -1,12 +1,11 @@
 package cmd
 
 import (
-	"davidallendj/magellan/api/smd"
 	magellan "davidallendj/magellan/internal"
+	"davidallendj/magellan/internal/api/smd"
 	"davidallendj/magellan/internal/db/sqlite"
-	"fmt"
 
-	"github.com/Cray-HPE/hms-xname/xnames"
+	"github.com/cznic/mathutil"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -26,93 +25,127 @@ var collectCmd = &cobra.Command{
 		l.Log.Errorf("could not get states: %v", err)
 	}
 
-	// generate custom xnames for bmcs
+	q := &magellan.QueryParams{
-	node := xnames.Node{
+		User: 		user,
-		Cabinet:		1000,
+		Pass: 		pass,
-		Chassis:		1,
+		Drivers: 	drivers,
-		ComputeModule:	7,
+		Timeout: 	timeout,
-		NodeBMC:		1,
+		Threads:	threads,
-		Node:			0,
+		Verbose: 	verbose,
+		WithSecureTLS: withSecureTLS,
 	}
 
-	// use the found results to query bmc information
+	// scan and store probe data in dbPath
-	users := [][]byte{}
+	if threads <= 0 {
-	for _, ps := range probeStates {
+		threads = mathutil.Clamp(len(probeStates), 1, 255)
-		if !ps.State {
+	}
-			continue
+	magellan.CollectInfo(&probeStates, l, q)
-		}
-		logrus.Infof("querying %v\n", ps)
-		q := magellan.QueryParams{
-			Host: ps.Host,
-			Port: ps.Port,
-			User: user,
-			Pass: pass,
-			Drivers: drivers,
-			Timeout: timeout,
-			Verbose: true,
-			WithSecureTLS: withSecureTLS,
-		}
 
-		client, err := magellan.NewClient(l, &q)
+	// generate custom xnames for bmcs
-		if err != nil {
+	// node := xnames.Node{
-			l.Log.Errorf("could not make client: %v", err)
+	// 	Cabinet:		1000,
-			return 
+	// 	Chassis:		1,
-		}
+	// 	ComputeModule:	7,
+	// 	NodeBMC:		1,
+	// 	Node:			0,
+	// }
 
-		// metadata
+	// // use the found results to query bmc information
-		_, err = magellan.QueryMetadata(client, l, &q)
+	// // users := [][]byte{}
-		if err != nil {
+	// probedHosts := []string{}
-			l.Log.Errorf("could not query metadata: %v\n", err)
+	// for _, ps := range probeStates {
-		}
 
-		// inventories
+	// 	// skip if found info from host
-		inventory, err := magellan.QueryInventory(client, l, &q)
+	// 	foundHost := slices.Index(probedHosts, ps.Host)
-		// inventory, err := magellan.QueryInventoryV2(q.Host, q.Port, q.User, q.Pass)
+	// 	if !ps.State || foundHost >= 0{
-		if err != nil {
+	// 		continue
-			l.Log.Errorf("could not query inventory: %v\n", err)
+	// 	}
-		}
 
-		node.NodeBMC += 1
+	// 	logrus.Printf("querying %v:%v (%v)\n", ps.Host, ps.Port, ps.Protocol)
 		
-		data := make(map[string]any)
-		data["ID"] 					= fmt.Sprintf("%v", node)
-		data["FQDN"]				= ps.Host
-		data["RediscoverOnUpdate"] 	= false
 
-		headers := make(map[string]string)
+	// 	client, err := magellan.NewClient(l, q)
-		headers["Content-Type"] = "application/json"
+	// 	if err != nil {
+	// 		l.Log.Errorf("could not make client: %v", err)
+	// 		return
+	// 	}
 
-		// add all endpoints to smd
+	// 	// metadata
-		err = smd.AddRedfishEndpoint(inventory, headers)
+	// 	// _, err = magellan.QueryMetadata(client, l, &q)
-		if err != nil {
+	// 	// if err != nil {
-			logrus.Errorf("could not add redfish endpoint: %v", err)
+	// 	// 	l.Log.Errorf("could not query metadata: %v\n", err)
-		}
+	// 	// }
 
-		// confirm the inventories were added
+	// 	// inventories
-		err = smd.GetRedfishEndpoints()
+	// 	inventory, err := magellan.QueryInventory(client, l, q)
-		if err != nil {
+	// 	if err != nil {
-			logrus.Errorf("could not get redfish endpoints: %v\n", err)
+	// 		l.Log.Errorf("could not query inventory: %v\n", err)
-		}
+	// 		continue
+	// 	}
+
+	// 	// chassis
+	// 	_, err = magellan.QueryChassis(client, l, q)
+	// 	if err != nil {
+	// 		l.Log.Errorf("could not query chassis: %v\n", err)
+	// 		continue
+	// 	}
+
+	// 	// got host information, so add to list of already probed hosts
+	// 	probedHosts = append(probedHosts, ps.Host)
+
+	// 	node.NodeBMC += 1
+
+	// 	headers := make(map[string]string)
+	// 	headers["Content-Type"] = "application/json"
+
+	// 	data := make(map[string]any)
+	// 	data["ID"] 					= fmt.Sprintf("%v", node)
+	// 	data["Type"]				= ""
+	// 	data["Name"]				= ""
+	// 	data["FQDN"]				= ps.Host
+	// 	data["RediscoverOnUpdate"] 	= false
+	// 	data["Inventory"] 			= inventory
+
+
+	// 	b, err := json.MarshalIndent(data, "", "    ")
+	// 	if err != nil {
+	// 		l.Log.Errorf("could not marshal JSON: %v\n", err)
+	// 		continue
+	// 	}
+
+	// 	// add all endpoints to smd
+	// 	err = smd.AddRedfishEndpoint(b, headers)
+	// 	if err != nil {
+	// 		logrus.Errorf("could not add redfish endpoint: %v", err)
+	// 		continue
+	// 	}
+
+	// 	// confirm the inventories were added
+	// 	err = smd.GetRedfishEndpoints()
+	// 	if err != nil {
+	// 		logrus.Errorf("could not get redfish endpoints: %v\n", err)
+	// 		continue
+	// 	}
 
 		// users
-		user, err := magellan.QueryUsers(client, l, &q)
+		// user, err := magellan.QueryUsers(client, l, &q)
-		if err != nil {
+		// if err != nil {
-			l.Log.Errorf("could not query users: %v\n", err)
+		// 	l.Log.Errorf("could not query users: %v\n", err)
-		}
+		// }
-		users = append(users, user)
+		// users = append(users, user)
 
-		// bios
+		// // bios
-		_, err = magellan.QueryBios(client, l, &q)
+		// _, err = magellan.QueryBios(client, l, &q)
-		if err != nil {
+		// if err != nil {
-			l.Log.Errorf("could not query bios: %v\n", err)
+		// 	l.Log.Errorf("could not query bios: %v\n", err)
-		}
+		// }
 
-		_, err = magellan.QueryPowerState(client, l, &q)
+		// _, err = magellan.QueryPowerState(client, l, &q)
-		if err != nil {
+		// if err != nil {
-			l.Log.Errorf("could not query power state: %v\n", err)
+		// 	l.Log.Errorf("could not query power state: %v\n", err)
-		}
+		// }
 		
-	}
+	// }
 	
 	},
 }

cmd/root.go

@@ -20,6 +20,7 @@ var (
 	drivers       []string
 	preferredDriver string
 	ipmitoolPath  string
+	verbose			bool
 )
 
 // TODO: discover bmc's on network (dora)
@@ -49,7 +50,6 @@ func Execute() {
 func init() {
 	rootCmd.PersistentFlags().IntVar(&threads, "threads", -1, "set the number of threads")
 	rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 10, "set the timeout")
-	
+	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", true, "set verbose flag")
 	rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", "/tmp/magellan.db", "set the probe storage path")
-	
 }

cmd/scan.go

@@ -42,7 +42,9 @@ var scanCmd = &cobra.Command{
 			threads = mathutil.Clamp(len(hostsToScan), 1, 255)
 		}
 		probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, threads, timeout)
-		fmt.Printf("probe states: %v\n", probeStates)
+		for _, r := range probeStates {
+			fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol)
+		}
 		sqlite.InsertProbeResults(dbpath, &probeStates)
 	},
 }

internal/api/smd/smd.go

@@ -4,7 +4,7 @@ package smd
 //	https://github.com/Cray-HPE/hms-smd/blob/master/docs/examples.adoc
 //	https://github.com/alexlovelltroy/hms-smd
 import (
-	"davidallendj/magellan/api"
+	"davidallendj/magellan/internal/api"
 	"fmt"
 	// hms "github.com/alexlovelltroy/hms-smd"
 )

internal/collect.go

@@ -3,21 +3,28 @@ package magellan
 import (
 	"context"
 	"crypto/x509"
+	"davidallendj/magellan/internal/api/smd"
 	"encoding/json"
 	"fmt"
 	"os"
+	"sync"
 	"time"
 
+	"github.com/Cray-HPE/hms-xname/xnames"
 	bmclib "github.com/bmc-toolbox/bmclib/v2"
 	"github.com/jacobweinstock/registrar"
 	_ "github.com/mattn/go-sqlite3"
+	"github.com/sirupsen/logrus"
+	"github.com/stmcginnis/gofish"
 	_ "github.com/stmcginnis/gofish"
+	"github.com/stmcginnis/gofish/redfish"
+	"golang.org/x/exp/slices"
 )
 
 const (
 	IPMI_PORT    = 623
 	SSH_PORT     = 22
-	TLS_PORT     = 443
+	HTTPS_PORT   = 443
 	REDFISH_PORT = 5000
 )
 
@@ -35,6 +42,7 @@ type QueryParams struct {
 	User          string
 	Pass          string
 	Drivers       []string
+	Threads			int
 	Preferred		string
 	Timeout       int
 	WithSecureTLS bool
@@ -56,10 +64,11 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
 
 	// init client
 	clientOpts := []bmclib.Option{
-		// bmclib.WithSecureTLS(),
+		// bmclib.WithSecureTLS(nil),
 		// bmclib.WithHTTPClient(&httpClient),
 		// bmclib.WithLogger(),
 		// bmclib.WithRedfishHTTPClient(&httpClient),
+		bmclib.WithDellRedfishUseBasicAuth(true),
 		bmclib.WithRedfishPort(fmt.Sprint(q.Port)),
 		bmclib.WithRedfishUseBasicAuth(true),
 		bmclib.WithIpmitoolPort(fmt.Sprint(IPMI_PORT)),
@@ -93,7 +102,6 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
 	} else {
 		url += q.Host
 	}
-
 	client := bmclib.NewClient(url, q.User, q.Pass, clientOpts...)
 	ds := registrar.Drivers{}
 	for _, driver := range q.Drivers {
@@ -104,6 +112,147 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
 	return client, nil
 }
 
+func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error {
+	if probeStates == nil {
+		return fmt.Errorf("no probe states found")
+	}
+	if len(*probeStates) <= 0 {
+		return fmt.Errorf("no probe states found")
+	}
+	
+	// generate custom xnames for bmcs
+	node := xnames.Node{
+		Cabinet:		1000,
+		Chassis:		1,
+		ComputeModule:	7,
+		NodeBMC:		1,
+		Node:			0,
+	}
+
+	found 			:= make([]string, 0, len(*probeStates))
+	done 			:= make(chan struct{}, q.Threads+1)
+	chanProbeState 	:= make(chan BMCProbeResult, q.Threads+1)
+
+	//
+	var wg sync.WaitGroup
+	wg.Add(q.Threads)
+	for i := 0; i < q.Threads; i++ {
+		go func() {
+			for {
+				ps, ok := <- chanProbeState
+				if !ok {
+					wg.Done()
+					return
+				}
+				q.Host = ps.Host
+				q.Port = ps.Port
+
+				logrus.Printf("querying %v:%v (%v)\n", ps.Host, ps.Port, ps.Protocol)
+
+				client, err := NewClient(l, q)
+				if err != nil {
+					l.Log.Errorf("could not make client: %v", err)
+					continue 
+				}
+
+				// metadata
+				// _, err = magellan.QueryMetadata(client, l, &q)
+				// if err != nil {
+				// 	l.Log.Errorf("could not query metadata: %v\n", err)
+				// }
+
+				// inventories
+				inventory, err := QueryInventory(client, l, q)
+				if err != nil {
+					l.Log.Errorf("could not query inventory: %v", err) 
+				}
+
+				// chassis
+				_, err = QueryChassis(client, l, q)
+				if err != nil {
+					l.Log.Errorf("could not query chassis: %v", err)
+				}
+
+				node.NodeBMC += 1
+
+				headers := make(map[string]string)
+				headers["Content-Type"] = "application/json"
+
+				data := make(map[string]any)
+				data["ID"] 					= fmt.Sprintf("%v", node)
+				data["Type"]				= ""
+				data["Name"]				= ""
+				data["FQDN"]				= ps.Host
+				data["RediscoverOnUpdate"] 	= false
+				data["Inventory"] 			= inventory
+
+				b, err := json.MarshalIndent(data, "", "    ")
+				if err != nil {
+					l.Log.Errorf("could not marshal JSON: %v", err)
+				}
+
+				// add all endpoints to smd
+				err = smd.AddRedfishEndpoint(b, headers)
+				if err != nil {
+					l.Log.Errorf("could not add redfish endpoint: %v", err)
+				}
+
+				// confirm the inventories were added
+				err = smd.GetRedfishEndpoints()
+				if err != nil {
+					l.Log.Errorf("could not get redfish endpoints: %v", err)
+				}
+
+				// users
+				// user, err := magellan.QueryUsers(client, l, &q)
+				// if err != nil {
+				// 	l.Log.Errorf("could not query users: %v\n", err)
+				// }
+				// users = append(users, user)
+
+				// bios
+				// _, err = magellan.QueryBios(client, l, &q)
+				// if err != nil {
+				// 	l.Log.Errorf("could not query bios: %v\n", err)
+				// }
+
+				// _, err = magellan.QueryPowerState(client, l, &q)
+				// if err != nil {
+				// 	l.Log.Errorf("could not query power state: %v\n", err)
+				// }
+
+				// got host information, so add to list of already probed hosts
+				found = append(found, ps.Host)
+			}
+		}()
+	}
+
+	// use the found results to query bmc information
+	for _, ps := range *probeStates {
+		// skip if found info from host
+		foundHost := slices.Index(found, ps.Host)
+		if !ps.State || foundHost >= 0{
+			continue
+		}
+		chanProbeState <- ps
+	}
+
+	go func() {
+		select {
+		case <-done:
+			wg.Done()
+			break
+		default:
+			time.Sleep(1000)
+		}
+	}()
+
+	close(chanProbeState)
+	wg.Wait()
+	close(done)
+	return nil
+}
+
 func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
 	// client, err := NewClient(l, q)
 
@@ -113,7 +262,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er
 	err := client.Open(ctx)
 	if err != nil {
 		ctxCancel()
-		return nil, fmt.Errorf("could not open BMC client: %v", err)
+		return nil, fmt.Errorf("could not connect to bmc: %v", err)
 	}
 
 	defer client.Close(ctx)
@@ -135,7 +284,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er
 		fmt.Printf("metadata: %v\n", string(b))
 	}
 	ctxCancel()
-	return []byte(b), nil
+	return b, nil
 }
 
 func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
@@ -168,7 +317,7 @@ func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, e
 		fmt.Printf("inventory: %v\n", string(b))
 	}
 	ctxCancel()
-	return []byte(b), nil
+	return b, nil
 }
 
 func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
@@ -198,7 +347,7 @@ func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte,
 		fmt.Printf("power state: %v\n", string(b))
 	}
 	ctxCancel()
-	return []byte(b), nil
+	return b, nil
 
 }
 
@@ -215,7 +364,7 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
 	err := client.Open(ctx)
 	if err != nil {
 		ctxCancel()
-		return nil, fmt.Errorf("could not open BMC client: %v", err)
+		return nil, fmt.Errorf("could not connect to bmc: %v", err)
 	}
 
 	defer client.Close(ctx)
@@ -238,7 +387,7 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
 	if q.Verbose {
 		fmt.Printf("users: %v\n", string(b))
 	}
-	return []byte(b), nil
+	return b, nil
 }
 
 func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
@@ -253,6 +402,46 @@ func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error)
 	return b, err
 }
 
+func QueryEthernetInterfaces(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
+	config := gofish.ClientConfig{
+
+	}
+	c, err := gofish.Connect(config)
+	if err != nil {
+
+	}
+
+	redfish.ListReferencedEthernetInterfaces(c, "")
+	return []byte{}, nil
+}
+
+func QueryChassis(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
+	config := gofish.ClientConfig {
+		Endpoint: "https://" + q.Host,
+		Username: q.User,
+		Password: q.Pass,
+		Insecure: q.WithSecureTLS,
+	}
+	c, err := gofish.Connect(config)
+	if err != nil {
+		return nil, fmt.Errorf("could not connect to bmc: %v", err)
+	}
+	chassis, err := c.Service.Chassis()
+	if err != nil {
+		return nil, fmt.Errorf("could not query chassis: %v", err)
+	}
+
+	b, err := json.MarshalIndent(chassis, "", "    ")
+	if err != nil {
+		return nil, fmt.Errorf("could not marshal JSON: %v", err)
+	}
+
+	if q.Verbose {
+		fmt.Printf("chassis: %v\n", string(b))
+	}
+	return b, nil
+}
+
 func makeRequest[T interface{}](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) {
 	ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout))
 	client.Registry.FilterForCompatible(ctx)

internal/db/sqlite/sqlite.go

@@ -16,10 +16,11 @@ func InsertProbeResults(path string, states *[]magellan.BMCProbeResult) error {
 	// create database if it doesn't already exist
 	schema := `
 	CREATE TABLE IF NOT EXISTS magellan_scanned_ports (
-		host TEXT PRIMARY KEY NOT NULL,
+		host TEXT NOT NULL,
-		port INTEGER,
+		port INTEGER NOT NULL,
 		protocol TEXT,
-		state INTEGER
+		state INTEGER,
+		PRIMARY KEY (host, port)
 	);
 	`
 	db, err := sqlx.Open("sqlite3", path)
@@ -52,7 +53,7 @@ func GetProbeResults(path string) ([]magellan.BMCProbeResult, error) {
 	}
 
 	results := []magellan.BMCProbeResult{}
-	err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC")
+	err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC, port ASC;")
 	if err != nil {
 		return nil, fmt.Errorf("could not retrieve probes: %v", err)
 	}

internal/scan.go

@@ -52,10 +52,11 @@ func GenerateHosts(subnet string, begin uint8, end uint8) []string {
 }
 
 func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCProbeResult {
-	states := make([]BMCProbeResult, 0, len(hosts))
+	results 	:= make([]BMCProbeResult, 0, len(hosts))
-	done := make(chan struct{}, threads+1)
+	done 		:= make(chan struct{}, threads+1)
-	chanHost := make(chan string, threads+1)
+	chanHost 	:= make(chan string, threads+1)
 	// chanPort := make(chan int, threads+1)
+	
 	var wg sync.WaitGroup
 	wg.Add(threads)
 	for i := 0; i < threads; i++ {
@@ -67,7 +68,7 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCP
 					return
 				}
 				s := rawConnect(host, ports, timeout, true)
-				states = append(states, s...)
+				results = append(results, s...)
 			}
 		}()
 	}
@@ -87,9 +88,9 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCP
 	close(chanHost)
 	wg.Wait()
 	close(done)
-	return states
+	return results
 }
 
 func GetDefaultPorts() []int {
-	return []int{SSH_PORT, TLS_PORT, IPMI_PORT, REDFISH_PORT}
+	return []int{SSH_PORT, HTTPS_PORT, IPMI_PORT, REDFISH_PORT}
 }