towk/magellan
1
0
Fork 0
mirror of https://github.com/davidallendj/magellan.git synced 2025-12-20 03:27:03 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add goroutine to info collect

Browse source 
Signed-off-by: David J. Allen <allend@lanl.gov>
This commit is contained in:
David J. Allen 2023-09-13 11:21:49 -06:00
parent 2edb9fdbb0
commit c5a6d2a2be
10 changed files with 326 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

4
bin/magellan.sh
View file

@ -4,7 +4,7 @@ function build(){
}
function scan() {
./magellan scan --subnet 172.16.0.0 --db.path data/assets.db --port 623
./magellan scan --subnet 172.16.0.0 --db.path data/assets.db --port 443,623,5000
}
function list(){
@ -12,6 +12,6 @@ function list(){
}
function collect() {
./magellan collect --db.path data/assets.db --driver ipmi --timeout 5 --user admin --pass password
./magellan collect --db.path data/assets.db --driver redfish --timeout 30 --user admin --pass password
}

183
cmd/collect.go
View file

@ -1,12 +1,11 @@
package cmd
import (
"davidallendj/magellan/api/smd"
magellan "davidallendj/magellan/internal"
"davidallendj/magellan/internal/api/smd"
"davidallendj/magellan/internal/db/sqlite"
"fmt"
"github.com/Cray-HPE/hms-xname/xnames"
"github.com/cznic/mathutil"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
@ -26,93 +25,127 @@ var collectCmd = &cobra.Command{
l.Log.Errorf("could not get states: %v", err)
}
// generate custom xnames for bmcs
node := xnames.Node{
Cabinet: 1000,
Chassis: 1,
ComputeModule: 7,
NodeBMC: 1,
Node: 0,
q := &magellan.QueryParams{
User: user,
Pass: pass,
Drivers: drivers,
Timeout: timeout,
Threads: threads,
Verbose: verbose,
WithSecureTLS: withSecureTLS,
}
// use the found results to query bmc information
users := [][]byte{}
for _, ps := range probeStates {
if !ps.State {
continue
}
logrus.Infof("querying %v\n", ps)
q := magellan.QueryParams{
Host: ps.Host,
Port: ps.Port,
User: user,
Pass: pass,
Drivers: drivers,
Timeout: timeout,
Verbose: true,
WithSecureTLS: withSecureTLS,
}
// scan and store probe data in dbPath
if threads <= 0 {
threads = mathutil.Clamp(len(probeStates), 1, 255)
}
magellan.CollectInfo(&probeStates, l, q)
client, err := magellan.NewClient(l, &q)
if err != nil {
l.Log.Errorf("could not make client: %v", err)
return
}
// generate custom xnames for bmcs
// node := xnames.Node{
// Cabinet: 1000,
// Chassis: 1,
// ComputeModule: 7,
// NodeBMC: 1,
// Node: 0,
// }
// metadata
_, err = magellan.QueryMetadata(client, l, &q)
if err != nil {
l.Log.Errorf("could not query metadata: %v\n", err)
}
// // use the found results to query bmc information
// // users := [][]byte{}
// probedHosts := []string{}
// for _, ps := range probeStates {
// inventories
inventory, err := magellan.QueryInventory(client, l, &q)
// inventory, err := magellan.QueryInventoryV2(q.Host, q.Port, q.User, q.Pass)
if err != nil {
l.Log.Errorf("could not query inventory: %v\n", err)
}
// // skip if found info from host
// foundHost := slices.Index(probedHosts, ps.Host)
// if !ps.State || foundHost >= 0{
// continue
// }
node.NodeBMC += 1
// logrus.Printf("querying %v:%v (%v)\n", ps.Host, ps.Port, ps.Protocol)
data := make(map[string]any)
data["ID"] = fmt.Sprintf("%v", node)
data["FQDN"] = ps.Host
data["RediscoverOnUpdate"] = false
// client, err := magellan.NewClient(l, q)
// if err != nil {
// l.Log.Errorf("could not make client: %v", err)
// return
// }
headers := make(map[string]string)
headers["Content-Type"] = "application/json"
// // metadata
// // _, err = magellan.QueryMetadata(client, l, &q)
// // if err != nil {
// // l.Log.Errorf("could not query metadata: %v\n", err)
// // }
// add all endpoints to smd
err = smd.AddRedfishEndpoint(inventory, headers)
if err != nil {
logrus.Errorf("could not add redfish endpoint: %v", err)
}
// // inventories
// inventory, err := magellan.QueryInventory(client, l, q)
// if err != nil {
// l.Log.Errorf("could not query inventory: %v\n", err)
// continue
// }
// confirm the inventories were added
err = smd.GetRedfishEndpoints()
if err != nil {
logrus.Errorf("could not get redfish endpoints: %v\n", err)
}
// // chassis
// _, err = magellan.QueryChassis(client, l, q)
// if err != nil {
// l.Log.Errorf("could not query chassis: %v\n", err)
// continue
// }
// // got host information, so add to list of already probed hosts
// probedHosts = append(probedHosts, ps.Host)
// node.NodeBMC += 1
// headers := make(map[string]string)
// headers["Content-Type"] = "application/json"
// data := make(map[string]any)
// data["ID"] = fmt.Sprintf("%v", node)
// data["Type"] = ""
// data["Name"] = ""
// data["FQDN"] = ps.Host
// data["RediscoverOnUpdate"] = false
// data["Inventory"] = inventory
// b, err := json.MarshalIndent(data, "", " ")
// if err != nil {
// l.Log.Errorf("could not marshal JSON: %v\n", err)
// continue
// }
// // add all endpoints to smd
// err = smd.AddRedfishEndpoint(b, headers)
// if err != nil {
// logrus.Errorf("could not add redfish endpoint: %v", err)
// continue
// }
// // confirm the inventories were added
// err = smd.GetRedfishEndpoints()
// if err != nil {
// logrus.Errorf("could not get redfish endpoints: %v\n", err)
// continue
// }
// users
user, err := magellan.QueryUsers(client, l, &q)
if err != nil {
l.Log.Errorf("could not query users: %v\n", err)
}
users = append(users, user)
// user, err := magellan.QueryUsers(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query users: %v\n", err)
// }
// users = append(users, user)
// bios
_, err = magellan.QueryBios(client, l, &q)
if err != nil {
l.Log.Errorf("could not query bios: %v\n", err)
}
// // bios
// _, err = magellan.QueryBios(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query bios: %v\n", err)
// }
_, err = magellan.QueryPowerState(client, l, &q)
if err != nil {
l.Log.Errorf("could not query power state: %v\n", err)
}
// _, err = magellan.QueryPowerState(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query power state: %v\n", err)
// }
}
// }
},
}

4
cmd/root.go
View file

@ -20,6 +20,7 @@ var (
drivers []string
preferredDriver string
ipmitoolPath string
verbose bool
)
// TODO: discover bmc's on network (dora)
@ -49,7 +50,6 @@ func Execute() {
func init() {
rootCmd.PersistentFlags().IntVar(&threads, "threads", -1, "set the number of threads")
rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 10, "set the timeout")
rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", true, "set verbose flag")
rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", "/tmp/magellan.db", "set the probe storage path")
}

4
cmd/scan.go
View file

@ -42,7 +42,9 @@ var scanCmd = &cobra.Command{
threads = mathutil.Clamp(len(hostsToScan), 1, 255)
}
probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, threads, timeout)
fmt.Printf("probe states: %v\n", probeStates)
for _, r := range probeStates {
fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol)
}
sqlite.InsertProbeResults(dbpath, &probeStates)
},
}

0
api/dora/dora.go → internal/api/dora/dora.go
View file

2
api/smd/smd.go → internal/api/smd/smd.go
View file

@ -4,7 +4,7 @@ package smd
// https://github.com/Cray-HPE/hms-smd/blob/master/docs/examples.adoc
// https://github.com/alexlovelltroy/hms-smd
import (
"davidallendj/magellan/api"
"davidallendj/magellan/internal/api"
"fmt"
// hms "github.com/alexlovelltroy/hms-smd"
)

0
api/util.go → internal/api/util.go
View file

207
internal/magellan.go → internal/collect.go
View file

@ -3,21 +3,28 @@ package magellan
import (
"context"
"crypto/x509"
"davidallendj/magellan/internal/api/smd"
"encoding/json"
"fmt"
"os"
"sync"
"time"
"github.com/Cray-HPE/hms-xname/xnames"
bmclib "github.com/bmc-toolbox/bmclib/v2"
"github.com/jacobweinstock/registrar"
_ "github.com/mattn/go-sqlite3"
"github.com/sirupsen/logrus"
"github.com/stmcginnis/gofish"
_ "github.com/stmcginnis/gofish"
"github.com/stmcginnis/gofish/redfish"
"golang.org/x/exp/slices"
)
const (
IPMI_PORT = 623
SSH_PORT = 22
TLS_PORT = 443
HTTPS_PORT = 443
REDFISH_PORT = 5000
)
@ -35,6 +42,7 @@ type QueryParams struct {
User string
Pass string
Drivers []string
Threads int
Preferred string
Timeout int
WithSecureTLS bool
@ -56,10 +64,11 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
// init client
clientOpts := []bmclib.Option{
// bmclib.WithSecureTLS(),
// bmclib.WithSecureTLS(nil),
// bmclib.WithHTTPClient(&httpClient),
// bmclib.WithLogger(),
// bmclib.WithRedfishHTTPClient(&httpClient),
bmclib.WithDellRedfishUseBasicAuth(true),
bmclib.WithRedfishPort(fmt.Sprint(q.Port)),
bmclib.WithRedfishUseBasicAuth(true),
bmclib.WithIpmitoolPort(fmt.Sprint(IPMI_PORT)),
@ -93,7 +102,6 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
} else {
url += q.Host
}
client := bmclib.NewClient(url, q.User, q.Pass, clientOpts...)
ds := registrar.Drivers{}
for _, driver := range q.Drivers {
@ -104,6 +112,147 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
return client, nil
}
func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error {
if probeStates == nil {
return fmt.Errorf("no probe states found")
}
if len(*probeStates) <= 0 {
return fmt.Errorf("no probe states found")
}
// generate custom xnames for bmcs
node := xnames.Node{
Cabinet: 1000,
Chassis: 1,
ComputeModule: 7,
NodeBMC: 1,
Node: 0,
}
found := make([]string, 0, len(*probeStates))
done := make(chan struct{}, q.Threads+1)
chanProbeState := make(chan BMCProbeResult, q.Threads+1)
//
var wg sync.WaitGroup
wg.Add(q.Threads)
for i := 0; i < q.Threads; i++ {
go func() {
for {
ps, ok := <- chanProbeState
if !ok {
wg.Done()
return
}
q.Host = ps.Host
q.Port = ps.Port
logrus.Printf("querying %v:%v (%v)\n", ps.Host, ps.Port, ps.Protocol)
client, err := NewClient(l, q)
if err != nil {
l.Log.Errorf("could not make client: %v", err)
continue
}
// metadata
// _, err = magellan.QueryMetadata(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query metadata: %v\n", err)
// }
// inventories
inventory, err := QueryInventory(client, l, q)
if err != nil {
l.Log.Errorf("could not query inventory: %v", err)
}
// chassis
_, err = QueryChassis(client, l, q)
if err != nil {
l.Log.Errorf("could not query chassis: %v", err)
}
node.NodeBMC += 1
headers := make(map[string]string)
headers["Content-Type"] = "application/json"
data := make(map[string]any)
data["ID"] = fmt.Sprintf("%v", node)
data["Type"] = ""
data["Name"] = ""
data["FQDN"] = ps.Host
data["RediscoverOnUpdate"] = false
data["Inventory"] = inventory
b, err := json.MarshalIndent(data, "", " ")
if err != nil {
l.Log.Errorf("could not marshal JSON: %v", err)
}
// add all endpoints to smd
err = smd.AddRedfishEndpoint(b, headers)
if err != nil {
l.Log.Errorf("could not add redfish endpoint: %v", err)
}
// confirm the inventories were added
err = smd.GetRedfishEndpoints()
if err != nil {
l.Log.Errorf("could not get redfish endpoints: %v", err)
}
// users
// user, err := magellan.QueryUsers(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query users: %v\n", err)
// }
// users = append(users, user)
// bios
// _, err = magellan.QueryBios(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query bios: %v\n", err)
// }
// _, err = magellan.QueryPowerState(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query power state: %v\n", err)
// }
// got host information, so add to list of already probed hosts
found = append(found, ps.Host)
}
}()
}
// use the found results to query bmc information
for _, ps := range *probeStates {
// skip if found info from host
foundHost := slices.Index(found, ps.Host)
if !ps.State || foundHost >= 0{
continue
}
chanProbeState <- ps
}
go func() {
select {
case <-done:
wg.Done()
break
default:
time.Sleep(1000)
}
}()
close(chanProbeState)
wg.Wait()
close(done)
return nil
}
func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
// client, err := NewClient(l, q)
@ -113,7 +262,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er
err := client.Open(ctx)
if err != nil {
ctxCancel()
return nil, fmt.Errorf("could not open BMC client: %v", err)
return nil, fmt.Errorf("could not connect to bmc: %v", err)
}
defer client.Close(ctx)
@ -135,7 +284,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er
fmt.Printf("metadata: %v\n", string(b))
}
ctxCancel()
return []byte(b), nil
return b, nil
}
func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
@ -168,7 +317,7 @@ func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, e
fmt.Printf("inventory: %v\n", string(b))
}
ctxCancel()
return []byte(b), nil
return b, nil
}
func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
@ -198,7 +347,7 @@ func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte,
fmt.Printf("power state: %v\n", string(b))
}
ctxCancel()
return []byte(b), nil
return b, nil
}
@ -215,7 +364,7 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
err := client.Open(ctx)
if err != nil {
ctxCancel()
return nil, fmt.Errorf("could not open BMC client: %v", err)
return nil, fmt.Errorf("could not connect to bmc: %v", err)
}
defer client.Close(ctx)
@ -238,7 +387,7 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
if q.Verbose {
fmt.Printf("users: %v\n", string(b))
}
return []byte(b), nil
return b, nil
}
func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
@ -253,6 +402,46 @@ func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error)
return b, err
}
func QueryEthernetInterfaces(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
config := gofish.ClientConfig{
}
c, err := gofish.Connect(config)
if err != nil {
}
redfish.ListReferencedEthernetInterfaces(c, "")
return []byte{}, nil
}
func QueryChassis(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
config := gofish.ClientConfig {
Endpoint: "https://" + q.Host,
Username: q.User,
Password: q.Pass,
Insecure: q.WithSecureTLS,
}
c, err := gofish.Connect(config)
if err != nil {
return nil, fmt.Errorf("could not connect to bmc: %v", err)
}
chassis, err := c.Service.Chassis()
if err != nil {
return nil, fmt.Errorf("could not query chassis: %v", err)
}
b, err := json.MarshalIndent(chassis, "", " ")
if err != nil {
return nil, fmt.Errorf("could not marshal JSON: %v", err)
}
if q.Verbose {
fmt.Printf("chassis: %v\n", string(b))
}
return b, nil
}
func makeRequest[T interface{}](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) {
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout))
client.Registry.FilterForCompatible(ctx)

9
internal/db/sqlite/sqlite.go
View file

@ -16,10 +16,11 @@ func InsertProbeResults(path string, states *[]magellan.BMCProbeResult) error {
// create database if it doesn't already exist
schema := `
CREATE TABLE IF NOT EXISTS magellan_scanned_ports (
host TEXT PRIMARY KEY NOT NULL,
port INTEGER,
host TEXT NOT NULL,
port INTEGER NOT NULL,
protocol TEXT,
state INTEGER
state INTEGER,
PRIMARY KEY (host, port)
);
`
db, err := sqlx.Open("sqlite3", path)
@ -52,7 +53,7 @@ func GetProbeResults(path string) ([]magellan.BMCProbeResult, error) {
}
results := []magellan.BMCProbeResult{}
err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC")
err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC, port ASC;")
if err != nil {
return nil, fmt.Errorf("could not retrieve probes: %v", err)
}

13
internal/scan.go
View file

@ -52,10 +52,11 @@ func GenerateHosts(subnet string, begin uint8, end uint8) []string {
}
func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCProbeResult {
states := make([]BMCProbeResult, 0, len(hosts))
done := make(chan struct{}, threads+1)
chanHost := make(chan string, threads+1)
results := make([]BMCProbeResult, 0, len(hosts))
done := make(chan struct{}, threads+1)
chanHost := make(chan string, threads+1)
// chanPort := make(chan int, threads+1)
var wg sync.WaitGroup
wg.Add(threads)
for i := 0; i < threads; i++ {
@ -67,7 +68,7 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCP
return
}
s := rawConnect(host, ports, timeout, true)
states = append(states, s...)
results = append(results, s...)
}
}()
}
@ -87,9 +88,9 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCP
close(chanHost)
wg.Wait()
close(done)
return states
return results
}
func GetDefaultPorts() []int {
return []int{SSH_PORT, TLS_PORT, IPMI_PORT, REDFISH_PORT}
return []int{SSH_PORT, HTTPS_PORT, IPMI_PORT, REDFISH_PORT}
}