towk/magellan
1
0
Fork 0
mirror of https://github.com/davidallendj/magellan.git synced 2025-12-20 03:27:03 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add goroutine to info collect

Browse source 
Signed-off-by: David J. Allen <allend@lanl.gov>
This commit is contained in:
David J. Allen 2023-09-13 11:21:49 -06:00
parent 2edb9fdbb0
commit c5a6d2a2be
10 changed files with 326 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

74
internal/api/dora/dora.go Normal file
View file

@ -0,0 +1,74 @@
package dora
import (
"davidallendj/magellan/api"
"encoding/json"
"fmt"
"github.com/jmoiron/sqlx"
)
const (
Host = "http://localhost"
DbType = "sqlite3"
DbPath = "../data/assets.db"
BaseEndpoint = "/v1"
Port = 8000
)
type ScannedResult struct {
id string
site any
cidr string
ip string
port int
protocol string
scanner string
state string
updated string
}
func makeEndpointUrl(endpoint string) string {
return Host + ":" + fmt.Sprint(Port) + BaseEndpoint + endpoint
}
// Scan for BMC assets uing dora scanner
func ScanForAssets() error {
return nil
}
// Query dora API to get scanned ports
func QueryScannedPorts() error {
// Perform scan and collect from dora server
url := makeEndpointUrl("/scanned_ports")
_, body, err := api.MakeRequest(url, "GET", nil, nil)
if err != nil {
return fmt.Errorf("could not discover assets: %v", err)
}
// get data from JSON
var res map[string]any
if err := json.Unmarshal(body, &res); err != nil {
return fmt.Errorf("could not unmarshal response body: %v", err)
}
data := res["data"]
fmt.Println(data)
return nil
}
// Loads scanned ports directly from DB
func LoadScannedPortsFromDB(dbPath string, dbType string) {
db, _ := sqlx.Open(dbType, dbPath)
sql := `SELECT * FROM scanned_port WHERE state='open'`
rows, _ := db.Query(sql)
for rows.Next() {
var r ScannedResult
rows.Scan(
&r.id, &r.site, &r.cidr, &r.ip, &r.port, &r.protocol, &r.scanner,
&r.state, &r.updated,
)
}
}

63
internal/api/smd/smd.go Normal file
View file

@ -0,0 +1,63 @@
package smd
// See ref for API docs:
// https://github.com/Cray-HPE/hms-smd/blob/master/docs/examples.adoc
// https://github.com/alexlovelltroy/hms-smd
import (
"davidallendj/magellan/internal/api"
"fmt"
// hms "github.com/alexlovelltroy/hms-smd"
)
var (
Host = "http://localhost"
BaseEndpoint = "/hsm/v2"
Port = 27779
)
func makeEndpointUrl(endpoint string) string {
return Host + ":" + fmt.Sprint(Port) + BaseEndpoint + endpoint
}
func GetRedfishEndpoints() error {
url := makeEndpointUrl("/Inventory/RedfishEndpoints")
_, body, err := api.MakeRequest(url, "GET", nil, nil)
if err != nil {
return fmt.Errorf("could not get endpoint: %v", err)
}
// fmt.Println(res)
fmt.Println(string(body))
return nil
}
func GetComponentEndpoint(xname string) error {
url := makeEndpointUrl("/Inventory/ComponentsEndpoints/" + xname)
res, body, err := api.MakeRequest(url, "GET", nil, nil)
if err != nil {
return fmt.Errorf("could not get endpoint: %v", err)
}
fmt.Println(res)
fmt.Println(string(body))
return nil
}
func AddRedfishEndpoint(data []byte, headers map[string]string) error {
if data == nil {
return fmt.Errorf("could not add redfish endpoint: no data found")
}
// var ep hms.RedfishEP
// _ = ep
// Add redfish endpoint via POST `/hsm/v2/Inventory/RedfishEndpoints` endpoint
url := makeEndpointUrl("/Inventory/RedfishEndpoints")
res, body, _ := api.MakeRequest(url, "POST", data, headers)
fmt.Println("smd url: ", url)
fmt.Println("res: ", res)
fmt.Println("body: ", string(body))
return nil
}
func UpdateRedfishEndpoint() {
// Update redfish endpoint via PUT `/hsm/v2/Inventory/RedfishEndpoints` endpoint
}

28
internal/api/util.go Normal file
View file

@ -0,0 +1,28 @@
package api
import (
"bytes"
"fmt"
"io"
"net/http"
)
func MakeRequest(url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) {
// url := getSmdEndpointUrl(endpoint)
req, _ := http.NewRequest(httpMethod, url, bytes.NewBuffer(body))
req.Header.Add("User-Agent", "magellan")
for k, v := range headers {
req.Header.Add(k, v)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("could not make request: %v", err)
}
b, err := io.ReadAll(res.Body)
res.Body.Close()
if err != nil {
return nil, nil, fmt.Errorf("could not read response body: %v", err)
}
return res, b, err
}

207
internal/magellan.go → internal/collect.go
View file

@ -3,21 +3,28 @@ package magellan
import (
"context"
"crypto/x509"
"davidallendj/magellan/internal/api/smd"
"encoding/json"
"fmt"
"os"
"sync"
"time"
"github.com/Cray-HPE/hms-xname/xnames"
bmclib "github.com/bmc-toolbox/bmclib/v2"
"github.com/jacobweinstock/registrar"
_ "github.com/mattn/go-sqlite3"
"github.com/sirupsen/logrus"
"github.com/stmcginnis/gofish"
_ "github.com/stmcginnis/gofish"
"github.com/stmcginnis/gofish/redfish"
"golang.org/x/exp/slices"
)
const (
IPMI_PORT = 623
SSH_PORT = 22
TLS_PORT = 443
HTTPS_PORT = 443
REDFISH_PORT = 5000
)
@ -35,6 +42,7 @@ type QueryParams struct {
User string
Pass string
Drivers []string
Threads int
Preferred string
Timeout int
WithSecureTLS bool
@ -56,10 +64,11 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
// init client
clientOpts := []bmclib.Option{
// bmclib.WithSecureTLS(),
// bmclib.WithSecureTLS(nil),
// bmclib.WithHTTPClient(&httpClient),
// bmclib.WithLogger(),
// bmclib.WithRedfishHTTPClient(&httpClient),
bmclib.WithDellRedfishUseBasicAuth(true),
bmclib.WithRedfishPort(fmt.Sprint(q.Port)),
bmclib.WithRedfishUseBasicAuth(true),
bmclib.WithIpmitoolPort(fmt.Sprint(IPMI_PORT)),
@ -93,7 +102,6 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
} else {
url += q.Host
}
client := bmclib.NewClient(url, q.User, q.Pass, clientOpts...)
ds := registrar.Drivers{}
for _, driver := range q.Drivers {
@ -104,6 +112,147 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
return client, nil
}
func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error {
if probeStates == nil {
return fmt.Errorf("no probe states found")
}
if len(*probeStates) <= 0 {
return fmt.Errorf("no probe states found")
}
// generate custom xnames for bmcs
node := xnames.Node{
Cabinet: 1000,
Chassis: 1,
ComputeModule: 7,
NodeBMC: 1,
Node: 0,
}
found := make([]string, 0, len(*probeStates))
done := make(chan struct{}, q.Threads+1)
chanProbeState := make(chan BMCProbeResult, q.Threads+1)
//
var wg sync.WaitGroup
wg.Add(q.Threads)
for i := 0; i < q.Threads; i++ {
go func() {
for {
ps, ok := <- chanProbeState
if !ok {
wg.Done()
return
}
q.Host = ps.Host
q.Port = ps.Port
logrus.Printf("querying %v:%v (%v)\n", ps.Host, ps.Port, ps.Protocol)
client, err := NewClient(l, q)
if err != nil {
l.Log.Errorf("could not make client: %v", err)
continue
}
// metadata
// _, err = magellan.QueryMetadata(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query metadata: %v\n", err)
// }
// inventories
inventory, err := QueryInventory(client, l, q)
if err != nil {
l.Log.Errorf("could not query inventory: %v", err)
}
// chassis
_, err = QueryChassis(client, l, q)
if err != nil {
l.Log.Errorf("could not query chassis: %v", err)
}
node.NodeBMC += 1
headers := make(map[string]string)
headers["Content-Type"] = "application/json"
data := make(map[string]any)
data["ID"] = fmt.Sprintf("%v", node)
data["Type"] = ""
data["Name"] = ""
data["FQDN"] = ps.Host
data["RediscoverOnUpdate"] = false
data["Inventory"] = inventory
b, err := json.MarshalIndent(data, "", " ")
if err != nil {
l.Log.Errorf("could not marshal JSON: %v", err)
}
// add all endpoints to smd
err = smd.AddRedfishEndpoint(b, headers)
if err != nil {
l.Log.Errorf("could not add redfish endpoint: %v", err)
}
// confirm the inventories were added
err = smd.GetRedfishEndpoints()
if err != nil {
l.Log.Errorf("could not get redfish endpoints: %v", err)
}
// users
// user, err := magellan.QueryUsers(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query users: %v\n", err)
// }
// users = append(users, user)
// bios
// _, err = magellan.QueryBios(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query bios: %v\n", err)
// }
// _, err = magellan.QueryPowerState(client, l, &q)
// if err != nil {
// l.Log.Errorf("could not query power state: %v\n", err)
// }
// got host information, so add to list of already probed hosts
found = append(found, ps.Host)
}
}()
}
// use the found results to query bmc information
for _, ps := range *probeStates {
// skip if found info from host
foundHost := slices.Index(found, ps.Host)
if !ps.State || foundHost >= 0{
continue
}
chanProbeState <- ps
}
go func() {
select {
case <-done:
wg.Done()
break
default:
time.Sleep(1000)
}
}()
close(chanProbeState)
wg.Wait()
close(done)
return nil
}
func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
// client, err := NewClient(l, q)
@ -113,7 +262,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er
err := client.Open(ctx)
if err != nil {
ctxCancel()
return nil, fmt.Errorf("could not open BMC client: %v", err)
return nil, fmt.Errorf("could not connect to bmc: %v", err)
}
defer client.Close(ctx)
@ -135,7 +284,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er
fmt.Printf("metadata: %v\n", string(b))
}
ctxCancel()
return []byte(b), nil
return b, nil
}
func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
@ -168,7 +317,7 @@ func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, e
fmt.Printf("inventory: %v\n", string(b))
}
ctxCancel()
return []byte(b), nil
return b, nil
}
func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
@ -198,7 +347,7 @@ func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte,
fmt.Printf("power state: %v\n", string(b))
}
ctxCancel()
return []byte(b), nil
return b, nil
}
@ -215,7 +364,7 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
err := client.Open(ctx)
if err != nil {
ctxCancel()
return nil, fmt.Errorf("could not open BMC client: %v", err)
return nil, fmt.Errorf("could not connect to bmc: %v", err)
}
defer client.Close(ctx)
@ -238,7 +387,7 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
if q.Verbose {
fmt.Printf("users: %v\n", string(b))
}
return []byte(b), nil
return b, nil
}
func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
@ -253,6 +402,46 @@ func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error)
return b, err
}
func QueryEthernetInterfaces(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
config := gofish.ClientConfig{
}
c, err := gofish.Connect(config)
if err != nil {
}
redfish.ListReferencedEthernetInterfaces(c, "")
return []byte{}, nil
}
func QueryChassis(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
config := gofish.ClientConfig {
Endpoint: "https://" + q.Host,
Username: q.User,
Password: q.Pass,
Insecure: q.WithSecureTLS,
}
c, err := gofish.Connect(config)
if err != nil {
return nil, fmt.Errorf("could not connect to bmc: %v", err)
}
chassis, err := c.Service.Chassis()
if err != nil {
return nil, fmt.Errorf("could not query chassis: %v", err)
}
b, err := json.MarshalIndent(chassis, "", " ")
if err != nil {
return nil, fmt.Errorf("could not marshal JSON: %v", err)
}
if q.Verbose {
fmt.Printf("chassis: %v\n", string(b))
}
return b, nil
}
func makeRequest[T interface{}](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) {
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout))
client.Registry.FilterForCompatible(ctx)

9
internal/db/sqlite/sqlite.go
View file

@ -16,10 +16,11 @@ func InsertProbeResults(path string, states *[]magellan.BMCProbeResult) error {
// create database if it doesn't already exist
schema := `
CREATE TABLE IF NOT EXISTS magellan_scanned_ports (
host TEXT PRIMARY KEY NOT NULL,
port INTEGER,
host TEXT NOT NULL,
port INTEGER NOT NULL,
protocol TEXT,
state INTEGER
state INTEGER,
PRIMARY KEY (host, port)
);
`
db, err := sqlx.Open("sqlite3", path)
@ -52,7 +53,7 @@ func GetProbeResults(path string) ([]magellan.BMCProbeResult, error) {
}
results := []magellan.BMCProbeResult{}
err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC")
err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC, port ASC;")
if err != nil {
return nil, fmt.Errorf("could not retrieve probes: %v", err)
}

13
internal/scan.go
View file

@ -52,10 +52,11 @@ func GenerateHosts(subnet string, begin uint8, end uint8) []string {
}
func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCProbeResult {
states := make([]BMCProbeResult, 0, len(hosts))
done := make(chan struct{}, threads+1)
chanHost := make(chan string, threads+1)
results := make([]BMCProbeResult, 0, len(hosts))
done := make(chan struct{}, threads+1)
chanHost := make(chan string, threads+1)
// chanPort := make(chan int, threads+1)
var wg sync.WaitGroup
wg.Add(threads)
for i := 0; i < threads; i++ {
@ -67,7 +68,7 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCP
return
}
s := rawConnect(host, ports, timeout, true)
states = append(states, s...)
results = append(results, s...)
}
}()
}
@ -87,9 +88,9 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCP
close(chanHost)
wg.Wait()
close(done)
return states
return results
}
func GetDefaultPorts() []int {
return []int{SSH_PORT, TLS_PORT, IPMI_PORT, REDFISH_PORT}
return []int{SSH_PORT, HTTPS_PORT, IPMI_PORT, REDFISH_PORT}
}