towk/magellan
1
0
Fork 0
mirror of https://github.com/davidallendj/magellan.git synced 2025-12-20 03:27:03 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3 from bikeshack/store-data

Browse source 
Added ability to store output of `collect` command
This commit is contained in:
Alex Lovell-Troy 2023-09-20 17:42:42 -04:00 • committed by GitHub
commit 16f7664931
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 135 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

3
README.md
View file

@ -83,6 +83,7 @@ List of things left to fix, do, or ideas...
* [ ] Switch to internal scanner if `dora` fails
* [ ] Set default port automatically depending on the driver used to scan
* [X] Test using different `bmclib` supported drivers (mainly 'redfish')
* [ ] Confirm loading different components into `hms-smd`
* [X] Confirm loading different components into `hms-smd`
* [ ] Add ability to set subnet mask for scanning
* [ ] Add unit tests for `scan`, `list`, and `collect` commands
* [ ] Clean up, remove unused, and tidy code

6
cmd/collect.go
View file

@ -4,7 +4,7 @@ import (
magellan "github.com/bikeshack/magellan/internal"
"github.com/bikeshack/magellan/internal/api/smd"
"github.com/bikeshack/magellan/internal/db/sqlite"
"github.com/bikeshack/magellan/internal/log"
"github.com/cznic/mathutil"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@ -15,7 +15,7 @@ var collectCmd = &cobra.Command{
Short: "Query information about BMC",
Run: func(cmd *cobra.Command, args []string) {
// make application logger
l := magellan.NewLogger(logrus.New(), logrus.DebugLevel)
l := log.NewLogger(logrus.New(), logrus.DebugLevel)
// get probe states stored in db from scan
probeStates, err := sqlite.GetProbeResults(dbpath)
@ -35,6 +35,7 @@ var collectCmd = &cobra.Command{
Threads: threads,
Verbose: verbose,
WithSecureTLS: withSecureTLS,
OutputPath: outputPath,
}
magellan.CollectInfo(&probeStates, l, q)
@ -52,6 +53,7 @@ func init() {
collectCmd.PersistentFlags().IntVar(&smd.Port, "port", smd.Port, "set the port to the smd API")
collectCmd.PersistentFlags().StringVar(&user, "user", "", "set the BMC user")
collectCmd.PersistentFlags().StringVar(&pass, "pass", "", "set the BMC password")
collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", "/tmp/magellan/data/", "set the path to store collection data")
collectCmd.PersistentFlags().StringVar(&preferredDriver, "preferred-driver", "ipmi", "set the preferred driver to use")
collectCmd.PersistentFlags().StringVar(&ipmitoolPath, "ipmitool.path", "/usr/bin/ipmitool", "set the path for ipmitool")
collectCmd.PersistentFlags().BoolVar(&withSecureTLS, "secure-tls", false, "enable secure TLS")

3
cmd/root.go
View file

@ -20,6 +20,7 @@ var (
drivers []string
preferredDriver string
ipmitoolPath string
outputPath string
verbose bool
)
@ -51,5 +52,5 @@ func init() {
rootCmd.PersistentFlags().IntVar(&threads, "threads", -1, "set the number of threads")
rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 30, "set the timeout")
rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", true, "set verbose flag")
rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", "/tmp/magellan.db", "set the probe storage path")
rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", "/tmp/magellan/magellan.db", "set the probe storage path")
}

9
cmd/scan.go
View file

@ -2,6 +2,8 @@ package cmd
import (
"fmt"
"os"
"path"
magellan "github.com/bikeshack/magellan/internal"
"github.com/bikeshack/magellan/internal/db/sqlite"
@ -46,6 +48,13 @@ var scanCmd = &cobra.Command{
for _, r := range probeStates {
fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol)
}
// make the dbpath dir if needed
err := os.MkdirAll(path.Dir(dbpath), 0766)
if err != nil {
fmt.Printf("could not make database directory: %v", err)
}
sqlite.InsertProbeResults(dbpath, &probeStates)
},
}

4
internal/api/dora/dora.go
View file

@ -4,7 +4,7 @@ import (
"encoding/json"
"fmt"
"github.com/bikeshack/magellan/internal/api"
"github.com/bikeshack/magellan/internal/util"
"github.com/jmoiron/sqlx"
)
@ -43,7 +43,7 @@ func ScanForAssets() error {
func QueryScannedPorts() error {
// Perform scan and collect from dora server
url := makeEndpointUrl("/scanned_ports")
_, body, err := api.MakeRequest(url, "GET", nil, nil)
_, body, err := util.MakeRequest(url, "GET", nil, nil)
if err != nil {
return fmt.Errorf("could not discover assets: %v", err)
}

8
internal/api/smd/smd.go
View file

@ -6,7 +6,7 @@ package smd
import (
"fmt"
"github.com/bikeshack/magellan/internal/api"
"github.com/bikeshack/magellan/internal/util"
// hms "github.com/alexlovelltroy/hms-smd"
)
@ -22,7 +22,7 @@ func makeEndpointUrl(endpoint string) string {
func GetRedfishEndpoints() error {
url := makeEndpointUrl("/Inventory/RedfishEndpoints")
_, body, err := api.MakeRequest(url, "GET", nil, nil)
_, body, err := util.MakeRequest(url, "GET", nil, nil)
if err != nil {
return fmt.Errorf("could not get endpoint: %v", err)
}
@ -33,7 +33,7 @@ func GetRedfishEndpoints() error {
func GetComponentEndpoint(xname string) error {
url := makeEndpointUrl("/Inventory/ComponentsEndpoints/" + xname)
res, body, err := api.MakeRequest(url, "GET", nil, nil)
res, body, err := util.MakeRequest(url, "GET", nil, nil)
if err != nil {
return fmt.Errorf("could not get endpoint: %v", err)
}
@ -51,7 +51,7 @@ func AddRedfishEndpoint(data []byte, headers map[string]string) error {
// _ = ep
// Add redfish endpoint via POST `/hsm/v2/Inventory/RedfishEndpoints` endpoint
url := makeEndpointUrl("/Inventory/RedfishEndpoints")
res, body, _ := api.MakeRequest(url, "POST", data, headers)
res, body, _ := util.MakeRequest(url, "POST", data, headers)
fmt.Println("smd url: ", url)
fmt.Println("res: ", res)
fmt.Println("body: ", string(body))

27
internal/api/util.go
View file

@ -1,27 +0,0 @@
package api
import (
"bytes"
"fmt"
"io"
"net/http"
)
func MakeRequest(url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) {
// url := getSmdEndpointUrl(endpoint)
req, _ := http.NewRequest(httpMethod, url, bytes.NewBuffer(body))
req.Header.Add("User-Agent", "magellan")
for k, v := range headers {
req.Header.Add(k, v)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("could not make request: %v", err)
}
b, err := io.ReadAll(res.Body)
res.Body.Close()
if err != nil {
return nil, nil, fmt.Errorf("could not read response body: %v", err)
}
return res, b, err
}

73
internal/collect.go
View file

@ -8,10 +8,14 @@ import (
"fmt"
"net/http"
"os"
"path"
"sync"
"time"
"github.com/bikeshack/magellan/internal/log"
"github.com/bikeshack/magellan/internal/api/smd"
"github.com/bikeshack/magellan/internal/util"
"github.com/Cray-HPE/hms-xname/xnames"
bmclib "github.com/bmc-toolbox/bmclib/v2"
@ -50,9 +54,10 @@ type QueryParams struct {
CertPoolFile string
Verbose bool
IpmitoolPath string
OutputPath string
}
func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
func NewClient(l *log.Logger, q *QueryParams) (*bmclib.Client, error) {
// NOTE: bmclib.NewClient(host, port, user, pass)
// ...seems like the `port` params doesn't work like expected depending on interface
@ -106,7 +111,7 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
return client, nil
}
func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error {
func CollectInfo(probeStates *[]BMCProbeResult, l *log.Logger, q *QueryParams) error {
// check for available probe states
if probeStates == nil {
return fmt.Errorf("no probe states found")
@ -123,6 +128,13 @@ func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error
NodeBMC: -1,
}
// make the output directory to store files
outputPath := path.Clean(q.OutputPath)
outputPath, err := util.MakeOutputDirectory(outputPath)
if err != nil {
l.Log.Errorf("could not make output directory: %v", err)
}
found := make([]string, 0, len(*probeStates))
done := make(chan struct{}, q.Threads+1)
chanProbeState := make(chan BMCProbeResult, q.Threads+1)
@ -171,6 +183,11 @@ func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error
headers := make(map[string]string)
headers["Content-Type"] = "application/json"
// unmarshal json to send in correct format
var iobj, cobj map[string]json.RawMessage
json.Unmarshal(inventory, &iobj)
json.Unmarshal(chassis, &cobj)
data := make(map[string]any)
data["ID"] = fmt.Sprintf("%v", node.String()[:len(node.String())-2])
data["Type"] = ""
@ -179,14 +196,20 @@ func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error
data["User"] = q.User
data["Password"] = q.Pass
data["RediscoverOnUpdate"] = false
data["Inventory"] = inventory
data["Chassis"] = chassis
data["Inventory"] = iobj
data["Chassis"] = cobj
b, err := json.MarshalIndent(data, "", " ")
if err != nil {
l.Log.Errorf("could not marshal JSON: %v", err)
}
// write JSON data to file
err = os.WriteFile(path.Clean(outputPath + "/" + q.Host + ".json"), b, os.ModePerm)
if err != nil {
l.Log.Errorf("could not write data to file: %v", err)
}
// add all endpoints to smd
err = smd.AddRedfishEndpoint(b, headers)
if err != nil {
@ -246,7 +269,7 @@ func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error
return nil
}
func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
func QueryMetadata(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
// client, err := NewClient(l, q)
// open BMC session and update driver registry
@ -280,7 +303,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er
return b, nil
}
func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
func QueryInventory(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
// open BMC session and update driver registry
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout))
client.Registry.FilterForCompatible(ctx)
@ -297,21 +320,23 @@ func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, e
return nil, fmt.Errorf("could not get inventory: %v", err)
}
// retrieve inventory data
b, err := json.MarshalIndent(inventory, "", " ")
data := map[string]any{"Inventory": inventory}
b, err := json.MarshalIndent(data, "", " ")
if err != nil {
ctxCancel()
return nil, fmt.Errorf("could not marshal JSON: %v", err)
}
if q.Verbose {
fmt.Printf("inventory: %v\n", string(b))
fmt.Printf("%v\n", string(b))
}
ctxCancel()
return b, nil
}
func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
func QueryPowerState(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout))
client.Registry.FilterForCompatible(ctx)
err := client.PreferProvider(q.Preferred).Open(ctx)
@ -321,28 +346,29 @@ func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte,
}
defer client.Close(ctx)
inventory, err := client.GetPowerState(ctx)
powerState, err := client.GetPowerState(ctx)
if err != nil {
ctxCancel()
return nil, fmt.Errorf("could not get inventory: %v", err)
}
// retrieve inventory data
b, err := json.MarshalIndent(inventory, "", " ")
data := map[string]any{"PowerState": powerState}
b, err := json.MarshalIndent(data, "", " ")
if err != nil {
ctxCancel()
return nil, fmt.Errorf("could not marshal JSON: %v", err)
}
if q.Verbose {
fmt.Printf("power state: %v\n", string(b))
fmt.Printf("%v\n", string(b))
}
ctxCancel()
return b, nil
}
func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
func QueryUsers(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
// open BMC session and update driver registry
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout))
client.Registry.FilterForCompatible(ctx)
@ -361,7 +387,8 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
}
// retrieve inventory data
b, err := json.MarshalIndent(users, "", " ")
data := map[string]any {"Users": users}
b, err := json.MarshalIndent(data, "", " ")
if err != nil {
ctxCancel()
return nil, fmt.Errorf("could not marshal JSON: %v", err)
@ -370,24 +397,24 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
// return b, nil
ctxCancel()
if q.Verbose {
fmt.Printf("users: %v\n", string(b))
fmt.Printf("%v\n", string(b))
}
return b, nil
}
func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
func QueryBios(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
// client, err := NewClient(l, q)
// if err != nil {
// return nil, fmt.Errorf("could not make query: %v", err)
// }
b, err := makeRequest(client, client.GetBiosConfiguration, q.Timeout)
if q.Verbose {
fmt.Printf("bios: %v\n", string(b))
fmt.Printf("%v\n", string(b))
}
return b, err
}
func QueryEthernetInterfaces(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
func QueryEthernetInterfaces(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
config := gofish.ClientConfig{
Endpoint: fmt.Sprintf("https://%s:%d", q.Host, q.Port),
Username: q.User,
@ -413,7 +440,6 @@ func QueryEthernetInterfaces(client *bmclib.Client, l *Logger, q *QueryParams) (
}
func QueryChassis(q *QueryParams) ([]byte, error) {
url := "https://"
if q.User != "" && q.Pass != "" {
url += fmt.Sprintf("%s:%s@", q.User, q.Pass)
@ -436,18 +462,19 @@ func QueryChassis(q *QueryParams) ([]byte, error) {
return nil, fmt.Errorf("could not query chassis (%v:%v): %v", q.Host, q.Port, err)
}
b, err := json.MarshalIndent(chassis, "", " ")
data := map[string]any{"chassis": chassis}
b, err := json.MarshalIndent(data, "", " ")
if err != nil {
return nil, fmt.Errorf("could not marshal JSON: %v", err)
}
if q.Verbose {
fmt.Printf("chassis: %v\n", string(b))
fmt.Printf("%v\n", string(b))
}
return b, nil
}
func makeRequest[T interface{}](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) {
func makeRequest[T any](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) {
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout))
client.Registry.FilterForCompatible(ctx)
err := client.Open(ctx)
@ -468,7 +495,7 @@ func makeRequest[T interface{}](client *bmclib.Client, fn func(context.Context)
return makeJson(response)
}
func makeJson(object interface{}) ([]byte, error) {
func makeJson(object any) ([]byte, error) {
b, err := json.MarshalIndent(object, "", " ")
if err != nil {
return nil, fmt.Errorf("could not marshal JSON: %v", err)

2
internal/logger.go → internal/log/logger.go
View file

@ -1,4 +1,4 @@
package magellan
package log
import (
"github.com/sirupsen/logrus"

60
internal/util/util.go Normal file
View file

@ -0,0 +1,60 @@
package util
import (
"bytes"
"fmt"
"io"
"net/http"
"os"
"time"
)
func PathExists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil { return true, nil }
if os.IsNotExist(err) { return false, nil }
return false, err
}
func MakeRequest(url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) {
// url := getSmdEndpointUrl(endpoint)
req, _ := http.NewRequest(httpMethod, url, bytes.NewBuffer(body))
req.Header.Add("User-Agent", "magellan")
for k, v := range headers {
req.Header.Add(k, v)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("could not make request: %v", err)
}
b, err := io.ReadAll(res.Body)
res.Body.Close()
if err != nil {
return nil, nil, fmt.Errorf("could not read response body: %v", err)
}
return res, b, err
}
func MakeOutputDirectory(path string) (string, error) {
// get the current data + time using Go's stupid formatting
t := time.Now()
dirname := t.Format("2006-01-01 15:04:05")
final := path + "/" + dirname
// check if path is valid and directory
pathExists, err := PathExists(final);
if err != nil {
return final, fmt.Errorf("could not check for existing path: %v", err)
}
if pathExists {
// make sure it is directory with 0o644 permissions
return final, fmt.Errorf("found existing path: %v", final)
}
// create directory with data + time
err = os.MkdirAll(final, 0766)
if err != nil {
return final, fmt.Errorf("could not make directory: %v", err)
}
return final, nil
}