Merge pull request #3 from bikeshack/store-data

Added ability to store output of `collect` command

README.md

@@ -83,6 +83,7 @@ List of things left to fix, do, or ideas...
 * [ ] Switch to internal scanner if `dora` fails
 * [ ] Set default port automatically depending on the driver used to scan
 * [X] Test using different `bmclib` supported drivers (mainly 'redfish')
-* [ ] Confirm loading different components into `hms-smd`
+* [X] Confirm loading different components into `hms-smd`
+* [ ] Add ability to set subnet mask for scanning
 * [ ] Add unit tests for `scan`, `list`, and `collect` commands
 * [ ] Clean up, remove unused, and tidy code

cmd/collect.go

@@ -4,7 +4,7 @@ import (
 	magellan "github.com/bikeshack/magellan/internal"
 	"github.com/bikeshack/magellan/internal/api/smd"
 	"github.com/bikeshack/magellan/internal/db/sqlite"
-
+	"github.com/bikeshack/magellan/internal/log"
 	"github.com/cznic/mathutil"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -15,7 +15,7 @@ var collectCmd = &cobra.Command{
 	Short: "Query information about BMC",
 	Run: func(cmd *cobra.Command, args []string) {
 		// make application logger
-		l := magellan.NewLogger(logrus.New(), logrus.DebugLevel)
+		l := log.NewLogger(logrus.New(), logrus.DebugLevel)
 
 		// get probe states stored in db from scan
 		probeStates, err := sqlite.GetProbeResults(dbpath)
@@ -35,6 +35,7 @@ var collectCmd = &cobra.Command{
 			Threads:       threads,
 			Verbose:       verbose,
 			WithSecureTLS: withSecureTLS,
+			OutputPath:    outputPath,
 		}
 		magellan.CollectInfo(&probeStates, l, q)
 
@@ -52,6 +53,7 @@ func init() {
 	collectCmd.PersistentFlags().IntVar(&smd.Port, "port", smd.Port, "set the port to the smd API")
 	collectCmd.PersistentFlags().StringVar(&user, "user", "", "set the BMC user")
 	collectCmd.PersistentFlags().StringVar(&pass, "pass", "", "set the BMC password")
+	collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", "/tmp/magellan/data/", "set the path to store collection data")
 	collectCmd.PersistentFlags().StringVar(&preferredDriver, "preferred-driver", "ipmi", "set the preferred driver to use")
 	collectCmd.PersistentFlags().StringVar(&ipmitoolPath, "ipmitool.path", "/usr/bin/ipmitool", "set the path for ipmitool")
 	collectCmd.PersistentFlags().BoolVar(&withSecureTLS, "secure-tls", false, "enable secure TLS")

cmd/root.go

@@ -20,6 +20,7 @@ var (
 	drivers         []string
 	preferredDriver string
 	ipmitoolPath    string
+	outputPath		string
 	verbose         bool
 )
 
@@ -51,5 +52,5 @@ func init() {
 	rootCmd.PersistentFlags().IntVar(&threads, "threads", -1, "set the number of threads")
 	rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 30, "set the timeout")
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", true, "set verbose flag")
-	rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", "/tmp/magellan.db", "set the probe storage path")
+	rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", "/tmp/magellan/magellan.db", "set the probe storage path")
 }

cmd/scan.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+	"path"
 
 	magellan "github.com/bikeshack/magellan/internal"
 	"github.com/bikeshack/magellan/internal/db/sqlite"
@@ -46,6 +48,13 @@ var scanCmd = &cobra.Command{
 		for _, r := range probeStates {
 			fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol)
 		}
+
+		// make the dbpath dir if needed
+		err := os.MkdirAll(path.Dir(dbpath), 0766)
+		if err != nil {
+			fmt.Printf("could not make database directory: %v", err)
+		}
+
 		sqlite.InsertProbeResults(dbpath, &probeStates)
 	},
 }

internal/api/dora/dora.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/bikeshack/magellan/internal/api"
+	"github.com/bikeshack/magellan/internal/util"
 
 	"github.com/jmoiron/sqlx"
 )
@@ -43,7 +43,7 @@ func ScanForAssets() error {
 func QueryScannedPorts() error {
 	// Perform scan and collect from dora server
 	url := makeEndpointUrl("/scanned_ports")
-	_, body, err := api.MakeRequest(url, "GET", nil, nil)
+	_, body, err := util.MakeRequest(url, "GET", nil, nil)
 	if err != nil {
 		return fmt.Errorf("could not discover assets: %v", err)
 	}

internal/api/smd/smd.go

@@ -6,7 +6,7 @@ package smd
 import (
 	"fmt"
 
-	"github.com/bikeshack/magellan/internal/api"
+	"github.com/bikeshack/magellan/internal/util"
 	// hms "github.com/alexlovelltroy/hms-smd"
 )
 
@@ -22,7 +22,7 @@ func makeEndpointUrl(endpoint string) string {
 
 func GetRedfishEndpoints() error {
 	url := makeEndpointUrl("/Inventory/RedfishEndpoints")
-	_, body, err := api.MakeRequest(url, "GET", nil, nil)
+	_, body, err := util.MakeRequest(url, "GET", nil, nil)
 	if err != nil {
 		return fmt.Errorf("could not get endpoint: %v", err)
 	}
@@ -33,7 +33,7 @@ func GetRedfishEndpoints() error {
 
 func GetComponentEndpoint(xname string) error {
 	url := makeEndpointUrl("/Inventory/ComponentsEndpoints/" + xname)
-	res, body, err := api.MakeRequest(url, "GET", nil, nil)
+	res, body, err := util.MakeRequest(url, "GET", nil, nil)
 	if err != nil {
 		return fmt.Errorf("could not get endpoint: %v", err)
 	}
@@ -51,7 +51,7 @@ func AddRedfishEndpoint(data []byte, headers map[string]string) error {
 	// _ = ep
 	// Add redfish endpoint via POST `/hsm/v2/Inventory/RedfishEndpoints` endpoint
 	url := makeEndpointUrl("/Inventory/RedfishEndpoints")
-	res, body, _ := api.MakeRequest(url, "POST", data, headers)
+	res, body, _ := util.MakeRequest(url, "POST", data, headers)
 	fmt.Println("smd url: ", url)
 	fmt.Println("res: ", res)
 	fmt.Println("body: ", string(body))

internal/api/util.go

@@ -1,27 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net/http"
-)
-
-func MakeRequest(url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) {
-	// url := getSmdEndpointUrl(endpoint)
-	req, _ := http.NewRequest(httpMethod, url, bytes.NewBuffer(body))
-	req.Header.Add("User-Agent", "magellan")
-	for k, v := range headers {
-		req.Header.Add(k, v)
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, nil, fmt.Errorf("could not make request: %v", err)
-	}
-	b, err := io.ReadAll(res.Body)
-	res.Body.Close()
-	if err != nil {
-		return nil, nil, fmt.Errorf("could not read response body: %v", err)
-	}
-	return res, b, err
-}

internal/collect.go

@@ -8,10 +8,14 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"path"
 	"sync"
 	"time"
 
+	"github.com/bikeshack/magellan/internal/log"
+
 	"github.com/bikeshack/magellan/internal/api/smd"
+	"github.com/bikeshack/magellan/internal/util"
 
 	"github.com/Cray-HPE/hms-xname/xnames"
 	bmclib "github.com/bmc-toolbox/bmclib/v2"
@@ -50,9 +54,10 @@ type QueryParams struct {
 	CertPoolFile  string
 	Verbose       bool
 	IpmitoolPath  string
+	OutputPath    string
 }
 
-func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
+func NewClient(l *log.Logger, q *QueryParams) (*bmclib.Client, error) {
 	// NOTE: bmclib.NewClient(host, port, user, pass)
 	// ...seems like the `port` params doesn't work like expected depending on interface
 
@@ -106,7 +111,7 @@ func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) {
 	return client, nil
 }
 
-func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error {
+func CollectInfo(probeStates *[]BMCProbeResult, l *log.Logger, q *QueryParams) error {
 	// check for available probe states
 	if probeStates == nil {
 		return fmt.Errorf("no probe states found")
@@ -123,6 +128,13 @@ func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error
 		NodeBMC:       -1,
 	}
 
+	// make the output directory to store files
+	outputPath := path.Clean(q.OutputPath)
+	outputPath, err := util.MakeOutputDirectory(outputPath)
+	if err != nil {
+		l.Log.Errorf("could not make output directory: %v", err)
+	}
+
 	found := make([]string, 0, len(*probeStates))
 	done := make(chan struct{}, q.Threads+1)
 	chanProbeState := make(chan BMCProbeResult, q.Threads+1)
@@ -171,6 +183,11 @@ func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error
 				headers := make(map[string]string)
 				headers["Content-Type"] = "application/json"
 
+				// unmarshal json to send in correct format
+				var iobj, cobj map[string]json.RawMessage
+				json.Unmarshal(inventory, &iobj)
+				json.Unmarshal(chassis, &cobj)
+
 				data := make(map[string]any)
 				data["ID"] = fmt.Sprintf("%v", node.String()[:len(node.String())-2])
 				data["Type"] = ""
@@ -179,14 +196,20 @@ func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error
 				data["User"] = q.User
 				data["Password"] = q.Pass
 				data["RediscoverOnUpdate"] = false
-				data["Inventory"] = inventory
+				data["Inventory"] = iobj
-				data["Chassis"] = chassis
+				data["Chassis"] = cobj
 
 				b, err := json.MarshalIndent(data, "", "    ")
 				if err != nil {
 					l.Log.Errorf("could not marshal JSON: %v", err)
 				}
 
+				// write JSON data to file
+				err = os.WriteFile(path.Clean(outputPath + "/" + q.Host + ".json"), b, os.ModePerm)
+				if err != nil {
+					l.Log.Errorf("could not write data to file: %v", err)
+				}
+
 				// add all endpoints to smd
 				err = smd.AddRedfishEndpoint(b, headers)
 				if err != nil {
@@ -246,7 +269,7 @@ func CollectInfo(probeStates *[]BMCProbeResult, l *Logger, q *QueryParams) error
 	return nil
 }
 
-func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
+func QueryMetadata(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
 	// client, err := NewClient(l, q)
 
 	// open BMC session and update driver registry
@@ -280,7 +303,7 @@ func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, er
 	return b, nil
 }
 
-func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
+func QueryInventory(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
 	// open BMC session and update driver registry
 	ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout))
 	client.Registry.FilterForCompatible(ctx)
@@ -297,21 +320,23 @@ func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, e
 		return nil, fmt.Errorf("could not get inventory: %v", err)
 	}
 
+	
 	// retrieve inventory data
-	b, err := json.MarshalIndent(inventory, "", "    ")
+	data := map[string]any{"Inventory": inventory}
+	b, err := json.MarshalIndent(data, "", "    ")
 	if err != nil {
 		ctxCancel()
 		return nil, fmt.Errorf("could not marshal JSON: %v", err)
 	}
 
 	if q.Verbose {
-		fmt.Printf("inventory: %v\n", string(b))
+		fmt.Printf("%v\n", string(b))
 	}
 	ctxCancel()
 	return b, nil
 }
 
-func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
+func QueryPowerState(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
 	ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout))
 	client.Registry.FilterForCompatible(ctx)
 	err := client.PreferProvider(q.Preferred).Open(ctx)
@@ -321,28 +346,29 @@ func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte,
 	}
 	defer client.Close(ctx)
 
-	inventory, err := client.GetPowerState(ctx)
+	powerState, err := client.GetPowerState(ctx)
 	if err != nil {
 		ctxCancel()
 		return nil, fmt.Errorf("could not get inventory: %v", err)
 	}
 
 	// retrieve inventory data
-	b, err := json.MarshalIndent(inventory, "", "    ")
+	data := map[string]any{"PowerState": powerState}
+	b, err := json.MarshalIndent(data, "", "    ")
 	if err != nil {
 		ctxCancel()
 		return nil, fmt.Errorf("could not marshal JSON: %v", err)
 	}
 
 	if q.Verbose {
-		fmt.Printf("power state: %v\n", string(b))
+		fmt.Printf("%v\n", string(b))
 	}
 	ctxCancel()
 	return b, nil
 
 }
 
-func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
+func QueryUsers(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
 	// open BMC session and update driver registry
 	ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout))
 	client.Registry.FilterForCompatible(ctx)
@@ -361,7 +387,8 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
 	}
 
 	// retrieve inventory data
-	b, err := json.MarshalIndent(users, "", "    ")
+	data := map[string]any {"Users": users}
+	b, err := json.MarshalIndent(data, "", "    ")
 	if err != nil {
 		ctxCancel()
 		return nil, fmt.Errorf("could not marshal JSON: %v", err)
@@ -370,24 +397,24 @@ func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error
 	// return b, nil
 	ctxCancel()
 	if q.Verbose {
-		fmt.Printf("users: %v\n", string(b))
+		fmt.Printf("%v\n", string(b))
 	}
 	return b, nil
 }
 
-func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
+func QueryBios(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
 	// client, err := NewClient(l, q)
 	// if err != nil {
 	// 	return nil, fmt.Errorf("could not make query: %v", err)
 	// }
 	b, err := makeRequest(client, client.GetBiosConfiguration, q.Timeout)
 	if q.Verbose {
-		fmt.Printf("bios: %v\n", string(b))
+		fmt.Printf("%v\n", string(b))
 	}
 	return b, err
 }
 
-func QueryEthernetInterfaces(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) {
+func QueryEthernetInterfaces(client *bmclib.Client, l *log.Logger, q *QueryParams) ([]byte, error) {
 	config := gofish.ClientConfig{
 		Endpoint:            fmt.Sprintf("https://%s:%d", q.Host, q.Port),
 		Username:            q.User,
@@ -413,7 +440,6 @@ func QueryEthernetInterfaces(client *bmclib.Client, l *Logger, q *QueryParams) (
 }
 
 func QueryChassis(q *QueryParams) ([]byte, error) {
-
 	url := "https://"
 	if q.User != "" && q.Pass != "" {
 		url += fmt.Sprintf("%s:%s@", q.User, q.Pass)
@@ -436,18 +462,19 @@ func QueryChassis(q *QueryParams) ([]byte, error) {
 		return nil, fmt.Errorf("could not query chassis (%v:%v): %v", q.Host, q.Port, err)
 	}
 
-	b, err := json.MarshalIndent(chassis, "", "    ")
+	data := map[string]any{"chassis": chassis}
+	b, err := json.MarshalIndent(data, "", "    ")
 	if err != nil {
 		return nil, fmt.Errorf("could not marshal JSON: %v", err)
 	}
 
 	if q.Verbose {
-		fmt.Printf("chassis: %v\n", string(b))
+		fmt.Printf("%v\n", string(b))
 	}
 	return b, nil
 }
 
-func makeRequest[T interface{}](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) {
+func makeRequest[T any](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) {
 	ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout))
 	client.Registry.FilterForCompatible(ctx)
 	err := client.Open(ctx)
@@ -468,7 +495,7 @@ func makeRequest[T interface{}](client *bmclib.Client, fn func(context.Context)
 	return makeJson(response)
 }
 
-func makeJson(object interface{}) ([]byte, error) {
+func makeJson(object any) ([]byte, error) {
 	b, err := json.MarshalIndent(object, "", "    ")
 	if err != nil {
 		return nil, fmt.Errorf("could not marshal JSON: %v", err)

internal/log/logger.go

@@ -1,4 +1,4 @@
-package magellan
+package log
 
 import (
 	"github.com/sirupsen/logrus"

internal/util/util.go

@@ -0,0 +1,60 @@
+package util
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+)
+
+func PathExists(path string) (bool, error) {
+    _, err := os.Stat(path)
+    if err == nil { return true, nil }
+    if os.IsNotExist(err) { return false, nil }
+    return false, err
+}
+
+func MakeRequest(url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) {
+	// url := getSmdEndpointUrl(endpoint)
+	req, _ := http.NewRequest(httpMethod, url, bytes.NewBuffer(body))
+	req.Header.Add("User-Agent", "magellan")
+	for k, v := range headers {
+		req.Header.Add(k, v)
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not make request: %v", err)
+	}
+	b, err := io.ReadAll(res.Body)
+	res.Body.Close()
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not read response body: %v", err)
+	}
+	return res, b, err
+}
+
+func MakeOutputDirectory(path string) (string, error) {
+    // get the current data + time using Go's stupid formatting
+    t := time.Now()
+    dirname := t.Format("2006-01-01 15:04:05")
+    final := path + "/" + dirname
+
+	// check if path is valid and directory
+    pathExists, err := PathExists(final); 
+    if err != nil {
+        return final, fmt.Errorf("could not check for existing path: %v", err) 
+    }
+	if pathExists {
+        // make sure it is directory with 0o644 permissions
+        return final, fmt.Errorf("found existing path: %v", final)
+    }
+	
+	// create directory with data + time
+	err = os.MkdirAll(final, 0766)
+	if err != nil {
+		return final, fmt.Errorf("could not make directory: %v", err)
+	}
+	return final, nil
+}