Updated 'cmd' package

cmd/collect.go

@@ -5,7 +5,7 @@ import (
 	"os/user"
 
 	magellan "github.com/OpenCHAMI/magellan/internal"
-	"github.com/OpenCHAMI/magellan/internal/db/sqlite"
+	"github.com/OpenCHAMI/magellan/internal/cache/sqlite"
 	"github.com/OpenCHAMI/magellan/internal/util"
 	"github.com/OpenCHAMI/magellan/pkg/client"
 	"github.com/cznic/mathutil"
@@ -31,7 +31,7 @@ var collectCmd = &cobra.Command{
 		"  magellan collect --host smd.example.com --port 27779 --username username --password password",
 	Run: func(cmd *cobra.Command, args []string) {
 		// get probe states stored in db from scan
-		scannedResults, err := sqlite.GetScannedResults(cachePath)
+		scannedResults, err := sqlite.GetScannedAssets(cachePath)
 		if err != nil {
 			log.Error().Err(err).Msgf("failed to get scanned results from cache")
 		}
@@ -53,7 +53,7 @@ var collectCmd = &cobra.Command{
 		if concurrency <= 0 {
 			concurrency = mathutil.Clamp(len(scannedResults), 1, 255)
 		}
-		q := &magellan.QueryParams{
+		err = magellan.CollectInventory(&scannedResults, &magellan.CollectParams{
 			Username:    username,
 			Password:    password,
 			Timeout:     timeout,
@@ -63,15 +63,10 @@ var collectCmd = &cobra.Command{
 			OutputPath:  outputPath,
 			ForceUpdate: forceUpdate,
 			AccessToken: accessToken,
-		}
+		})
-		err = magellan.CollectInventory(&scannedResults, q)
 		if err != nil {
 			log.Error().Err(err).Msgf("failed to collect data")
 		}
-
-		// add necessary headers for final request (like token)
-		header := util.HTTPHeader{}
-		header.Authorization(q.AccessToken)
 	},
 }
 
@@ -81,8 +76,9 @@ func init() {
 	collectCmd.PersistentFlags().IntVarP(&client.Port, "port", "p", client.Port, "set the port to the SMD API")
 	collectCmd.PersistentFlags().StringVar(&username, "username", "", "set the BMC user")
 	collectCmd.PersistentFlags().StringVar(&password, "password", "", "set the BMC password")
-	collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "set the protocol used to query")
+	collectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "set the scheme used to query")
-	collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/data/", currentUser.Username+"/"), "set the path to store collection data")
+	collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "set the protocol used to query")
+	collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "set the path to store collection data")
 	collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD")
 	collectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)")
 

cmd/crawl.go

@@ -17,7 +17,9 @@ import (
 var crawlCmd = &cobra.Command{
 	Use:   "crawl [uri]",
 	Short: "Crawl a single BMC for inventory information",
-	Long: "Crawl a single BMC for inventory information\n" +
+	Long: "Crawl a single BMC for inventory information. This command does NOT store information" +
+		"store information about the scan into cache after completion. To do so, use the 'collect'" +
+		"command instead\n" +
 		"\n" +
 		"Examples:\n" +
 		"  magellan crawl https://bmc.example.com\n" +

cmd/list.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/OpenCHAMI/magellan/internal/db/sqlite"
+	"github.com/OpenCHAMI/magellan/internal/cache/sqlite"
 	"github.com/rs/zerolog/log"
 
 	"github.com/sirupsen/logrus"
@@ -25,9 +25,9 @@ var listCmd = &cobra.Command{
 		"  magellan list\n" +
 		"  magellan list --cache ./assets.db",
 	Run: func(cmd *cobra.Command, args []string) {
-		scannedResults, err := sqlite.GetScannedResults(cachePath)
+		scannedResults, err := sqlite.GetScannedAssets(cachePath)
 		if err != nil {
-			logrus.Errorf("failed toget probe results: %v\n", err)
+			logrus.Errorf("failed to get scanned assets: %v\n", err)
 		}
 		format = strings.ToLower(format)
 		if format == "json" {

cmd/root.go

@@ -42,6 +42,7 @@ var (
 	outputPath  string
 	configPath  string
 	verbose     bool
+	debug       bool
 )
 
 // The `root` command doesn't do anything on it's own except display
@@ -70,9 +71,10 @@ func init() {
 	currentUser, _ = user.Current()
 	cobra.OnInitialize(InitializeConfig)
 	rootCmd.PersistentFlags().IntVar(&concurrency, "concurrency", -1, "set the number of concurrent processes")
-	rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 30, "set the timeout")
+	rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 5, "set the timeout")
 	rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config file path")
-	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set output verbosity")
+	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set to enable/disable verbose output")
+	rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "set to enable/disable debug messages")
 	rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "set the access token")
 	rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Username+"/"), "set the scanning result cache path")
 
@@ -101,9 +103,10 @@ func InitializeConfig() {
 // instead of in this file.
 func SetDefaults() {
 	viper.SetDefault("threads", 1)
-	viper.SetDefault("timeout", 30)
+	viper.SetDefault("timeout", 5)
 	viper.SetDefault("config", "")
 	viper.SetDefault("verbose", false)
+	viper.SetDefault("debug", false)
 	viper.SetDefault("cache", "/tmp/magellan/magellan.db")
 	viper.SetDefault("scan.hosts", []string{})
 	viper.SetDefault("scan.ports", []int{})
@@ -115,7 +118,7 @@ func SetDefaults() {
 	viper.SetDefault("collect.port", client.Port)
 	viper.SetDefault("collect.user", "")
 	viper.SetDefault("collect.pass", "")
-	viper.SetDefault("collect.protocol", "https")
+	viper.SetDefault("collect.protocol", "tcp")
 	viper.SetDefault("collect.output", "/tmp/magellan/data/")
 	viper.SetDefault("collect.force-update", false)
 	viper.SetDefault("collect.ca-cert", "")
@@ -124,7 +127,7 @@ func SetDefaults() {
 	viper.SetDefault("user", "")
 	viper.SetDefault("pass", "")
 	viper.SetDefault("transfer-protocol", "HTTP")
-	viper.SetDefault("protocol", "https")
+	viper.SetDefault("protocol", "tcp")
 	viper.SetDefault("firmware-url", "")
 	viper.SetDefault("firmware-version", "")
 	viper.SetDefault("component", "")

cmd/scan.go

@@ -6,11 +6,11 @@ import (
 	"net"
 	"os"
 	"path"
-	"strings"
-	"time"
 
 	magellan "github.com/OpenCHAMI/magellan/internal"
-	"github.com/OpenCHAMI/magellan/internal/db/sqlite"
+	"github.com/OpenCHAMI/magellan/internal/cache/sqlite"
+	"github.com/OpenCHAMI/magellan/internal/util"
+	"github.com/rs/zerolog/log"
 
 	"github.com/cznic/mathutil"
 	"github.com/spf13/cobra"
@@ -18,9 +18,12 @@ import (
 )
 
 var (
+	scheme         string
 	subnets        []string
-	subnetMasks    []net.IP
+	subnetMask     net.IPMask
+	targetHosts    [][]string
 	disableProbing bool
+	disableCache   bool
 )
 
 // The `scan` command is usually the first step to using the CLI tool.
@@ -30,85 +33,165 @@ var (
 // See the `ScanForAssets()` function in 'internal/scan.go' for details
 // related to the implementation.
 var scanCmd = &cobra.Command{
-	Use:   "scan",
+	Use:   "scan urls...",
-	Short: "Scan for BMC nodes on a network",
+	Short: "Scan to discover BMC nodes on a network",
 	Long: "Perform a net scan by attempting to connect to each host and port specified and getting a response. " +
-		"If the '--disable-probe` flag is used, the tool will not send another request to probe for available " +
+		"Each host is passed *with a full URL* including the protocol and port. Additional subnets can be added " +
-		"Redfish services.\n\n" +
+		"by using the '--subnet' flag and providing an IP address on the subnet as well as a CIDR. If no CIDR is " +
-		"Example:\n" +
+		"provided, then the subnet mask specified with the '--subnet-mask' flag will be used instead (will use " +
-		"  magellan scan --subnet 172.16.0.0/24 --host 10.0.0.101\n" +
+		"default mask if not set).\n" +
-		"  magellan scan --subnet 172.16.0.0 --subnet-mask 255.255.255.0 --cache ./assets.db",
+		"Similarly, any host provided with no port with use either the ports specified" +
+		"with `--port` or the default port used with each specified protocol. The default protocol is 'tcp' unless " +
+		"specified. The `--scheme` flag works similarly and the default value is 'https' in the host URL or with the " +
+		"'--protocol' flag.\n" +
+		"If the '--disable-probe` flag is used, the tool will not send another request to probe for available. " +
+		"Redfish services. This is not recommended, since the extra request makes the scan a bit more reliable " +
+		"for determining which hosts to collect inventory data.\n\n" +
+		"Examples:\n" +
+		// assumes host https://10.0.0.101:443
+		"  magellan scan 10.0.0.101\n" +
+		// assumes subnet using HTTPS and port 443 except for specified host
+		"  magellan scan http://10.0.0.101:80 https://user:password@10.0.0.102:443 http://172.16.0.105:8080 --subnet 172.16.0.0/24\n" +
+		// assumes hosts http://10.0.0.101:8080 and http://10.0.0.102:8080
+		"  magellan scan 10.0.0.101 10.0.0.102 https://172.16.0.10:443 --port 8080 --protocol tcp" +
+		// assumes subnet using default unspecified subnet-masks
+		"  magellan scan --subnet 10.0.0.0" +
+		// assumes subnet using HTTPS and port 443 with specified CIDR
+		"  magellan scan --subnet 10.0.0.0/16" +
+		// assumes subnet using HTTP and port 5000 similar to 192.168.0.0/16
+		"  magellan scan --subnet 192.168.0.0 --protocol tcp --scheme https --port 5000 --subnet-mask 255.255.0.0" +
+		// assumes subnet without CIDR has a subnet-mask of 255.255.0.0
+		"  magellan scan --subnet 10.0.0.0/24 --subnet 172.16.0.0 --subnet-mask 255.255.0.0 --cache ./assets.db",
 	Run: func(cmd *cobra.Command, args []string) {
-		var (
+		// format and combine flag and positional args
-			hostsToScan []string
+		targetHosts = append(targetHosts, util.FormatHostUrls(args, ports, scheme, verbose)...)
-			portsToScan []int
+		targetHosts = append(targetHosts, util.FormatHostUrls(hosts, ports, scheme, verbose)...)
-		)
 
-		// start by adding `--host` supplied to scan
+		// add more hosts specified with `--subnet` flag
-		if len(hosts) > 0 {
+		if debug {
-			hostsToScan = hosts
+			log.Debug().Msg("adding hosts from subnets")
 		}
-
+		for _, subnet := range subnets {
-		// add hosts from `--subnets` and `--subnet-mask`
-		for i, subnet := range subnets {
 			// subnet string is empty so nothing to do here
 			if subnet == "" {
 				continue
 			}
 
-			// NOTE: should we check if subnet is valid here or is it done elsewhere (maybe in GenerateHosts)?
-
-			// no subnet masks supplied so add a default one for class C private networks
-			if len(subnetMasks) < i+1 {
-				subnetMasks = append(subnetMasks, net.IP{255, 255, 255, 0})
-			}
-
 			// generate a slice of all hosts to scan from subnets
-			hostsToScan = append(hostsToScan, magellan.GenerateHosts(subnet, &subnetMasks[i])...)
+			targetHosts = append(targetHosts, magellan.GenerateHostsWithSubnet(subnet, &subnetMask, ports, scheme)...)
 		}
 
-		// add ports to use for scanning
+		// convert everything into full addresses for scanning
-		if len(ports) > 0 {
+		for _, host := range hosts {
-			portsToScan = ports
+			var targets []string
+			for _, port := range ports {
+				_ = port
+				targets = append(targets, host)
+			}
+			targetHosts = append(targetHosts, targets)
+		}
+
+		// if there are no target hosts, then there's nothing to do
+		if len(targetHosts) <= 0 {
+			log.Warn().Msg("nothing to do (no target hosts)")
+			return
 		} else {
-			// no ports supplied so only use defaults
+			if len(targetHosts[0]) <= 0 {
-			portsToScan = magellan.GetDefaultPorts()
+				log.Warn().Msg("nothing to do (no target hosts)")
+				return
+			}
+		}
+
+		// add default ports for hosts if none are specified with flag
+		if len(ports) == 0 {
+			if debug {
+				log.Debug().Msg("adding default ports")
+			}
+			ports = magellan.GetDefaultPorts()
+		}
+
+		// show the parameters going into the scan
+		if debug {
+			combinedTargetHosts := []string{}
+			for _, targetHost := range targetHosts {
+				combinedTargetHosts = append(combinedTargetHosts, targetHost...)
+			}
+			c := map[string]any{
+				"hosts":           combinedTargetHosts,
+				"cache":           cachePath,
+				"concurrency":     concurrency,
+				"protocol":        protocol,
+				"subnets":         subnets,
+				"subnet-masks":    subnetMask,
+				"cert":            cacertPath,
+				"disable-probing": disableProbing,
+				"disable-caching": disableCache,
+			}
+			b, _ := json.MarshalIndent(c, "", "    ")
+			fmt.Printf("%s", string(b))
+		}
+
+		// set the number of concurrent requests (1 request per BMC node)
+		//
+		// NOTE: The number of concurrent job is equal to the number of hosts by default.
+		// The max concurrent jobs cannot be greater than the number of hosts.
+		if concurrency <= 0 {
+			concurrency = len(targetHosts)
+		} else {
+			concurrency = mathutil.Clamp(len(targetHosts), 1, len(targetHosts))
 		}
 
 		// scan and store scanned data in cache
-		if concurrency <= 0 {
+		foundAssets := magellan.ScanForAssets(&magellan.ScanParams{
-			concurrency = mathutil.Clamp(len(hostsToScan), 1, 255)
+			TargetHosts:    targetHosts,
+			Scheme:         scheme,
+			Protocol:       protocol,
+			Concurrency:    concurrency,
+			Timeout:        timeout,
+			DisableProbing: disableProbing,
+			Verbose:        verbose,
+			Debug:          debug,
+		})
+
+		if len(foundAssets) > 0 && verbose {
+			log.Info().Any("assets", foundAssets).Msgf("found assets from scan")
 		}
-		probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, concurrency, timeout, disableProbing, verbose)
+
-		if verbose {
+		if !disableCache && cachePath != "" {
-			format = strings.ToLower(format)
+			// make the cache directory path if needed
-			if format == "json" {
+			err := os.MkdirAll(path.Dir(cachePath), 0755)
-				b, _ := json.Marshal(probeStates)
+			if err != nil {
-				fmt.Printf("%s\n", string(b))
+				log.Printf("failed to make cache directory: %v", err)
-			} else {
+			}
-				for _, r := range probeStates {
+
-					fmt.Printf("%s:%d (%s) @ %s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate))
+			// TODO: change this to use an extensible plugin system for storage solutions
+			// (i.e. something like cache.InsertScannedAssets(path, assets) which implements a Cache interface)
+			if len(foundAssets) > 0 {
+				err = sqlite.InsertScannedAssets(cachePath, foundAssets...)
+				if err != nil {
+					log.Error().Err(err).Msg("failed to write scanned assets to cache")
 				}
+				if verbose {
+					log.Info().Msgf("saved assets to cache: %s", cachePath)
+				}
+			} else {
+				log.Warn().Msg("no assets found to save")
 			}
 		}
 
-		// make the dbpath dir if needed
-		err := os.MkdirAll(path.Dir(cachePath), 0766)
-		if err != nil {
-			fmt.Printf("failed tomake database directory: %v", err)
-		}
-
-		sqlite.InsertProbeResults(cachePath, &probeStates)
 	},
 }
 
 func init() {
-	scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan")
+	// scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan")
-	scanCmd.Flags().IntSliceVar(&ports, "port", []int{}, "set the ports to scan")
+	scanCmd.Flags().StringSliceVar(&hosts, "host", nil, "Add individual hosts to scan. (example: https://my.bmc.com:5000; same as using positional args)")
-	scanCmd.Flags().StringVar(&format, "format", "", "set the output format")
+	scanCmd.Flags().IntSliceVar(&ports, "port", nil, "Adds additional ports to scan for each host with unspecified ports.")
-	scanCmd.Flags().StringSliceVar(&subnets, "subnet", []string{}, "set additional subnets")
+	scanCmd.Flags().StringVar(&scheme, "scheme", "https", "Set the default scheme to use if not specified in host URI. (default is 'https')")
-	scanCmd.Flags().IPSliceVar(&subnetMasks, "subnet-mask", []net.IP{}, "set the subnet masks to use for network (must match number of subnets)")
+	scanCmd.Flags().StringVar(&protocol, "protocol", "tcp", "Set the default protocol to use in scan. (default is 'tcp')")
-	scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "disable probing scanned results for BMC nodes")
+	scanCmd.Flags().StringSliceVar(&subnets, "subnet", nil, "Add additional hosts from specified subnets to scan.")
+	scanCmd.Flags().IPMaskVar(&subnetMask, "subnet-mask", net.IPv4Mask(255, 255, 255, 0), "Set the default subnet mask to use for with all subnets not using CIDR notation.")
+	scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "disable probing found assets for Redfish service(s) running on BMC nodes")
+	scanCmd.Flags().BoolVar(&disableCache, "disable-cache", false, "disable saving found assets to a cache database specified with 'cache' flag")
 
 	viper.BindPFlag("scan.hosts", scanCmd.Flags().Lookup("host"))
 	viper.BindPFlag("scan.ports", scanCmd.Flags().Lookup("port"))

cmd/update.go

@@ -28,21 +28,6 @@ var updateCmd = &cobra.Command{
 		"  magellan update --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" +
 		"  magellan update --status --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password",
 	Run: func(cmd *cobra.Command, args []string) {
-		// set up update parameters
-		q := &magellan.UpdateParams{
-			FirmwarePath:     firmwareUrl,
-			FirmwareVersion:  firmwareVersion,
-			Component:        component,
-			TransferProtocol: transferProtocol,
-			QueryParams: magellan.QueryParams{
-				Host:     host,
-				Username: username,
-				Password: password,
-				Timeout:  timeout,
-				Port:     port,
-			},
-		}
-
 		// check if required params are set
 		if host == "" || username == "" || password == "" {
 			log.Error().Msg("requires host, user, and pass to be set")
@@ -50,7 +35,19 @@ var updateCmd = &cobra.Command{
 
 		// get status if flag is set and exit
 		if status {
-			err := magellan.GetUpdateStatus(q)
+			err := magellan.GetUpdateStatus(&magellan.UpdateParams{
+				FirmwarePath:     firmwareUrl,
+				FirmwareVersion:  firmwareVersion,
+				Component:        component,
+				TransferProtocol: transferProtocol,
+				CollectParams: magellan.CollectParams{
+					Host:     host,
+					Username: username,
+					Password: password,
+					Timeout:  timeout,
+					Port:     port,
+				},
+			})
 			if err != nil {
 				log.Error().Err(err).Msgf("failed to get update status")
 			}
@@ -58,7 +55,19 @@ var updateCmd = &cobra.Command{
 		}
 
 		// initiate a remote update
-		err := magellan.UpdateFirmwareRemote(q)
+		err := magellan.UpdateFirmwareRemote(&magellan.UpdateParams{
+			FirmwarePath:     firmwareUrl,
+			FirmwareVersion:  firmwareVersion,
+			Component:        component,
+			TransferProtocol: transferProtocol,
+			CollectParams: magellan.CollectParams{
+				Host:     host,
+				Username: username,
+				Password: password,
+				Timeout:  timeout,
+				Port:     port,
+			},
+		})
 		if err != nil {
 			log.Error().Err(err).Msgf("failed to update firmware")
 		}
@@ -71,7 +80,6 @@ func init() {
 	updateCmd.Flags().StringVar(&username, "username", "", "set the BMC user")
 	updateCmd.Flags().StringVar(&password, "password", "", "set the BMC password")
 	updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "set the transfer protocol")
-	updateCmd.Flags().StringVar(&protocol, "protocol", "https", "set the Redfish protocol")
 	updateCmd.Flags().StringVar(&firmwareUrl, "firmware.url", "", "set the path to the firmware")
 	updateCmd.Flags().StringVar(&firmwareVersion, "firmware.version", "", "set the version of firmware to be installed")
 	updateCmd.Flags().StringVar(&component, "component", "", "set the component to upgrade")