From 2c841906b29e38ee37619a5f90dd4cdbc881f96f Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 7 Aug 2024 10:58:03 -0600 Subject: [PATCH] Updated 'cmd' package --- cmd/collect.go | 18 ++--- cmd/crawl.go | 4 +- cmd/list.go | 6 +- cmd/root.go | 13 ++-- cmd/scan.go | 199 +++++++++++++++++++++++++++++++++++-------------- cmd/update.go | 44 ++++++----- 6 files changed, 188 insertions(+), 96 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 670dca2..b2cd151 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -5,7 +5,7 @@ import ( "os/user" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/db/sqlite" + "github.com/OpenCHAMI/magellan/internal/cache/sqlite" "github.com/OpenCHAMI/magellan/internal/util" "github.com/OpenCHAMI/magellan/pkg/client" "github.com/cznic/mathutil" @@ -31,7 +31,7 @@ var collectCmd = &cobra.Command{ " magellan collect --host smd.example.com --port 27779 --username username --password password", Run: func(cmd *cobra.Command, args []string) { // get probe states stored in db from scan - scannedResults, err := sqlite.GetScannedResults(cachePath) + scannedResults, err := sqlite.GetScannedAssets(cachePath) if err != nil { log.Error().Err(err).Msgf("failed to get scanned results from cache") } @@ -53,7 +53,7 @@ var collectCmd = &cobra.Command{ if concurrency <= 0 { concurrency = mathutil.Clamp(len(scannedResults), 1, 255) } - q := &magellan.QueryParams{ + err = magellan.CollectInventory(&scannedResults, &magellan.CollectParams{ Username: username, Password: password, Timeout: timeout, @@ -63,15 +63,10 @@ var collectCmd = &cobra.Command{ OutputPath: outputPath, ForceUpdate: forceUpdate, AccessToken: accessToken, - } - err = magellan.CollectInventory(&scannedResults, q) + }) if err != nil { log.Error().Err(err).Msgf("failed to collect data") } - - // add necessary headers for final request (like token) - header := util.HTTPHeader{} - header.Authorization(q.AccessToken) }, } @@ -81,8 +76,9 @@ func init() { collectCmd.PersistentFlags().IntVarP(&client.Port, "port", "p", client.Port, "set the port to the SMD API") collectCmd.PersistentFlags().StringVar(&username, "username", "", "set the BMC user") collectCmd.PersistentFlags().StringVar(&password, "password", "", "set the BMC password") - collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "set the protocol used to query") - collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/data/", currentUser.Username+"/"), "set the path to store collection data") + collectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "set the scheme used to query") + collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "set the protocol used to query") + collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "set the path to store collection data") collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD") collectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") diff --git a/cmd/crawl.go b/cmd/crawl.go index ba94636..2df487b 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -17,7 +17,9 @@ import ( var crawlCmd = &cobra.Command{ Use: "crawl [uri]", Short: "Crawl a single BMC for inventory information", - Long: "Crawl a single BMC for inventory information\n" + + Long: "Crawl a single BMC for inventory information. This command does NOT store information" + + "store information about the scan into cache after completion. To do so, use the 'collect'" + + "command instead\n" + "\n" + "Examples:\n" + " magellan crawl https://bmc.example.com\n" + diff --git a/cmd/list.go b/cmd/list.go index e1b254d..8d788d2 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/OpenCHAMI/magellan/internal/db/sqlite" + "github.com/OpenCHAMI/magellan/internal/cache/sqlite" "github.com/rs/zerolog/log" "github.com/sirupsen/logrus" @@ -25,9 +25,9 @@ var listCmd = &cobra.Command{ " magellan list\n" + " magellan list --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { - scannedResults, err := sqlite.GetScannedResults(cachePath) + scannedResults, err := sqlite.GetScannedAssets(cachePath) if err != nil { - logrus.Errorf("failed toget probe results: %v\n", err) + logrus.Errorf("failed to get scanned assets: %v\n", err) } format = strings.ToLower(format) if format == "json" { diff --git a/cmd/root.go b/cmd/root.go index 3c00d91..09a38c1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -42,6 +42,7 @@ var ( outputPath string configPath string verbose bool + debug bool ) // The `root` command doesn't do anything on it's own except display @@ -70,9 +71,10 @@ func init() { currentUser, _ = user.Current() cobra.OnInitialize(InitializeConfig) rootCmd.PersistentFlags().IntVar(&concurrency, "concurrency", -1, "set the number of concurrent processes") - rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 30, "set the timeout") + rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 5, "set the timeout") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config file path") - rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set output verbosity") + rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set to enable/disable verbose output") + rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "set to enable/disable debug messages") rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "set the access token") rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Username+"/"), "set the scanning result cache path") @@ -101,9 +103,10 @@ func InitializeConfig() { // instead of in this file. func SetDefaults() { viper.SetDefault("threads", 1) - viper.SetDefault("timeout", 30) + viper.SetDefault("timeout", 5) viper.SetDefault("config", "") viper.SetDefault("verbose", false) + viper.SetDefault("debug", false) viper.SetDefault("cache", "/tmp/magellan/magellan.db") viper.SetDefault("scan.hosts", []string{}) viper.SetDefault("scan.ports", []int{}) @@ -115,7 +118,7 @@ func SetDefaults() { viper.SetDefault("collect.port", client.Port) viper.SetDefault("collect.user", "") viper.SetDefault("collect.pass", "") - viper.SetDefault("collect.protocol", "https") + viper.SetDefault("collect.protocol", "tcp") viper.SetDefault("collect.output", "/tmp/magellan/data/") viper.SetDefault("collect.force-update", false) viper.SetDefault("collect.ca-cert", "") @@ -124,7 +127,7 @@ func SetDefaults() { viper.SetDefault("user", "") viper.SetDefault("pass", "") viper.SetDefault("transfer-protocol", "HTTP") - viper.SetDefault("protocol", "https") + viper.SetDefault("protocol", "tcp") viper.SetDefault("firmware-url", "") viper.SetDefault("firmware-version", "") viper.SetDefault("component", "") diff --git a/cmd/scan.go b/cmd/scan.go index e5866bf..65baa16 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -6,11 +6,11 @@ import ( "net" "os" "path" - "strings" - "time" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/db/sqlite" + "github.com/OpenCHAMI/magellan/internal/cache/sqlite" + "github.com/OpenCHAMI/magellan/internal/util" + "github.com/rs/zerolog/log" "github.com/cznic/mathutil" "github.com/spf13/cobra" @@ -18,9 +18,12 @@ import ( ) var ( + scheme string subnets []string - subnetMasks []net.IP + subnetMask net.IPMask + targetHosts [][]string disableProbing bool + disableCache bool ) // The `scan` command is usually the first step to using the CLI tool. @@ -30,85 +33,165 @@ var ( // See the `ScanForAssets()` function in 'internal/scan.go' for details // related to the implementation. var scanCmd = &cobra.Command{ - Use: "scan", - Short: "Scan for BMC nodes on a network", + Use: "scan urls...", + Short: "Scan to discover BMC nodes on a network", Long: "Perform a net scan by attempting to connect to each host and port specified and getting a response. " + - "If the '--disable-probe` flag is used, the tool will not send another request to probe for available " + - "Redfish services.\n\n" + - "Example:\n" + - " magellan scan --subnet 172.16.0.0/24 --host 10.0.0.101\n" + - " magellan scan --subnet 172.16.0.0 --subnet-mask 255.255.255.0 --cache ./assets.db", + "Each host is passed *with a full URL* including the protocol and port. Additional subnets can be added " + + "by using the '--subnet' flag and providing an IP address on the subnet as well as a CIDR. If no CIDR is " + + "provided, then the subnet mask specified with the '--subnet-mask' flag will be used instead (will use " + + "default mask if not set).\n" + + "Similarly, any host provided with no port with use either the ports specified" + + "with `--port` or the default port used with each specified protocol. The default protocol is 'tcp' unless " + + "specified. The `--scheme` flag works similarly and the default value is 'https' in the host URL or with the " + + "'--protocol' flag.\n" + + "If the '--disable-probe` flag is used, the tool will not send another request to probe for available. " + + "Redfish services. This is not recommended, since the extra request makes the scan a bit more reliable " + + "for determining which hosts to collect inventory data.\n\n" + + "Examples:\n" + + // assumes host https://10.0.0.101:443 + " magellan scan 10.0.0.101\n" + + // assumes subnet using HTTPS and port 443 except for specified host + " magellan scan http://10.0.0.101:80 https://user:password@10.0.0.102:443 http://172.16.0.105:8080 --subnet 172.16.0.0/24\n" + + // assumes hosts http://10.0.0.101:8080 and http://10.0.0.102:8080 + " magellan scan 10.0.0.101 10.0.0.102 https://172.16.0.10:443 --port 8080 --protocol tcp" + + // assumes subnet using default unspecified subnet-masks + " magellan scan --subnet 10.0.0.0" + + // assumes subnet using HTTPS and port 443 with specified CIDR + " magellan scan --subnet 10.0.0.0/16" + + // assumes subnet using HTTP and port 5000 similar to 192.168.0.0/16 + " magellan scan --subnet 192.168.0.0 --protocol tcp --scheme https --port 5000 --subnet-mask 255.255.0.0" + + // assumes subnet without CIDR has a subnet-mask of 255.255.0.0 + " magellan scan --subnet 10.0.0.0/24 --subnet 172.16.0.0 --subnet-mask 255.255.0.0 --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { - var ( - hostsToScan []string - portsToScan []int - ) + // format and combine flag and positional args + targetHosts = append(targetHosts, util.FormatHostUrls(args, ports, scheme, verbose)...) + targetHosts = append(targetHosts, util.FormatHostUrls(hosts, ports, scheme, verbose)...) - // start by adding `--host` supplied to scan - if len(hosts) > 0 { - hostsToScan = hosts + // add more hosts specified with `--subnet` flag + if debug { + log.Debug().Msg("adding hosts from subnets") } - - // add hosts from `--subnets` and `--subnet-mask` - for i, subnet := range subnets { + for _, subnet := range subnets { // subnet string is empty so nothing to do here if subnet == "" { continue } - // NOTE: should we check if subnet is valid here or is it done elsewhere (maybe in GenerateHosts)? - - // no subnet masks supplied so add a default one for class C private networks - if len(subnetMasks) < i+1 { - subnetMasks = append(subnetMasks, net.IP{255, 255, 255, 0}) - } - // generate a slice of all hosts to scan from subnets - hostsToScan = append(hostsToScan, magellan.GenerateHosts(subnet, &subnetMasks[i])...) + targetHosts = append(targetHosts, magellan.GenerateHostsWithSubnet(subnet, &subnetMask, ports, scheme)...) } - // add ports to use for scanning - if len(ports) > 0 { - portsToScan = ports + // convert everything into full addresses for scanning + for _, host := range hosts { + var targets []string + for _, port := range ports { + _ = port + targets = append(targets, host) + } + targetHosts = append(targetHosts, targets) + } + + // if there are no target hosts, then there's nothing to do + if len(targetHosts) <= 0 { + log.Warn().Msg("nothing to do (no target hosts)") + return } else { - // no ports supplied so only use defaults - portsToScan = magellan.GetDefaultPorts() + if len(targetHosts[0]) <= 0 { + log.Warn().Msg("nothing to do (no target hosts)") + return + } + } + + // add default ports for hosts if none are specified with flag + if len(ports) == 0 { + if debug { + log.Debug().Msg("adding default ports") + } + ports = magellan.GetDefaultPorts() + } + + // show the parameters going into the scan + if debug { + combinedTargetHosts := []string{} + for _, targetHost := range targetHosts { + combinedTargetHosts = append(combinedTargetHosts, targetHost...) + } + c := map[string]any{ + "hosts": combinedTargetHosts, + "cache": cachePath, + "concurrency": concurrency, + "protocol": protocol, + "subnets": subnets, + "subnet-masks": subnetMask, + "cert": cacertPath, + "disable-probing": disableProbing, + "disable-caching": disableCache, + } + b, _ := json.MarshalIndent(c, "", " ") + fmt.Printf("%s", string(b)) + } + + // set the number of concurrent requests (1 request per BMC node) + // + // NOTE: The number of concurrent job is equal to the number of hosts by default. + // The max concurrent jobs cannot be greater than the number of hosts. + if concurrency <= 0 { + concurrency = len(targetHosts) + } else { + concurrency = mathutil.Clamp(len(targetHosts), 1, len(targetHosts)) } // scan and store scanned data in cache - if concurrency <= 0 { - concurrency = mathutil.Clamp(len(hostsToScan), 1, 255) + foundAssets := magellan.ScanForAssets(&magellan.ScanParams{ + TargetHosts: targetHosts, + Scheme: scheme, + Protocol: protocol, + Concurrency: concurrency, + Timeout: timeout, + DisableProbing: disableProbing, + Verbose: verbose, + Debug: debug, + }) + + if len(foundAssets) > 0 && verbose { + log.Info().Any("assets", foundAssets).Msgf("found assets from scan") } - probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, concurrency, timeout, disableProbing, verbose) - if verbose { - format = strings.ToLower(format) - if format == "json" { - b, _ := json.Marshal(probeStates) - fmt.Printf("%s\n", string(b)) - } else { - for _, r := range probeStates { - fmt.Printf("%s:%d (%s) @ %s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate)) + + if !disableCache && cachePath != "" { + // make the cache directory path if needed + err := os.MkdirAll(path.Dir(cachePath), 0755) + if err != nil { + log.Printf("failed to make cache directory: %v", err) + } + + // TODO: change this to use an extensible plugin system for storage solutions + // (i.e. something like cache.InsertScannedAssets(path, assets) which implements a Cache interface) + if len(foundAssets) > 0 { + err = sqlite.InsertScannedAssets(cachePath, foundAssets...) + if err != nil { + log.Error().Err(err).Msg("failed to write scanned assets to cache") } + if verbose { + log.Info().Msgf("saved assets to cache: %s", cachePath) + } + } else { + log.Warn().Msg("no assets found to save") } } - // make the dbpath dir if needed - err := os.MkdirAll(path.Dir(cachePath), 0766) - if err != nil { - fmt.Printf("failed tomake database directory: %v", err) - } - - sqlite.InsertProbeResults(cachePath, &probeStates) }, } func init() { - scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan") - scanCmd.Flags().IntSliceVar(&ports, "port", []int{}, "set the ports to scan") - scanCmd.Flags().StringVar(&format, "format", "", "set the output format") - scanCmd.Flags().StringSliceVar(&subnets, "subnet", []string{}, "set additional subnets") - scanCmd.Flags().IPSliceVar(&subnetMasks, "subnet-mask", []net.IP{}, "set the subnet masks to use for network (must match number of subnets)") - scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "disable probing scanned results for BMC nodes") + // scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan") + scanCmd.Flags().StringSliceVar(&hosts, "host", nil, "Add individual hosts to scan. (example: https://my.bmc.com:5000; same as using positional args)") + scanCmd.Flags().IntSliceVar(&ports, "port", nil, "Adds additional ports to scan for each host with unspecified ports.") + scanCmd.Flags().StringVar(&scheme, "scheme", "https", "Set the default scheme to use if not specified in host URI. (default is 'https')") + scanCmd.Flags().StringVar(&protocol, "protocol", "tcp", "Set the default protocol to use in scan. (default is 'tcp')") + scanCmd.Flags().StringSliceVar(&subnets, "subnet", nil, "Add additional hosts from specified subnets to scan.") + scanCmd.Flags().IPMaskVar(&subnetMask, "subnet-mask", net.IPv4Mask(255, 255, 255, 0), "Set the default subnet mask to use for with all subnets not using CIDR notation.") + scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "disable probing found assets for Redfish service(s) running on BMC nodes") + scanCmd.Flags().BoolVar(&disableCache, "disable-cache", false, "disable saving found assets to a cache database specified with 'cache' flag") viper.BindPFlag("scan.hosts", scanCmd.Flags().Lookup("host")) viper.BindPFlag("scan.ports", scanCmd.Flags().Lookup("port")) diff --git a/cmd/update.go b/cmd/update.go index 592cd0f..67bba16 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -28,21 +28,6 @@ var updateCmd = &cobra.Command{ " magellan update --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + " magellan update --status --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password", Run: func(cmd *cobra.Command, args []string) { - // set up update parameters - q := &magellan.UpdateParams{ - FirmwarePath: firmwareUrl, - FirmwareVersion: firmwareVersion, - Component: component, - TransferProtocol: transferProtocol, - QueryParams: magellan.QueryParams{ - Host: host, - Username: username, - Password: password, - Timeout: timeout, - Port: port, - }, - } - // check if required params are set if host == "" || username == "" || password == "" { log.Error().Msg("requires host, user, and pass to be set") @@ -50,7 +35,19 @@ var updateCmd = &cobra.Command{ // get status if flag is set and exit if status { - err := magellan.GetUpdateStatus(q) + err := magellan.GetUpdateStatus(&magellan.UpdateParams{ + FirmwarePath: firmwareUrl, + FirmwareVersion: firmwareVersion, + Component: component, + TransferProtocol: transferProtocol, + CollectParams: magellan.CollectParams{ + Host: host, + Username: username, + Password: password, + Timeout: timeout, + Port: port, + }, + }) if err != nil { log.Error().Err(err).Msgf("failed to get update status") } @@ -58,7 +55,19 @@ var updateCmd = &cobra.Command{ } // initiate a remote update - err := magellan.UpdateFirmwareRemote(q) + err := magellan.UpdateFirmwareRemote(&magellan.UpdateParams{ + FirmwarePath: firmwareUrl, + FirmwareVersion: firmwareVersion, + Component: component, + TransferProtocol: transferProtocol, + CollectParams: magellan.CollectParams{ + Host: host, + Username: username, + Password: password, + Timeout: timeout, + Port: port, + }, + }) if err != nil { log.Error().Err(err).Msgf("failed to update firmware") } @@ -71,7 +80,6 @@ func init() { updateCmd.Flags().StringVar(&username, "username", "", "set the BMC user") updateCmd.Flags().StringVar(&password, "password", "", "set the BMC password") updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "set the transfer protocol") - updateCmd.Flags().StringVar(&protocol, "protocol", "https", "set the Redfish protocol") updateCmd.Flags().StringVar(&firmwareUrl, "firmware.url", "", "set the path to the firmware") updateCmd.Flags().StringVar(&firmwareVersion, "firmware.version", "", "set the version of firmware to be installed") updateCmd.Flags().StringVar(&component, "component", "", "set the component to upgrade")