From e93f49eb82e1eb86c37d60606bba70f12efc2cb9 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 9 May 2024 13:18:38 -0600 Subject: [PATCH 01/15] Added more output for debugging --- cmd/scan.go | 2 +- internal/collect.go | 10 ++++++++-- internal/scan.go | 17 ++++++++++++----- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/cmd/scan.go b/cmd/scan.go index ca2fdd0..50534a2 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -57,7 +57,7 @@ var scanCmd = &cobra.Command{ if threads <= 0 { threads = mathutil.Clamp(len(hostsToScan), 1, 255) } - probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, threads, timeout, disableProbing) + probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, threads, timeout, disableProbing, verbose) if verbose { for _, r := range probeStates { fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol) diff --git a/internal/collect.go b/internal/collect.go index d97b6e6..9eff14f 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -125,7 +125,10 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err l.Log.Errorf("failed to query chassis: %v", err) continue } - json.Unmarshal(chassis, &rm) + err = json.Unmarshal(chassis, &rm) + if err != nil { + l.Log.Errorf("failed to unmarshal chassis JSON: %v", err) + } data["Chassis"] = rm["Chassis"] // systems @@ -133,7 +136,10 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err if err != nil { l.Log.Errorf("failed to query systems: %v", err) } - json.Unmarshal(systems, &rm) + err = json.Unmarshal(systems, &rm) + if err != nil { + l.Log.Errorf("failed to unmarshal system JSON: %v", err) + } data["Systems"] = rm["Systems"] // add other fields from systems diff --git a/internal/scan.go b/internal/scan.go index 2b8c14e..4ff6fc4 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -95,11 +95,12 @@ func generateHosts(ip *net.IP, mask *net.IPMask) []string { return hosts } -func ScanForAssets(hosts []string, ports []int, threads int, timeout int, disableProbing bool) []ScannedResult { - results := make([]ScannedResult, 0, len(hosts)) - done := make(chan struct{}, threads+1) - chanHost := make(chan string, threads+1) - // chanPort := make(chan int, threads+1) +func ScanForAssets(hosts []string, ports []int, threads int, timeout int, disableProbing bool, verbose bool) []ScannedResult { + var ( + results = make([]ScannedResult, 0, len(hosts)) + done = make(chan struct{}, threads+1) + chanHost = make(chan string, threads+1) + ) var wg sync.WaitGroup wg.Add(threads) @@ -118,8 +119,14 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int, disabl url := fmt.Sprintf("https://%s:%d/redfish/v1/", result.Host, result.Port) res, _, err := util.MakeRequest(nil, url, "GET", nil, nil) if err != nil || res == nil { + if verbose { + fmt.Printf("failed to make request: %v\n", err) + } continue } else if res.StatusCode != http.StatusOK { + if verbose { + fmt.Printf("request returned code: %v\n", res.StatusCode) + } continue } else { probeResults = append(probeResults, result) From b6d429f072710e6a5a756a8b32f8b1fb085a65f7 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 9 May 2024 14:53:44 -0600 Subject: [PATCH 02/15] Changed the default storage location for scanning cache --- cmd/collect.go | 8 ++++---- cmd/root.go | 9 ++++++--- cmd/update.go | 10 +++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 0dccd70..bcefa56 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -48,8 +48,8 @@ var collectCmd = &cobra.Command{ threads = mathutil.Clamp(len(probeStates), 1, 255) } q := &magellan.QueryParams{ - User: user, - Pass: pass, + User: username, + Pass: password, Protocol: protocol, Drivers: drivers, Preferred: preferredDriver, @@ -75,8 +75,8 @@ func init() { collectCmd.PersistentFlags().StringSliceVar(&drivers, "driver", []string{"redfish"}, "set the driver(s) and fallback drivers to use") collectCmd.PersistentFlags().StringVar(&smd.Host, "host", smd.Host, "set the host to the smd API") collectCmd.PersistentFlags().IntVarP(&smd.Port, "port", "p", smd.Port, "set the port to the smd API") - collectCmd.PersistentFlags().StringVar(&user, "user", "", "set the BMC user") - collectCmd.PersistentFlags().StringVar(&pass, "pass", "", "set the BMC password") + collectCmd.PersistentFlags().StringVar(&username, "user", "", "set the BMC user") + collectCmd.PersistentFlags().StringVar(&password, "pass", "", "set the BMC password") collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "set the protocol used to query") collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", "/tmp/magellan/data/", "set the path to store collection data") collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD ") diff --git a/cmd/root.go b/cmd/root.go index bb38247..cccc69a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "os" + "os/user" magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/api/smd" @@ -19,8 +20,8 @@ var ( hosts []string protocol string cacertPath string - user string - pass string + username string + password string dbpath string drivers []string preferredDriver string @@ -76,13 +77,15 @@ func LoadAccessToken() (string, error) { } func init() { + // get the current user + currentUser, _ := user.Current() cobra.OnInitialize(InitializeConfig) rootCmd.PersistentFlags().IntVar(&threads, "threads", -1, "set the number of threads") rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 30, "set the timeout") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config file path") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set verbose flag") rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "set the access token") - rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", "/tmp/magellan/magellan.db", "set the probe storage path") + rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Name+"/"), "set the probe storage path") // bind viper config flags with cobra viper.BindPFlag("threads", rootCmd.Flags().Lookup("threads")) diff --git a/cmd/update.go b/cmd/update.go index c099fe0..0a124d9 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -33,15 +33,15 @@ var updateCmd = &cobra.Command{ Preferred: "redfish", Protocol: protocol, Host: host, - User: user, - Pass: pass, + User: username, + Pass: password, Timeout: timeout, Port: port, }, } // check if required params are set - if host == "" || user == "" || pass == "" { + if host == "" || username == "" || password == "" { l.Log.Fatal("requires host, user, and pass to be set") } @@ -69,8 +69,8 @@ var updateCmd = &cobra.Command{ func init() { updateCmd.Flags().StringVar(&host, "bmc-host", "", "set the BMC host") updateCmd.Flags().IntVar(&port, "bmc-port", 443, "set the BMC port") - updateCmd.Flags().StringVar(&user, "user", "", "set the BMC user") - updateCmd.Flags().StringVar(&pass, "pass", "", "set the BMC password") + updateCmd.Flags().StringVar(&username, "user", "", "set the BMC user") + updateCmd.Flags().StringVar(&password, "pass", "", "set the BMC password") updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "set the transfer protocol") updateCmd.Flags().StringVar(&protocol, "protocol", "https", "set the Redfish protocol") updateCmd.Flags().StringVar(&firmwareUrl, "firmware-url", "", "set the path to the firmware") From ef82a5c45d9fe442534e5235f522c1a35e171314 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 9 May 2024 15:47:18 -0600 Subject: [PATCH 03/15] Updated go files --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b6f4fd5..703858f 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.17.0 - github.com/stmcginnis/gofish v0.14.0 + github.com/stmcginnis/gofish v0.17.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) diff --git a/go.sum b/go.sum index 8b63fe3..12da651 100644 --- a/go.sum +++ b/go.sum @@ -228,6 +228,8 @@ github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= github.com/stmcginnis/gofish v0.14.0 h1:geECNAiG33JDB2x2xDkerpOOuXFqxp5YP3EFE3vd5iM= github.com/stmcginnis/gofish v0.14.0/go.mod h1:BLDSFTp8pDlf/xDbLZa+F7f7eW0E/CHCboggsu8CznI= +github.com/stmcginnis/gofish v0.17.0 h1:KWpxf3arkfxBFuCi01e1UYoII8UW1RmSW2ugh7f6ULk= +github.com/stmcginnis/gofish v0.17.0/go.mod h1:BLDSFTp8pDlf/xDbLZa+F7f7eW0E/CHCboggsu8CznI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= From fba9b547ee081c3e16beeda85074c78c137f33dd Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 10:07:54 -0600 Subject: [PATCH 04/15] Updated example config --- config.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/config.yaml b/config.yaml index 33808c3..455fe0d 100644 --- a/config.yaml +++ b/config.yaml @@ -17,13 +17,7 @@ collect: output: "/tmp/magellan/data/" threads: 1 force-update: false - preferred-driver: "redfish" - secure-tls: false - cert-pool: - drivers: - - "redfish" - ipmitool: - path: "/usr/bin/ipmitool" + ca-cert: "cacert.pem" update: bmc-host: bmc-port: 443 @@ -36,7 +30,7 @@ update: component: secure-tls: false status: false -threads: 1 +concurrency: 1 timeout: 30 verbose: true db: From ebee86ccbef8ab88fab475b745c467471a6d5dbc Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 10:08:26 -0600 Subject: [PATCH 05/15] Updated README.md (WIP) --- README.md | 205 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 146 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index d8f8d06..8a06117 100644 --- a/README.md +++ b/README.md @@ -1,88 +1,155 @@ # Magellan -Magellan is a board management controller discovery tool designed to scan a network -and collect information about a BMC node and load that data into an -[`hms-smd`](https://github.com/bikeshack/smd/tree/master) instance. +The `magellan` CLI tool is a Redfish-based, board management controller (BMC) discovery tool designed to scan networks and is written in Go. The tool collects information from BMC nodes using the provided Redfish RESTful API with [`gofish`](https://github.com/stmcginnis/gofish) and loads the queried data into an [SMD](https://github.com/OpenCHAMI/smd/tree/master) instance. The tool strives to be more flexible by implementing multiple methods of discovery to work for a wider range of systems (WIP) and is capable of using independently of other tools or services. -## How It Works +## Getting Started -Magellan is designed to do three things: - -1. Scan for BMC nodes in cluster available on a network -2. Query information about each BMC node -3. Store queried information into a database - -Magellan first tries to probe for specified hosts using the [`dora`](https://github.com/bmc-toolbox/dora) -API. If that fails, it then tries to use its own built-in, simpler scanner as a fallback. -This is done by sending a raw TCP request to a number of potential hosts over a -network, and noting which requests are successful. At this point, `magellan` sees -no difference between a services. - -Next, it tries to query information about the BMC node using `bmclib` functions, -but requires access to a redfish interface on the node to work. Once the BMC -information is received, it is then stored into `hms-smd` using its API. - -In summary, `magellan` needs at minimum the following configured to work on each node: - -1. Available redfish interface with its host and port -2. A running instance of `hms-smd` with its host and port -3. Additional dependencies for `bmclib` such as `ipmitool` +[Build](#building) and [run on bare metal](#running-the-tool) or run and test with Docker using the [latest prebuilt image](#running-with-docker). ## Building -Install Go, clone the repo, and then run the following in the project root: +The `magellan` tool can be built to run on bare metal. Install the required Go tools, clone the repo, and then build the binary in the root directory with the following: ```bash -git clone https://github.com/bikeshack/magellan +git clone https://github.com/OpenCHAMI/magellan cd magellan go mod tidy && go build ``` -This should find and download all of the required dependencies. Although other -versions of Go may work, the project has only been tested with v1.20. +And that's it. The last line should find and download all of the required dependencies to build the project. Although other versions of Go may work, the project has been tested to work with versions v1.20 and later on MacOS and Linux. -To build the Docker container, run `docker build -t magellan:latest .` in the -project's directory. +### Docker + +The tool can also run using Docker. To build the Docker container, run `docker build -t magellan:testing .` in the project's directory. This is useful if you to run `magellan` on a different system through Docker desktop without having to install and build with Go (or if you can't do so for some reason). [Prebuilt images](https://github.com/OpenCHAMI/magellan/pkgs/container/magellan) are available as well on `ghcr`. Images can be pulled directly from the repository: + +```bash +docker pull ghcr.io/openchami/magellan:latest +``` + +See the ["Running with Docker"](#running-with-docker) section below about running with the Docker container. ## Usage -There are three main commands to use with the tool: `scan`, `list`, and `collect`. -To scan a network for BMC nodes, use the `scan` command. If the port is not specified, -`magellan` will probe ports 623, 442 (redfish and IPMI) by default: +The sections below assume that the BMC nodes have an IP address available to query Redfish. Currently, `magellan` does not support discovery with MAC addresses although that may change in the future. + +### Checking for Redfish + +Before using the tool, confirm that the identified node has Redfish with `curl`. Assuming the IP address for the BMC node is `172.16.0.10`, we can send a request to see if it we get a response. You might need to pass the `-k` flag if the node uses TLS or point to the appropriate certificate. ```bash -./magellan scan --subnet 192.168.0.0 --db.path data/assets.db --port 623 +curl -k https://172.16.0.10/redfish/v1 --cacert cacert.pem | jq ``` -This will scan the `192.168.0.0` subnet returning the host and port that return a response -and store the results in database with path `data/assets.db`. Additional flags can -be set such as `host` to add more hosts to scan not included on the subnet, `timeout` to set how long -to wait for a response from the BMC node, or `threads` to set the number of requests -to make concurrently. Try using `./magellan help scan` for a complete set of options. +This should return a JSON response with general information. The output below has been truncated: -To see the available BMC nodes found from the scan, use the `list` command. Make -sure to point to the same database used before: +```json +{ + "@odata.context": "/redfish/v1/$metadata#ServiceRoot.ServiceRoot", + "@odata.etag": "W/\"1715279084\"", + "@odata.id": "/redfish/v1/", + "@odata.type": "#ServiceRoot.v1_5_2.ServiceRoot", + "AccountService": { + "@odata.id": "/redfish/v1/AccountService" + }, + "CertificateService": { + "@odata.id": "/redfish/v1/CertificateService" + }, + "Chassis": { + "@odata.id": "/redfish/v1/Chassis" + }, + ... +} +``` + +To see all of the available commands, run `magellan` with the `help` subcommand: + +```bash +./magellan help +Tool for BMC discovery + +Usage: + magellan [flags] + magellan [command] + +Available Commands: + collect Query information about BMC + completion Generate the autocompletion script for the specified shell + help Help about any command + list List information from scan + login Log in with identity provider for access token + scan Scan for BMC nodes on a network + update Update BMC node firmware + +Flags: + --access-token string set the access token + -c, --config string set the config file path + --db.path string set the probe storage path (default "/tmp/magellan/magellan.db") + -h, --help help for magellan + --threads int set the number of threads (default -1) + --timeout int set the timeout (default 30) + -v, --verbose set verbose flag + +Use "magellan [command] --help" for more information about a command. +``` + +### Running the Tool + +There are three main commands to use with the tool: `scan`, `list`, and `collect`. To start a network scan for BMC nodes, use the `scan` command. If the port is not specified, `magellan` will probe ports 623 and 443 by default: + +```bash +./magellan scan \ + --subnet 172.16.0.0 \ + --subnet-mask 255.255.255.0 \ + --db.path data/assets.db --port 443 +``` + +This will scan the `172.16.0.0` subnet returning the host and port that return a response and store the results in a local cache with at the `data/assets.db` path. Additional flags can be set such as `host` to add more hosts to scan not included on the subnet, `timeout` to set how long to wait for a response from the BMC node, or `threads` to set the number of requests to make concurrently. Try using `./magellan help scan` for a complete set of options this subcommand. + +To inspect the cache, use the `list` command. Make sure to point to the same database used before: ```bash ./magellan list --db.path data/assets.db ``` -This will print a list of IP address and ports found and stored from the scan. -Finally, run the `collect` command to store BMC info into `hms-smd`: +This will print a list of node info found and stored from the scan. + +Finally, set the `MAGELLAN_ACCESS_TOKEN`run the `collect` command to query the node from cache and send the info to be stored into SMD: ```bash -./magellan collect --db.path data/assets.db --driver ipmi --timeout 5 --user admin --pass password +./magellan collect \ + --db.path data/assets.db \ + --timeout 5 \ + --user admin \ + --pass password \ + --host https://example.openchami.cluster \ + --port 27779 \ + --ca-cert cacert.pem ``` -This uses the info store in the database above to request information about each -BMC node if possible. It uses the driver specified by the `driver` flag which is -passed to and set in `bmclib`. Like with the scan, the time to wait for a response -can be set with the `timeout` flag as well. This command also requires the `user` -and `pass/password` flag to be set to use `ipmitool` (which must installed as well). -Additionally, it may be necessary to set the `host` and `port` flags for `magellan` -to find the `hms-smd` API. +This uses the info stored in cache to request information about each BMC node if possible. Like with the scan, the time to wait for a response can be set with the `timeout` flag as well. This command also requires the `user` and `pass/password` flag to be set if access the Redfish service requires basic authentication. Additionally, it may be necessary to set the `host` and `port` flags for `magellan` to find the SMD API (not the root API endpoint /hsm/v2). -Note: If the `db.path` flag is not set, `magellan` will use /tmp/magellan.db by default. +Note: If the `db.path` flag is not set, `magellan` will use "/tmp/$USER/magellan.db" by default. + +### Getting an Access Token (WIP) + +The `magellan` tool has a `login` subcommand that works with the [`opaal`](https://github.com/OpenCHAMI/opaal) service to obtain a token needed to access the SMD service. If the SMD instance requires authentication, set the `MAGELLAN_ACCESS_TOKEN` environment variable to have `magellan` include it in the header for HTTP requests to SMD. + +```bash +./magellan login --url https://opaal:4444/login + +# ...complete login flow +export MAGELLAN_ACCESS_TOKEN= +``` + +Alternatively, if you are running the OpenCHAMI quickstart, you can run the provided script to generate a token and set the environment variable that way. + +```bash +quickstart_dir=path/to/deployment/recipes/quickstart +source $quickstart_dir/bash_functions.sh +export MAGELLAN_ACCESS_TOKEN=$(gen_access_token) +``` + +### Running with Docker Both the `scan` and `collect` commands can be ran via Docker after pulling the image: @@ -91,17 +158,37 @@ docker pull bikeshack/magellan:latest docker run bikeshack/magellan:latest /magellan.sh --scan "--subnet 172.16.0.0 --port 443 --timeout 3" --collect "--user admin --pass password --host http://vm01 --port 27779" ``` +## How It Works + +At its core, `magellan` is designed to do three basic things: + +1. Scan for BMC nodes in cluster available on a network +2. Query information about each BMC node through Redfish API +3. Store queried information into a system management database + +First, the tool performs a scan to find running services on a network. This is done by sending a raw TCP packet to all specified hosts (either IP or host name) and taking note which services respond. At this point, `magellan` has no way of knowing whether this is a Redfish service or not, so another HTTP request is made to verify. Once the BMC responds with an OK status code, `magellan` will store the necessary information in a local cache database to allow collecting more information about the node later. This allows for users to only have to scan their cluster once to find systems that are currently available and scannable. + +Next, the tool queries information about the BMC node using `gofish` API functions, but requires access to BMC node found in the scanning step mentioned above to work. If the node requires basic authentication, a user name and password is required to be supplied as well. Once the BMC information is retrived from each node, the info is aggregated and a HTTP request is made to a SMD instance to be stored. Optionally, the information can be written to disk for inspection and debugging purposes. + +In summary, `magellan` needs at minimum the following configured to work on each node: + +1. Available Redfish service with its known host and port +2. A running instance of SMD service with its known host and port +3. Docker to pull and run containers or Go to build binaries + ## TODO -List of things left to fix, do, or ideas... +See the [issue list](https://github.com/OpenCHAMI/magellan/issues) for plans for `magellan`. Here is a list of other features left to add, fix, or do (and some ideas!): -* [ ] Switch to internal scanner if `dora` fails -* [ ] Set default port automatically depending on the driver used to scan -* [X] Test using different `bmclib` supported drivers (mainly 'redfish') -* [X] Confirm loading different components into `hms-smd` +* [X] Confirm loading different components into SMD * [X] Add ability to set subnet mask for scanning +* [ ] Add ability to scan with other protocols like LLDP +* [ ] Add more debugging messages with the `-v/--verbose` flag +* [ ] Separate `collect` subcommand with making request to endpoint +* [X] Support logging in with `opaal` to get access token +* [X] Support using CA certificates with HTTP requests to SMD * [ ] Add unit tests for `scan`, `list`, and `collect` commands -* [X] Clean up, remove unused, and tidy code +* [ ] Clean up, remove unused, and tidy code ## Copyright From 9663b677ba986efb36bde939afba55679226f6fe Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 10:09:37 -0600 Subject: [PATCH 06/15] Removed and updated flags --- cmd/collect.go | 27 +++++++++++---------------- cmd/root.go | 44 ++++++++++++++++++++------------------------ cmd/scan.go | 6 +++--- 3 files changed, 34 insertions(+), 43 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index bcefa56..7c26c32 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "os/user" magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/api/smd" @@ -44,17 +45,15 @@ var collectCmd = &cobra.Command{ } // - if threads <= 0 { - threads = mathutil.Clamp(len(probeStates), 1, 255) + if concurrency <= 0 { + concurrency = mathutil.Clamp(len(probeStates), 1, 255) } q := &magellan.QueryParams{ User: username, Pass: password, Protocol: protocol, - Drivers: drivers, - Preferred: preferredDriver, Timeout: timeout, - Threads: threads, + Concurrency: concurrency, Verbose: verbose, CaCertPath: cacertPath, OutputPath: outputPath, @@ -72,16 +71,14 @@ var collectCmd = &cobra.Command{ } func init() { - collectCmd.PersistentFlags().StringSliceVar(&drivers, "driver", []string{"redfish"}, "set the driver(s) and fallback drivers to use") - collectCmd.PersistentFlags().StringVar(&smd.Host, "host", smd.Host, "set the host to the smd API") - collectCmd.PersistentFlags().IntVarP(&smd.Port, "port", "p", smd.Port, "set the port to the smd API") + currentUser, _ = user.Current() + collectCmd.PersistentFlags().StringVar(&smd.Host, "host", smd.Host, "set the host to the SMD API") + collectCmd.PersistentFlags().IntVarP(&smd.Port, "port", "p", smd.Port, "set the port to the SMD API") collectCmd.PersistentFlags().StringVar(&username, "user", "", "set the BMC user") collectCmd.PersistentFlags().StringVar(&password, "pass", "", "set the BMC password") collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "set the protocol used to query") - collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", "/tmp/magellan/data/", "set the path to store collection data") - collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD ") - collectCmd.PersistentFlags().StringVar(&preferredDriver, "preferred-driver", "ipmi", "set the preferred driver to use") - collectCmd.PersistentFlags().StringVar(&ipmitoolPath, "ipmitool.path", "/usr/bin/ipmitool", "set the path for ipmitool") + collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/data/", currentUser.Username), "set the path to store collection data") + collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD") collectCmd.PersistentFlags().StringVar(&cacertPath, "ca-cert", "", "path to CA cert. (defaults to system CAs)") viper.BindPFlag("collect.driver", collectCmd.Flags().Lookup("driver")) @@ -92,10 +89,8 @@ func init() { viper.BindPFlag("collect.protocol", collectCmd.Flags().Lookup("protocol")) viper.BindPFlag("collect.output", collectCmd.Flags().Lookup("output")) viper.BindPFlag("collect.force-update", collectCmd.Flags().Lookup("force-update")) - viper.BindPFlag("collect.preferred-driver", collectCmd.Flags().Lookup("preferred-driver")) - viper.BindPFlag("collect.ipmitool.path", collectCmd.Flags().Lookup("ipmitool.path")) - viper.BindPFlag("collect.secure-tls", collectCmd.Flags().Lookup("secure-tls")) - viper.BindPFlag("collect.cert-pool", collectCmd.Flags().Lookup("cert-pool")) + viper.BindPFlag("collect.ca-cert", collectCmd.Flags().Lookup("secure-tls")) + viper.BindPFlags(collectCmd.Flags()) rootCmd.AddCommand(collectCmd) } diff --git a/cmd/root.go b/cmd/root.go index cccc69a..41b0d20 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,22 +13,20 @@ import ( ) var ( - accessToken string - timeout int - threads int - ports []int - hosts []string - protocol string - cacertPath string - username string - password string - dbpath string - drivers []string - preferredDriver string - ipmitoolPath string - outputPath string - configPath string - verbose bool + currentUser *user.User + accessToken string + timeout int + concurrency int + ports []int + hosts []string + protocol string + cacertPath string + username string + password string + dbpath string + outputPath string + configPath string + verbose bool ) // TODO: discover bmc's on network (dora) @@ -77,28 +75,26 @@ func LoadAccessToken() (string, error) { } func init() { - // get the current user - currentUser, _ := user.Current() + currentUser, _ = user.Current() cobra.OnInitialize(InitializeConfig) - rootCmd.PersistentFlags().IntVar(&threads, "threads", -1, "set the number of threads") + rootCmd.PersistentFlags().IntVar(&concurrency, "concurrency", -1, "set the number of concurrent processes") rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 30, "set the timeout") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config file path") - rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set verbose flag") + rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set output verbosity") rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "set the access token") - rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Name+"/"), "set the probe storage path") + rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Username+"/"), "set the probe storage path") // bind viper config flags with cobra - viper.BindPFlag("threads", rootCmd.Flags().Lookup("threads")) + viper.BindPFlag("concurrency", rootCmd.Flags().Lookup("concurrency")) viper.BindPFlag("timeout", rootCmd.Flags().Lookup("timeout")) viper.BindPFlag("verbose", rootCmd.Flags().Lookup("verbose")) viper.BindPFlag("db.path", rootCmd.Flags().Lookup("db.path")) - // viper.BindPFlags(rootCmd.Flags()) + viper.BindPFlags(rootCmd.Flags()) } func InitializeConfig() { if configPath != "" { magellan.LoadConfig(configPath) - fmt.Printf("subnets: %v\n", viper.Get("scan.subnets")) } } diff --git a/cmd/scan.go b/cmd/scan.go index 50534a2..e114749 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -54,10 +54,10 @@ var scanCmd = &cobra.Command{ } // scan and store probe data in dbPath - if threads <= 0 { - threads = mathutil.Clamp(len(hostsToScan), 1, 255) + if concurrency <= 0 { + concurrency = mathutil.Clamp(len(hostsToScan), 1, 255) } - probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, threads, timeout, disableProbing, verbose) + probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, concurrency, timeout, disableProbing, verbose) if verbose { for _, r := range probeStates { fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol) From a2b841b4017d8effe54105ec02c8de0ba32c0006 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 10:10:21 -0600 Subject: [PATCH 07/15] Changed flag name from threads to concurrency --- internal/collect.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index 9eff14f..4f0dca3 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -39,7 +39,7 @@ type QueryParams struct { User string Pass string Drivers []string - Threads int + Concurrency int Preferred string Timeout int CaCertPath string @@ -71,14 +71,14 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err offset = 0 wg sync.WaitGroup found = make([]string, 0, len(*probeStates)) - done = make(chan struct{}, q.Threads+1) - chanProbeState = make(chan ScannedResult, q.Threads+1) + done = make(chan struct{}, q.Concurrency+1) + chanProbeState = make(chan ScannedResult, q.Concurrency+1) client = smd.NewClient( smd.WithSecureTLS(q.CaCertPath), ) ) - wg.Add(q.Threads) - for i := 0; i < q.Threads; i++ { + wg.Add(q.Concurrency) + for i := 0; i < q.Concurrency; i++ { go func() { for { ps, ok := <-chanProbeState From b3ba75ea9ad4a00942bec76c056716ab61248c80 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 13:41:08 -0600 Subject: [PATCH 08/15] Added format flag to convery output to JSON and slighly changed scanning logic --- cmd/list.go | 13 +++++++++-- cmd/root.go | 1 + cmd/scan.go | 64 +++++++++++++++++++++++++++++++++-------------------- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index c5ce1c4..5c1a46c 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -1,7 +1,9 @@ package cmd import ( + "encoding/json" "fmt" + "strings" "github.com/OpenCHAMI/magellan/internal/db/sqlite" @@ -17,12 +19,19 @@ var listCmd = &cobra.Command{ if err != nil { logrus.Errorf("failed toget probe results: %v\n", err) } - for _, r := range probeResults { - fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol) + format = strings.ToLower(format) + if format == "json" { + b, _ := json.Marshal(probeResults) + fmt.Printf("%s\n", string(b)) + } else { + for _, r := range probeResults { + fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol) + } } }, } func init() { + listCmd.Flags().StringVar(&format, "format", "", "set the output format") rootCmd.AddCommand(listCmd) } diff --git a/cmd/root.go b/cmd/root.go index 41b0d20..0e1697c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,6 +15,7 @@ import ( var ( currentUser *user.User accessToken string + format string timeout int concurrency int ports []int diff --git a/cmd/scan.go b/cmd/scan.go index e114749..557d061 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -1,10 +1,12 @@ package cmd import ( + "encoding/json" "fmt" "net" "os" "path" + "strings" magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/db/sqlite" @@ -26,41 +28,56 @@ var scanCmd = &cobra.Command{ Use: "scan", Short: "Scan for BMC nodes on a network", Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("subnets in cmd: %v\n", subnets) - // set hosts to use for scanning - hostsToScan := []string{} + var ( + hostsToScan []string + portsToScan []int + ) + + // start by adding `--host` supplied to scan if len(hosts) > 0 { hostsToScan = hosts - } else { - for i, subnet := range subnets { - if len(subnet) <= 0 { - return - } - - if len(subnetMasks) < i+1 { - subnetMasks = append(subnetMasks, net.IP{255, 255, 255, 0}) - } - - hostsToScan = append(hostsToScan, magellan.GenerateHosts(subnet, &subnetMasks[i])...) - } } - // set ports to use for scanning - portsToScan := []int{} + // add hosts from `--subnets` and `--subnet-mask` + for i, subnet := range subnets { + // subnet string is empty so nothing to do here + if subnet == "" { + continue + } + + // NOTE: should we check if subnet is valid here or is it done elsewhere (maybe in GenerateHosts)? + + // no subnet masks supplied so add a default one for class C private networks + if len(subnetMasks) < i+1 { + subnetMasks = append(subnetMasks, net.IP{255, 255, 255, 0}) + } + + // generate a slice of all hosts to scan from subnets + hostsToScan = append(hostsToScan, magellan.GenerateHosts(subnet, &subnetMasks[i])...) + } + + // add ports to use for scanning if len(ports) > 0 { portsToScan = ports } else { - portsToScan = append(magellan.GetDefaultPorts(), ports...) + // no ports supplied so only use defaults + portsToScan = magellan.GetDefaultPorts() } - // scan and store probe data in dbPath + // scan and store scanned data in cache if concurrency <= 0 { concurrency = mathutil.Clamp(len(hostsToScan), 1, 255) } probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, concurrency, timeout, disableProbing, verbose) if verbose { - for _, r := range probeStates { - fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol) + format = strings.ToLower(format) + if format == "json" { + b, _ := json.Marshal(probeStates) + fmt.Printf("%s\n", string(b)) + } else { + for _, r := range probeStates { + fmt.Printf("%s:%d (%s)\n", r.Host, r.Port, r.Protocol) + } } } @@ -77,10 +94,9 @@ var scanCmd = &cobra.Command{ func init() { scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan") scanCmd.Flags().IntSliceVar(&ports, "port", []int{}, "set the ports to scan") - // scanCmd.Flags().Uint8Var(&begin, "begin", 0, "set the starting point for range of IP addresses") - // scanCmd.Flags().Uint8Var(&end, "end", 255, "set the ending point for range of IP addresses") + scanCmd.Flags().StringVar(&format, "format", "", "set the output format") scanCmd.Flags().StringSliceVar(&subnets, "subnet", []string{}, "set additional subnets") - scanCmd.Flags().IPSliceVar(&subnetMasks, "subnet-mask", []net.IP{}, "set the subnet masks to use for network") + scanCmd.Flags().IPSliceVar(&subnetMasks, "subnet-mask", []net.IP{}, "set the subnet masks to use for network (must match number of subnets)") scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "disable probing scanned results for BMC nodes") viper.BindPFlag("scan.hosts", scanCmd.Flags().Lookup("host")) From 886a5d4d827f9466e8a41f460bb9b7f2247f5247 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 13:46:51 -0600 Subject: [PATCH 09/15] Fixed default output path for flag --- cmd/collect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/collect.go b/cmd/collect.go index 7c26c32..4b2c402 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -77,7 +77,7 @@ func init() { collectCmd.PersistentFlags().StringVar(&username, "user", "", "set the BMC user") collectCmd.PersistentFlags().StringVar(&password, "pass", "", "set the BMC password") collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "set the protocol used to query") - collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/data/", currentUser.Username), "set the path to store collection data") + collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/data/", currentUser.Username+"/"), "set the path to store collection data") collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD") collectCmd.PersistentFlags().StringVar(&cacertPath, "ca-cert", "", "path to CA cert. (defaults to system CAs)") From e0d02e5651eb69e0b243a7aee9a95e5caef3822f Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 13:48:36 -0600 Subject: [PATCH 10/15] Removed extra comments --- internal/collect.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index 4f0dca3..ee63c76 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -339,10 +339,6 @@ func CollectUsers(client *bmclib.Client, q *QueryParams) ([]byte, error) { } func CollectBios(client *bmclib.Client, q *QueryParams) ([]byte, error) { - // client, err := NewClient(l, q) - // if err != nil { - // return nil, fmt.Errorf("failed to make query: %v", err) - // } b, err := makeRequest(client, client.GetBiosConfiguration, q.Timeout) return b, err } From ac86a12ce026e9905e2b5dfdc49937f33cf3e3e3 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 13:52:00 -0600 Subject: [PATCH 11/15] Made user and pass flags required together --- cmd/collect.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/collect.go b/cmd/collect.go index 4b2c402..2065101 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -80,6 +80,7 @@ func init() { collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/data/", currentUser.Username+"/"), "set the path to store collection data") collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD") collectCmd.PersistentFlags().StringVar(&cacertPath, "ca-cert", "", "path to CA cert. (defaults to system CAs)") + collectCmd.MarkFlagsRequiredTogether("user", "pass") viper.BindPFlag("collect.driver", collectCmd.Flags().Lookup("driver")) viper.BindPFlag("collect.host", collectCmd.Flags().Lookup("host")) From 24cbae6570d1fe0555623c72a09616f4ad32feed Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 14:50:11 -0600 Subject: [PATCH 12/15] Updated README.md --- README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8a06117..5e2aecc 100644 --- a/README.md +++ b/README.md @@ -100,18 +100,19 @@ There are three main commands to use with the tool: `scan`, `list`, and `collect ./magellan scan \ --subnet 172.16.0.0 \ --subnet-mask 255.255.255.0 \ + --format json \ --db.path data/assets.db --port 443 ``` -This will scan the `172.16.0.0` subnet returning the host and port that return a response and store the results in a local cache with at the `data/assets.db` path. Additional flags can be set such as `host` to add more hosts to scan not included on the subnet, `timeout` to set how long to wait for a response from the BMC node, or `threads` to set the number of requests to make concurrently. Try using `./magellan help scan` for a complete set of options this subcommand. +This will scan the `172.16.0.0` subnet returning the host and port that return a response and store the results in a local cache with at the `data/assets.db` path. Additional flags can be set such as `--host` to add more hosts to scan not included on the subnet, `--timeout` to set how long to wait for a response from the BMC node, or `--concurrency` to set the number of requests to make concurrently. Setting the `--format=json` will format the output in JSON. Try using `./magellan help scan` for a complete set of options this subcommand. To inspect the cache, use the `list` command. Make sure to point to the same database used before: ```bash -./magellan list --db.path data/assets.db +./magellan list --db.path data/assets.db --format json ``` -This will print a list of node info found and stored from the scan. +This will print a list of node info found and stored from the scan. Like the `scan` subcommand, the output format can be set using the `--format` flag. Finally, set the `MAGELLAN_ACCESS_TOKEN`run the `collect` command to query the node from cache and send the info to be stored into SMD: @@ -123,10 +124,11 @@ Finally, set the `MAGELLAN_ACCESS_TOKEN`run the `collect` command to query the n --pass password \ --host https://example.openchami.cluster \ --port 27779 \ + --output logs/ --ca-cert cacert.pem ``` -This uses the info stored in cache to request information about each BMC node if possible. Like with the scan, the time to wait for a response can be set with the `timeout` flag as well. This command also requires the `user` and `pass/password` flag to be set if access the Redfish service requires basic authentication. Additionally, it may be necessary to set the `host` and `port` flags for `magellan` to find the SMD API (not the root API endpoint /hsm/v2). +This uses the info stored in cache to request information about each BMC node if possible. Like with the scan, the time to wait for a response can be set with the `--timeout` flag as well. This command also requires the `--user` and `--pass` flags to be set if access the Redfish service requires basic authentication. Additionally, it may be necessary to set the `--host` and `--port` flags for `magellan` to find the SMD API (not the root API endpoint "/hsm/v2"). The output of the `collect` can be saved by using the `--output` Note: If the `db.path` flag is not set, `magellan` will use "/tmp/$USER/magellan.db" by default. @@ -135,10 +137,11 @@ Note: If the `db.path` flag is not set, `magellan` will use "/tmp/$USER/magellan The `magellan` tool has a `login` subcommand that works with the [`opaal`](https://github.com/OpenCHAMI/opaal) service to obtain a token needed to access the SMD service. If the SMD instance requires authentication, set the `MAGELLAN_ACCESS_TOKEN` environment variable to have `magellan` include it in the header for HTTP requests to SMD. ```bash +# must have a running OPAAL instance ./magellan login --url https://opaal:4444/login -# ...complete login flow -export MAGELLAN_ACCESS_TOKEN= +# ...complete login flow to get token +export MAGELLAN_ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c ``` Alternatively, if you are running the OpenCHAMI quickstart, you can run the provided script to generate a token and set the environment variable that way. From f4540ea4ce8d6ac951e9cd49ab4722c4eaf060e6 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 14:54:34 -0600 Subject: [PATCH 13/15] Changed db.path flag to cache --- README.md | 10 +++++----- cmd/collect.go | 2 +- cmd/list.go | 2 +- cmd/root.go | 8 ++++---- cmd/scan.go | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5e2aecc..912d90a 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ Available Commands: Flags: --access-token string set the access token -c, --config string set the config file path - --db.path string set the probe storage path (default "/tmp/magellan/magellan.db") + --cache string set the probe storage path (default "/tmp/magellan/magellan.db") -h, --help help for magellan --threads int set the number of threads (default -1) --timeout int set the timeout (default 30) @@ -101,7 +101,7 @@ There are three main commands to use with the tool: `scan`, `list`, and `collect --subnet 172.16.0.0 \ --subnet-mask 255.255.255.0 \ --format json \ - --db.path data/assets.db --port 443 + --cache data/assets.db --port 443 ``` This will scan the `172.16.0.0` subnet returning the host and port that return a response and store the results in a local cache with at the `data/assets.db` path. Additional flags can be set such as `--host` to add more hosts to scan not included on the subnet, `--timeout` to set how long to wait for a response from the BMC node, or `--concurrency` to set the number of requests to make concurrently. Setting the `--format=json` will format the output in JSON. Try using `./magellan help scan` for a complete set of options this subcommand. @@ -109,7 +109,7 @@ This will scan the `172.16.0.0` subnet returning the host and port that return a To inspect the cache, use the `list` command. Make sure to point to the same database used before: ```bash -./magellan list --db.path data/assets.db --format json +./magellan list --cache data/assets.db --format json ``` This will print a list of node info found and stored from the scan. Like the `scan` subcommand, the output format can be set using the `--format` flag. @@ -118,7 +118,7 @@ Finally, set the `MAGELLAN_ACCESS_TOKEN`run the `collect` command to query the n ```bash ./magellan collect \ - --db.path data/assets.db \ + --cache data/assets.db \ --timeout 5 \ --user admin \ --pass password \ @@ -130,7 +130,7 @@ Finally, set the `MAGELLAN_ACCESS_TOKEN`run the `collect` command to query the n This uses the info stored in cache to request information about each BMC node if possible. Like with the scan, the time to wait for a response can be set with the `--timeout` flag as well. This command also requires the `--user` and `--pass` flags to be set if access the Redfish service requires basic authentication. Additionally, it may be necessary to set the `--host` and `--port` flags for `magellan` to find the SMD API (not the root API endpoint "/hsm/v2"). The output of the `collect` can be saved by using the `--output` -Note: If the `db.path` flag is not set, `magellan` will use "/tmp/$USER/magellan.db" by default. +Note: If the `cache` flag is not set, `magellan` will use "/tmp/$USER/magellan.db" by default. ### Getting an Access Token (WIP) diff --git a/cmd/collect.go b/cmd/collect.go index 2065101..fc3454c 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -26,7 +26,7 @@ var collectCmd = &cobra.Command{ l := log.NewLogger(logrus.New(), logrus.DebugLevel) // get probe states stored in db from scan - probeStates, err := sqlite.GetProbeResults(dbpath) + probeStates, err := sqlite.GetProbeResults(cachePath) if err != nil { l.Log.Errorf("failed toget states: %v", err) } diff --git a/cmd/list.go b/cmd/list.go index 5c1a46c..845da29 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -15,7 +15,7 @@ var listCmd = &cobra.Command{ Use: "list", Short: "List information from scan", Run: func(cmd *cobra.Command, args []string) { - probeResults, err := sqlite.GetProbeResults(dbpath) + probeResults, err := sqlite.GetProbeResults(cachePath) if err != nil { logrus.Errorf("failed toget probe results: %v\n", err) } diff --git a/cmd/root.go b/cmd/root.go index 0e1697c..ef488bc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -24,7 +24,7 @@ var ( cacertPath string username string password string - dbpath string + cachePath string outputPath string configPath string verbose bool @@ -83,13 +83,13 @@ func init() { rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config file path") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set output verbosity") rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "set the access token") - rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Username+"/"), "set the probe storage path") + rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Username+"/"), "set the scanning result cache path") // bind viper config flags with cobra viper.BindPFlag("concurrency", rootCmd.Flags().Lookup("concurrency")) viper.BindPFlag("timeout", rootCmd.Flags().Lookup("timeout")) viper.BindPFlag("verbose", rootCmd.Flags().Lookup("verbose")) - viper.BindPFlag("db.path", rootCmd.Flags().Lookup("db.path")) + viper.BindPFlag("cache", rootCmd.Flags().Lookup("cache")) viper.BindPFlags(rootCmd.Flags()) } @@ -104,7 +104,7 @@ func SetDefaults() { viper.SetDefault("timeout", 30) viper.SetDefault("config", "") viper.SetDefault("verbose", false) - viper.SetDefault("db.path", "/tmp/magellan/magellan.db") + viper.SetDefault("cache", "/tmp/magellan/magellan.db") viper.SetDefault("scan.hosts", []string{}) viper.SetDefault("scan.ports", []int{}) viper.SetDefault("scan.subnets", []string{}) diff --git a/cmd/scan.go b/cmd/scan.go index 557d061..563629f 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -82,12 +82,12 @@ var scanCmd = &cobra.Command{ } // make the dbpath dir if needed - err := os.MkdirAll(path.Dir(dbpath), 0766) + err := os.MkdirAll(path.Dir(cachePath), 0766) if err != nil { fmt.Printf("failed tomake database directory: %v", err) } - sqlite.InsertProbeResults(dbpath, &probeStates) + sqlite.InsertProbeResults(cachePath, &probeStates) }, } From 0f74e1e7f77baa5c3c6408b7b1820a2fb46fe2d3 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 10 May 2024 17:00:21 -0600 Subject: [PATCH 14/15] Added more output messages for handling errors --- internal/collect.go | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index ee63c76..7860228 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -100,7 +100,7 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err gofishClient, err := connectGofish(q) if err != nil { - l.Log.Errorf("failed to connect to bmc (%v:%v): %v", q.Host, q.Port, err) + l.Log.Errorf("failed to connect to BMC (%v:%v): %v", q.Host, q.Port, err) } // data to be sent to smd @@ -145,9 +145,15 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err // add other fields from systems if len(rm["Systems"]) > 0 { var s map[string][]interface{} - json.Unmarshal(rm["Systems"], &s) + err = json.Unmarshal(rm["Systems"], &s) + if err != nil { + l.Log.Errorf("failed to unmarshal systems JSON: %v", err) + } data["Name"] = s["Name"] } + } else { + l.Log.Errorf("invalid client (client is nil)") + continue } headers := make(map[string]string) @@ -349,17 +355,29 @@ func CollectEthernetInterfaces(c *gofish.APIClient, q *QueryParams, systemID str return nil, fmt.Errorf("failed to query storage systems (%v:%v): %v", q.Host, q.Port, err) } - var interfaces []*redfish.EthernetInterface + var ( + interfaces []*redfish.EthernetInterface + errList []error + ) + + // get all of the ethernet interfaces in our systems for _, system := range systems { i, err := redfish.ListReferencedEthernetInterfaces(c, "/redfish/v1/Systems/"+system.ID+"/EthernetInterfaces/") if err != nil { + errList = append(errList, err) continue } interfaces = append(interfaces, i...) } - if len(interfaces) <= 0 { - return nil, fmt.Errorf("failed to get ethernet interfaces: %v", err) + // format the error message for printing + for i, e := range errList { + err = fmt.Errorf("\t[%d] %v\n", i, e) + } + + // print any report errors + if len(errList) > 0 { + return nil, fmt.Errorf("failed to get ethernet interfaces with %d errors: \n%v", len(errList), err) } data := map[string]any{"EthernetInterfaces": interfaces} @@ -425,7 +443,10 @@ func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]byte, error) { continue } var i map[string]any - json.Unmarshal(interfaces, &i) + err = json.Unmarshal(interfaces, &i) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal interface: %v", err) + } temp = append(temp, map[string]any{ "Data": system, "EthernetInterfaces": i["EthernetInterfaces"], From d86532e96a59ccab6c5956d5565f429644b7d90b Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 13 May 2024 09:30:17 -0600 Subject: [PATCH 15/15] Added a firmware updating section to README.md --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 912d90a..4d76d61 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,27 @@ This uses the info stored in cache to request information about each BMC node if Note: If the `cache` flag is not set, `magellan` will use "/tmp/$USER/magellan.db" by default. +### Updating Firmware + +The `magellan` tool is capable of updating firmware with using the `update` subcommand via the Redfish API. This may sometimes necessary if some of the `collect` output is missing or is not including what is expected. The subcommand expects there to be a running HTTP/HTTPS server running that has an accessbile URL path to the firmware download. Specify the URL with the `--firmware-path` flag and the firmware type with the `--component` flag with all the other usual arguments like in the example below: + +```bash +./magellan update \ + --host 172.16.0.108 \ + --port 443 \ + --user username \ --pass password \ + --firmware-path http://172.16.0.255:8005/firmware/bios/image.RBU \ + --component BIOS +``` + +Then, the update status can be viewed by including the `--status` flag along with the other usual arguments or with the `watch` command: + +```bash +./magellan update --status --host 172.16.0.110 --user admin --pass password | jq '.' +# ...or... +watch -n 1 "./magellan update --status --host 172.16.0.110 --user admin --pass password | jq '.'" +``` + ### Getting an Access Token (WIP) The `magellan` tool has a `login` subcommand that works with the [`opaal`](https://github.com/OpenCHAMI/opaal) service to obtain a token needed to access the SMD service. If the SMD instance requires authentication, set the `MAGELLAN_ACCESS_TOKEN` environment variable to have `magellan` include it in the header for HTTP requests to SMD.