Split the collect Command For Customization (#93)

* feat: initial implementation of command split

* feat: update collect and new send cmd

* chore: cleanup unused code

* chore: refactored getting username

* chore: more refactoring and cleanup

* feat: update send cmd implementation

* chore: changed/updated example config

* chore: made cmd more consistent and added formatting

* refactor: removed --host flag from scan

* chore: cleaned up and fixed issue with client

* chore: cleaned up CLI flags in collect cmd

* feat: updated crawl to include managers and output YAML optionally

* refactor: updated and improved send implementation

* refactor: minor improvements

* refactor: added util func to check for empty slices

* fix: issue with reading from stdin

* refactor: added scheme trimming function for URIs

* refactor: changed host arg back to positional

* refactor: removed unused vars and added --output-dir flag

* fix: make -f for secrets persistent

* refactor: removed --host flag and request in collect

* refactor: changed --output flag to --output-file

* fix: updated flags for collect

* fix: typo in crawler error

* fix: dir being created when outputDir not set

* fix: reading stdin and data args

* fix: made output using -v and -o consistent

* readme: added info about command split

* updated changelog adding missing version entries

* chore: updated example to use host as positional arg

* fix: issue with reading --data arg

* fix: remove unused import from collect pkg

Signed-off-by: Devon Bautista <devonb@lanl.gov>

---------

Signed-off-by: David Allen <16520934+davidallendj@users.noreply.github.com>
Signed-off-by: Devon Bautista <devonb@lanl.gov>
Co-authored-by: Devon Bautista <devonb@lanl.gov>

CHANGELOG.md

@@ -5,6 +5,135 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.2]
+
+### Changed
+
+ * Split the `collect` command into `collect` and `send` commands
+ * Allowed piping `collect` output into `send` to allow intermediate data modification
+
+## [0.2.1]
+
+### Added
+
+ * Added defaults to secret store
+ * Added secret store support to `update` command
+ * Added `pkg/bmc` package to handle credentials internal
+  
+### Changed
+
+ * Changed behavior of  `--username` and `--password` flags to partial override credentials
+ * Changed CLI to have more consistent flags
+ 
+## [0.2.0]
+
+### Added
+
+ * Added `secrets` command for managing secrets with `SecretStore`
+ * Added `--username` and `--password` flags to `collect` command
+ * Added short option flags for `--username` and `--password` flags
+ * Added `--secrets-file` flag to `crawl` command
+ * Added static secrets store as fallback
+ * Added function to remove secrets from secrets store
+ * Added secrets lookup to `collect` command
+
+### Changed
+
+ * Changed short options for secret store
+ * Changed details to error messages
+
+### Fixed
+
+ * Fixed `golangci-lint` install command
+ * Fixed issues from running `golangci-lint`
+
+### Updated
+
+ * Updated `golangci-lint` version
+ * Updated logging to be use consistentn JSON formatting
+
+## [0.1.10]
+
+### Fixed
+
+ * Fixed README documentation
+
+## [0.1.9]
+
+### Added
+
+ * Added collection of data return from `CollectInventory()` output
+ * Added initial `SecretStore` interface with `StaticStore` and `LocalStore` implementations for credentials management
+ * Added `--insecure` flag to allow skipping TLS verification for firmware updates
+
+### Fixed
+
+ * Fixed dependabot security issues related to `crypto` package
+ * Fixed URL param not being set for `UpdateFirmwareRemote()`
+ * Fixed links in README documentation
+
+### Changes
+
+ * Improved firmware updating functionality and added BMC identification support
+ * Improved Redfish service connection handling and update status retrieval
+ * Moved internal implementations to `pkg` and updated references
+ * Updated `update` command to use `gofish` package internally
+
+## [0.1.8]
+
+ * Updated build workflow and added container build script
+ * Exported `cobra` commands for external use
+ * Fixed AMD64 microcode version in attestation
+
+## [0.1.7]
+
+ * Refactor how versioning information is indicated in the build and source
+ * Updated Go version
+
+## [0.1.6]
+
+### Added
+
+ * Added functionality to fetch BMC manager data and include in `crawler`'s output
+ * Added IP to manager's ethernet interfaces
+ * Added check to exclude ethernet interface without IPs
+ * Added MACAddr to manager's output
+ * Added function to wait for emulator to start in tests
+ * Added API tests
+ * Added revision to `go install` commands
+ * Added PKGBUILD to install `magellan` as binary on Arch Linux
+ * Added `version` command and corresponding implementation
+
+### Fixed
+
+ * Fixed issue writing output to file with `--output` flag
+ * Fixed hook to output correct filename with `goreleaser`
+ * Fixed typo in Makefile
+ * Fixed releaser .PHONY in Makefile
+ * Fixed issue with tests not working
+
+### Updated
+
+ * Updated `crawl` to fetch and include BMC `Manager` data in output
+ * Updated and refactored `util` package
+ * Updated README.md documentation
+ * Updated `goreleaser` to v2 (v2.3.2)
+ * Updated go dependencies
+ * Updated tests to fix some issues
+ * Updated .gitignore file
+ * Updated Makefile to include `magellan.1` rule
+ * Updated Makefile to build with ldflags
+
+### Changed
+
+ * Changed `crawler`'s internal function names
+ * Changed `test` rule in Makefile to use specific tests
+
+### Removed
+
+ * Removed extra unused `gofish` imports
+ * Removed internal version implementation
+
 ## [0.1.5]
 
 ### Added

README.md

@@ -7,23 +7,24 @@ The `magellan` CLI tool is a Redfish-based, board management controller (BMC) di
 
 <!-- TOC start (generated with https://github.com/derlin/bitdowntoc) -->
 
-  * [Main Features](#main-features)
+- [OpenCHAMI Magellan](#openchami-magellan)
-  * [Getting Started](#getting-started)
+  - [Main Features](#main-features)
-  * [Building the Executable](#building-the-executable)
+  - [Getting Started](#getting-started)
-    + [Building on Debian 12 (Bookworm)](#building-on-debian-12-bookworm)
+  - [Building the Executable](#building-the-executable)
-    + [Docker](#docker)
+    - [Building on Debian 12 (Bookworm)](#building-on-debian-12-bookworm)
-    + [Arch Linux (AUR)](#arch-linux-aur)
+    - [Docker](#docker)
-  * [Usage](#usage)
+    - [Arch Linux (AUR)](#arch-linux-aur)
-    + [Checking for Redfish](#checking-for-redfish)
+  - [Usage](#usage)
-    + [Running the Tool](#running-the-tool)
+    - [Checking for Redfish](#checking-for-redfish)
-    + [Managing Secrets](#managing-secrets)
+    - [Running the Tool](#running-the-tool)
-    + [Starting the Emulator](#starting-the-emulator)
+    - [Managing Secrets](#managing-secrets)
-    + [Updating Firmware](#updating-firmware)
+    - [Starting the Emulator](#starting-the-emulator)
-    + [Getting an Access Token (WIP)](#getting-an-access-token-wip)
+    - [Updating Firmware](#updating-firmware)
-    + [Running with Docker](#running-with-docker)
+    - [Getting an Access Token (WIP)](#getting-an-access-token-wip)
-  * [How It Works](#how-it-works)
+    - [Running with Docker](#running-with-docker)
-  * [TODO](#todo)
+  - [How It Works](#how-it-works)
-  * [Copyright](#copyright)
+  - [TODO](#todo)
+  - [Copyright](#copyright)
 
 <!-- TOC end -->
 
@@ -193,7 +194,9 @@ Once the scan is complete, inspect the cache to see a list of found hosts with t
 ./magellan list --cache data/assets.db
 ```
 
-This will print a list of host information needed for the `collect` step. Set the `ACCESS_TOKEN` if necessary and invoke `magellan` again with the `collect` subcommand to query the node BMCs stored in cache. If the `--host` flag is set, then an additional request will be made to send the output to the specified URL. The `--userame` and `--password` flags must be set if the BMC requires basic authentication.
+This will print a list of host information needed for the `collect` step. Set the `ACCESS_TOKEN` if necessary and invoke `magellan` again with the `collect` subcommand to query the node BMCs stored in cache.
+
+We can then save the output and make a request with the `send` subcommand or pipe the output directly to the specified URL. The `-u/--username` and `-p/--password` flags must be set if the BMC requires basic authentication if the `--secrets-file` flag and `MASTER_KEY` environment variable is not set.
 
 ```bash
 ./magellan collect \
@@ -202,13 +205,50 @@ This will print a list of host information needed for the `collect` step. Set th
     --username $USERNAME \
     --password $PASSWORD \
     --host https://example.openchami.cluster:8443 \
-    --output logs/
+    --format yaml \
+    --output-file nodes.yaml \
     --cacert cacert.pem
 ```
 
-This will initiate a crawler that will find as much inventory data as possible. The data can be viewed from standard output by setting the `--verbose` flag. This output can also be saved by using the `--output` flag and providing a path argument.
+This will initiate a crawler that fetch inventory data from the specified BMC host. The data can be saved, viewed, or modified from standard output by setting the `-v/--verbose` flag. Similarly, this output can also be saved by using the `-o/--output-file` flag and providing a path argument.
 
-Note: If the `cache` flag is not set, `magellan` will use `/tmp/$USER/magellan.db` by default.
+To make a request with the `collect` output, we specify the `-d/--data` flag for `send`. For files, use the `@` symbol before the file path. Make sure that you set the correct input format with `-F/--format`. Finally, specify the host as a positional argument.
+
+```bash
+magellan send -F yaml -d @nodes.yaml https://example.openchami.cluster:8443
+```
+
+This allows for modification of the data before making the request. However, be cautious as there is no data validation done before the request is made.
+
+Alternatively, we can pass the output of `collect` into `send` using pipes. The `--verbose` flag is currently required to do this.
+
+```bash
+# collect and send data in YAML format
+magellan collect -u $USERNAME -p $PASSWORD -v -F yaml | magellan send -F yaml https://example.openchami.cluster:8443
+
+# collect and send data using default JSON format and secret store (see below)
+export MASTER_KEY=mysecret
+magellan secrets store default $USERNAME:$PASSWORD
+magellan collect -v | magellan send https://example.openchami.cluster:8443
+```
+
+This maintains the original behavior of passing the `--host` flag to `collect` with the added flexibility of having the intermediate step.
+
+> [!TIP]
+> If the `cache` flag is not set, `magellan` will use `/tmp/$USER/magellan.db` by default.
+
+
+> [!TIP]
+> The output of `collect` can be saved in separate directories using the `-O/--output-dir` flag. The output will be organized similar to below for the following command in YAML format:
+>
+> ```bash
+> ./magellan collect -F yaml -v -O nodes
+> nodes
+> ├── x1000c1s7b0
+> │   └── 1747550498.yaml
+> └── x1000c1s7b1
+>     └── 1747550498.yaml
+> ```
 
 ### Managing Secrets
 
@@ -222,15 +262,13 @@ To store secrets using `magellan`:
 export MASTER_KEY=$(magellan secrets generatekey)
 ```
 
-2. Store secret credentials for hosts shown by `magellan list`:
+2. Store secret credentials for hosts shown by `magellan list`. There should be no output unless an error occurred.
 
 ```bash
 export bmc_host=https://172.16.0.105:443
 magellan secrets store $bmc_host $bmc_username:$bmc_password
 ```
 
-There should be no output unless an error occurred.
-
 3. Print the list of hosts to confirm secrets are stored.
 
 ```bash
@@ -256,11 +294,12 @@ magellan collect \
 
 If you pass arguments with the `--username/--password` flags, the arguments will override all credentials set in the secret store for each flag. However, it is possible only override a single flag (e.g. `magellan collect --username`).
 
-> [!NOTE]
+> [!WARNING]
 > Make sure that the `secretID` is EXACTLY as show with `magellan list`. Otherwise, `magellan` will not be able to do the lookup from the secret store correctly.
 
 > [!TIP]
 > You can set default fallback credentials by storing a secret with the `secretID` of "default". This is used if no `secretID` is found in the local store for the specified host. This is useful when you want to set a username and password that is the same for all BMCs with the exception of the ones specified.
+> 
 > ```bash
 > magellan secrets default $username:$password
 > ```
@@ -382,7 +421,7 @@ See the [issue list](https://github.com/OpenCHAMI/magellan/issues) for plans for
 * [X] Add ability to set subnet mask for scanning
 * [ ] Add ability to scan with other protocols like LLDP and SSDP
 * [X] Add more debugging messages with the `-v/--verbose` flag
-* [ ] Separate `collect` subcommand with making request to endpoint
+* [X] Separate `collect` subcommand with making request to endpoint
 * [X] Support logging in with `opaal` to get access token
 * [X] Support using CA certificates with HTTP requests to SMD
 * [X] Add tests for the regressions and compatibility

cmd/collect.go

@@ -2,8 +2,6 @@ package cmd
 
 import (
 	"encoding/json"
-	"fmt"
-	"os/user"
 
 	"github.com/OpenCHAMI/magellan/internal/cache/sqlite"
 	urlx "github.com/OpenCHAMI/magellan/internal/url"
@@ -17,6 +15,8 @@ import (
 	"github.com/spf13/viper"
 )
 
+var collectOutputFormat string
+
 // The `collect` command fetches data from a collection of BMC nodes.
 // This command should be ran after the `scan` to find available hosts
 // on a subnet.
@@ -122,6 +122,8 @@ var CollectCmd = &cobra.Command{
 			Verbose:     verbose,
 			CaCertPath:  cacertPath,
 			OutputPath:  outputPath,
+			OutputDir:   outputDir,
+			Format:      collectOutputFormat,
 			ForceUpdate: forceUpdate,
 			AccessToken: accessToken,
 			SecretStore: store,
@@ -140,22 +142,24 @@ var CollectCmd = &cobra.Command{
 }
 
 func init() {
-	currentUser, _ = user.Current()
-	CollectCmd.Flags().StringVar(&host, "host", "", "Set the URI to the SMD root endpoint")
 	CollectCmd.Flags().StringVarP(&username, "username", "u", "", "Set the master BMC username")
 	CollectCmd.Flags().StringVarP(&password, "password", "p", "", "Set the master BMC password")
 	CollectCmd.Flags().StringVar(&secretsFile, "secrets-file", "", "Set path to the node secrets file")
 	CollectCmd.Flags().StringVar(&scheme, "scheme", "https", "Set the default scheme used to query when not included in URI")
 	CollectCmd.Flags().StringVar(&protocol, "protocol", "tcp", "Set the protocol used to query")
-	CollectCmd.Flags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "Set the path to store collection data")
+	CollectCmd.Flags().StringVarP(&outputPath, "output-file", "o", "", "Set the path to store collection data using HIVE partitioning")
+	CollectCmd.Flags().StringVarP(&outputDir, "output-dir", "O", "", "Set the path to store collection data using HIVE partitioning")
 	CollectCmd.Flags().BoolVar(&forceUpdate, "force-update", false, "Set flag to force update data sent to SMD")
-	CollectCmd.Flags().StringVar(&cacertPath, "cacert", "", "Set the path to CA cert file. (defaults to system CAs when blank)")
+	CollectCmd.Flags().StringVar(&cacertPath, "cacert", "", "Set the path to CA cert file (defaults to system CAs when blank)")
+	CollectCmd.Flags().StringVarP(&collectOutputFormat, "format", "F", FORMAT_JSON, "Set the output format (json|yaml)")
+
+	CollectCmd.MarkFlagsMutuallyExclusive("output-file", "output-dir")
 
 	// bind flags to config properties
-	checkBindFlagError(viper.BindPFlag("collect.host", CollectCmd.Flags().Lookup("host")))
 	checkBindFlagError(viper.BindPFlag("collect.scheme", CollectCmd.Flags().Lookup("scheme")))
 	checkBindFlagError(viper.BindPFlag("collect.protocol", CollectCmd.Flags().Lookup("protocol")))
-	checkBindFlagError(viper.BindPFlag("collect.output", CollectCmd.Flags().Lookup("output")))
+	checkBindFlagError(viper.BindPFlag("collect.output-file", CollectCmd.Flags().Lookup("output-file")))
+	checkBindFlagError(viper.BindPFlag("collect.output-dir", CollectCmd.Flags().Lookup("output-dir")))
 	checkBindFlagError(viper.BindPFlag("collect.force-update", CollectCmd.Flags().Lookup("force-update")))
 	checkBindFlagError(viper.BindPFlag("collect.cacert", CollectCmd.Flags().Lookup("cacert")))
 	checkBindFlagError(viper.BindPFlags(CollectCmd.Flags()))

cmd/crawl.go

@@ -3,8 +3,10 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 
 	"github.com/rs/zerolog/log"
+	"gopkg.in/yaml.v3"
 
 	urlx "github.com/OpenCHAMI/magellan/internal/url"
 	"github.com/OpenCHAMI/magellan/pkg/bmc"
@@ -14,6 +16,8 @@ import (
 	"github.com/spf13/viper"
 )
 
+var crawlOutputFormat string
+
 // The `crawl` command walks a collection of Redfish endpoints to collect
 // specfic inventory detail. This command only expects host names and does
 // not require a scan to be performed beforehand.
@@ -37,9 +41,10 @@ var CrawlCmd = &cobra.Command{
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var (
-			uri   = args[0]
+			uri    = args[0]
-			store secrets.SecretStore
+			store  secrets.SecretStore
-			err   error
+			output []byte
+			err    error
 		)
 
 		if username != "" && password != "" {
@@ -76,24 +81,53 @@ var CrawlCmd = &cobra.Command{
 			store = &nodeCreds
 		}
 
-		systems, err := crawler.CrawlBMCForSystems(crawler.CrawlerConfig{
+		var (
-			URI:             uri,
+			systems  []crawler.InventoryDetail
-			CredentialStore: store,
+			managers []crawler.Manager
-			Insecure:        insecure,
+			config   = crawler.CrawlerConfig{
-			UseDefault:      true,
+				URI:             uri,
-		})
+				CredentialStore: store,
+				Insecure:        insecure,
+				UseDefault:      true,
+			}
+		)
+
+		systems, err = crawler.CrawlBMCForSystems(config)
 		if err != nil {
-			log.Error().Err(err).Msg("failed to crawl BMC")
+			log.Error().Err(err).Msg("failed to crawl BMC for systems")
 		}
-		// Marshal the inventory details to JSON
+		managers, err = crawler.CrawlBMCForManagers(config)
-		jsonData, err := json.MarshalIndent(systems, "", "  ")
 		if err != nil {
-			log.Error().Err(err).Msg("failed to marshal JSON")
+			log.Error().Err(err).Msg("failed to crawl BMC for managers")
-			return
 		}
 
-		// Print the pretty JSON
+		data := map[string]any{
-		fmt.Println(string(jsonData))
+			"Systems":  systems,
+			"Managers": managers,
+		}
+
+		switch crawlOutputFormat {
+		case FORMAT_JSON:
+			// Marshal the inventory details to JSON
+			output, err = json.MarshalIndent(data, "", "  ")
+			if err != nil {
+				log.Error().Err(err).Msg("failed to marshal JSON")
+				return
+			}
+		case FORMAT_YAML:
+			// Marshal the inventory details to JSON
+			output, err = yaml.Marshal(data)
+			if err != nil {
+				log.Error().Err(err).Msg("failed to marshal JSON")
+				return
+			}
+		default:
+			log.Error().Str("hint", "Try setting --format/-F to 'json' or 'yaml'").Msg("unrecognized format")
+			os.Exit(1)
+		}
+
+		// Print the pretty JSON or YAML
+		fmt.Println(string(output))
 	},
 }
 
@@ -101,8 +135,11 @@ func init() {
 	CrawlCmd.Flags().StringVarP(&username, "username", "u", "", "Set the username for the BMC")
 	CrawlCmd.Flags().StringVarP(&password, "password", "p", "", "Set the password for the BMC")
 	CrawlCmd.Flags().BoolVarP(&insecure, "insecure", "i", false, "Ignore SSL errors")
-	CrawlCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "set the secrets file with BMC credentials")
+	CrawlCmd.Flags().StringVarP(&secretsFile, "secrets-file", "f", "secrets.json", "Set path to the node secrets file")
+	CrawlCmd.Flags().StringVarP(&crawlOutputFormat, "format", "F", FORMAT_JSON, "Set the output format (json|yaml)")
 
+	checkBindFlagError(viper.BindPFlag("crawl.insecure", CrawlCmd.Flags().Lookup("insecure")))
+	checkBindFlagError(viper.BindPFlag("crawl.insecure", CrawlCmd.Flags().Lookup("insecure")))
 	checkBindFlagError(viper.BindPFlag("crawl.insecure", CrawlCmd.Flags().Lookup("insecure")))
 
 	rootCmd.AddCommand(CrawlCmd)

cmd/list.go

@@ -3,17 +3,20 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/OpenCHAMI/magellan/internal/cache/sqlite"
 	"github.com/rs/zerolog/log"
+	"gopkg.in/yaml.v3"
 
 	"github.com/spf13/cobra"
 )
 
 var (
-	showCache bool
+	showCache        bool
+	listOutputFormat string
 )
 
 // The `list` command provides an easy way to show what was found
@@ -41,23 +44,32 @@ var ListCmd = &cobra.Command{
 		if err != nil {
 			log.Error().Err(err).Msg("failed to get scanned assets")
 		}
-		format = strings.ToLower(format)
+		switch strings.ToLower(listOutputFormat) {
-		if format == "json" {
+		case FORMAT_JSON:
 			b, err := json.Marshal(scannedResults)
 			if err != nil {
-				log.Error().Err(err).Msgf("failed to unmarshal scanned results")
+				log.Error().Err(err).Msgf("failed to unmarshal cached data to JSON")
 			}
 			fmt.Printf("%s\n", string(b))
-		} else {
+		case FORMAT_YAML:
+			b, err := yaml.Marshal(scannedResults)
+			if err != nil {
+				log.Error().Err(err).Msgf("failed to unmarshal cached data to YAML")
+			}
+			fmt.Printf("%s\n", string(b))
+		case FORMAT_LIST:
 			for _, r := range scannedResults {
 				fmt.Printf("%s:%d (%s) @%s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate))
 			}
+		default:
+			log.Error().Msg("unrecognized format")
+			os.Exit(1)
 		}
 	},
 }
 
 func init() {
-	ListCmd.Flags().StringVar(&format, "format", "", "Set the output format (json|default)")
+	ListCmd.Flags().StringVarP(&listOutputFormat, "format", "F", FORMAT_LIST, "Set the output format (json|yaml|table)")
 	ListCmd.Flags().BoolVar(&showCache, "cache-info", false, "Show cache information and exit")
 	rootCmd.AddCommand(ListCmd)
 }

cmd/root.go

@@ -18,28 +18,33 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"os/user"
 
 	magellan "github.com/OpenCHAMI/magellan/internal"
+	"github.com/OpenCHAMI/magellan/internal/util"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
+const (
+	FORMAT_LIST = "list"
+	FORMAT_JSON = "json"
+	FORMAT_YAML = "yaml"
+)
+
+// CLI arguments as variables to not fiddle with error-prone strings
 var (
-	currentUser *user.User
 	accessToken string
-	format      string
 	timeout     int
 	concurrency int
 	ports       []int
-	hosts       []string
 	protocol    string
 	cacertPath  string
 	username    string
 	password    string
 	cachePath   string
 	outputPath  string
+	outputDir   string
 	configPath  string
 	verbose     bool
 	debug       bool
@@ -73,15 +78,14 @@ func Execute() {
 }
 
 func init() {
-	currentUser, _ = user.Current()
 	cobra.OnInitialize(InitializeConfig)
 	rootCmd.PersistentFlags().IntVarP(&concurrency, "concurrency", "j", -1, "Set the number of concurrent processes")
-	rootCmd.PersistentFlags().IntVarP(&timeout, "timeout", "t", 5, "Set the timeout for requests")
+	rootCmd.PersistentFlags().IntVarP(&timeout, "timeout", "t", 5, "Set the timeout for requests in seconds")
 	rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "Set the config file path")
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Set to enable/disable verbose output")
-	rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Set to enable/disable debug messages")
+	rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Set to enable/disable debug messages")
 	rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "Set the access token")
-	rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", currentUser.Username), "Set the scanning result cache path")
+	rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", util.GetCurrentUsername()), "Set the scanning result cache path")
 
 	// bind viper config flags with cobra
 	checkBindFlagError(viper.BindPFlag("concurrency", rootCmd.PersistentFlags().Lookup("concurrency")))
@@ -117,13 +121,12 @@ func InitializeConfig() {
 // TODO: This function should probably be moved to 'internal/config.go'
 // instead of in this file.
 func SetDefaults() {
-	currentUser, _ = user.Current()
 	viper.SetDefault("threads", 1)
 	viper.SetDefault("timeout", 5)
 	viper.SetDefault("config", "")
 	viper.SetDefault("verbose", false)
 	viper.SetDefault("debug", false)
-	viper.SetDefault("cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", currentUser.Username))
+	viper.SetDefault("cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", util.GetCurrentUsername()))
 	viper.SetDefault("scan.hosts", []string{})
 	viper.SetDefault("scan.ports", []int{})
 	viper.SetDefault("scan.subnets", []string{})

cmd/scan.go

@@ -79,18 +79,8 @@ var ScanCmd = &cobra.Command{
 
 		// format and combine flag and positional args
 		targetHosts = append(targetHosts, urlx.FormatHosts(args, ports, scheme, verbose)...)
-		targetHosts = append(targetHosts, urlx.FormatHosts(hosts, ports, scheme, verbose)...)
 
-		// add more hosts specified with `--subnet` flag
-		if debug {
-			log.Debug().Msg("adding hosts from subnets")
-		}
 		for _, subnet := range subnets {
-			// subnet string is empty so nothing to do here
-			if subnet == "" {
-				continue
-			}
-
 			// generate a slice of all hosts to scan from subnets
 			subnetHosts := magellan.GenerateHostsWithSubnet(subnet, &subnetMask, ports, scheme)
 			targetHosts = append(targetHosts, subnetHosts...)
@@ -180,8 +170,6 @@ var ScanCmd = &cobra.Command{
 }
 
 func init() {
-	// scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan")
-	ScanCmd.Flags().StringSliceVar(&hosts, "host", nil, "Add individual hosts to scan. (example: https://my.bmc.com:5000; same as using positional args)")
 	ScanCmd.Flags().IntSliceVar(&ports, "port", nil, "Adds additional ports to scan for each host with unspecified ports.")
 	ScanCmd.Flags().StringVar(&scheme, "scheme", "https", "Set the default scheme to use if not specified in host URI. (default is 'https')")
 	ScanCmd.Flags().StringVar(&protocol, "protocol", "tcp", "Set the default protocol to use in scan. (default is 'tcp')")
@@ -190,7 +178,6 @@ func init() {
 	ScanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "Disable probing found assets for Redfish service(s) running on BMC nodes")
 	ScanCmd.Flags().BoolVar(&disableCache, "disable-cache", false, "Disable saving found assets to a cache database specified with 'cache' flag")
 
-	checkBindFlagError(viper.BindPFlag("scan.hosts", ScanCmd.Flags().Lookup("host")))
 	checkBindFlagError(viper.BindPFlag("scan.ports", ScanCmd.Flags().Lookup("port")))
 	checkBindFlagError(viper.BindPFlag("scan.scheme", ScanCmd.Flags().Lookup("scheme")))
 	checkBindFlagError(viper.BindPFlag("scan.protocol", ScanCmd.Flags().Lookup("protocol")))

cmd/secrets.go

@@ -256,9 +256,9 @@ var secretsRemoveCmd = &cobra.Command{
 }
 
 func init() {
-	secretsCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "set the secrets file with BMC credentials")
+	secretsCmd.PersistentFlags().StringVarP(&secretsFile, "file", "f", "secrets.json", "Set the secrets file with BMC credentials.")
-	secretsStoreCmd.Flags().StringVarP(&secretsStoreFormat, "format", "F", "basic", "set the input format for the secrets file (basic|json|base64)")
+	secretsStoreCmd.Flags().StringVarP(&secretsStoreFormat, "format", "F", "basic", "Set the input format for the secrets file (basic|json|base64).")
-	secretsStoreCmd.Flags().StringVarP(&secretsStoreInputFile, "input-file", "i", "", "set the file to read as input")
+	secretsStoreCmd.Flags().StringVarP(&secretsStoreInputFile, "input-file", "i", "", "Set the file to read as input.")
 
 	secretsCmd.AddCommand(secretsGenerateKeyCmd)
 	secretsCmd.AddCommand(secretsStoreCmd)
@@ -268,7 +268,7 @@ func init() {
 
 	rootCmd.AddCommand(secretsCmd)
 
-	checkBindFlagError(viper.BindPFlags(secretsCmd.Flags()))
+	checkBindFlagError(viper.BindPFlags(secretsCmd.PersistentFlags()))
 	checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags()))
 	checkBindFlagError(viper.BindPFlags(secretsStoreCmd.Flags()))
 	checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags()))

cmd/send.go

@@ -0,0 +1,283 @@
+package cmd
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	urlx "github.com/OpenCHAMI/magellan/internal/url"
+	"github.com/OpenCHAMI/magellan/pkg/auth"
+	"github.com/OpenCHAMI/magellan/pkg/client"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	sendInputFormat string
+	sendDataArgs    []string
+)
+
+var sendCmd = &cobra.Command{
+	Use: "send [data]",
+	Example: `  // minimal working example
+  magellan send -d @inventory.json https://smd.openchami.cluster
+
+  // send data from multiple files (must specify -f/--format if not JSON)
+  magellan send -d @cluster-1.json -d @cluster-2.json https://smd.openchami.cluster
+  magellan send -d '{...}' -d @cluster-1.json https://proxy.example.com
+
+  // send data to remote host by piping output of collect directly
+  magellan collect -v -F yaml | magellan send -d @inventory.yaml -F yaml https://smd.openchami.cluster`,
+	Short: "Send collected node information to specified host.",
+	Args: func(cmd *cobra.Command, args []string) error {
+		return nil
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+
+		// try to load access token either from env var, file, or config if var not set
+		if accessToken == "" {
+			var err error
+			accessToken, err = auth.LoadAccessToken(tokenPath)
+			if err != nil && verbose {
+				log.Warn().Err(err).Msgf("could not load access token")
+			} else if debug && accessToken != "" {
+				log.Debug().Str("access_token", accessToken).Msg("using access token")
+			}
+		}
+
+		// try and load cert if argument is passed for client
+		var smdClient = client.NewSmdClient()
+		if cacertPath != "" {
+			log.Debug().Str("path", cacertPath).Msg("using provided certificate path")
+			err := client.LoadCertificateFromPath(smdClient, cacertPath)
+			if err != nil {
+				log.Warn().Err(err).Msg("could not load certificate")
+			}
+		}
+
+		// make one request be host positional argument (restricted to 1 for now)
+		var inputData []map[string]any
+		temp := append(handleArgs(args), processDataArgs(sendDataArgs)...)
+		for _, data := range temp {
+			if data != nil {
+				inputData = append(inputData, data)
+			}
+		}
+		if len(inputData) == 0 {
+			log.Error().Msg("must include data with standard input or -d/--data flag")
+			os.Exit(1)
+		}
+
+		// show the data that was just loaded as input
+		if verbose {
+			output, err := json.Marshal(inputData)
+			if err != nil {
+				log.Error().Err(err).Msg("failed to marshal input data")
+			}
+			fmt.Println(string(output))
+		}
+
+		for _, host := range args {
+			var (
+				body []byte
+				err  error
+			)
+
+			smdClient.URI = host
+			for _, dataObject := range inputData {
+				// skip on to the next thing if it's does not exist
+				if dataObject == nil {
+					continue
+				}
+
+				// create and set headers for request
+				headers := client.HTTPHeader{}
+				headers.Authorization(accessToken)
+				headers.ContentType("application/json")
+
+				host, err = urlx.Sanitize(host)
+				if err != nil {
+					log.Warn().Err(err).Str("host", host).Msg("could not sanitize host")
+				}
+
+				// convert to JSON to send data
+				body, err = json.MarshalIndent(dataObject, "", "  ")
+				if err != nil {
+					log.Error().Err(err).Msg("failed to marshal request data")
+					continue
+				}
+
+				err = smdClient.Add(body, headers)
+				if err != nil {
+					// try updating instead
+					if forceUpdate {
+						smdClient.Xname = dataObject["ID"].(string)
+						err = smdClient.Update(body, headers)
+						if err != nil {
+							log.Error().Err(err).Msgf("failed to forcibly update Redfish endpoint with ID %s", smdClient.Xname)
+						}
+					} else {
+						log.Error().Err(err).Msgf("failed to add Redfish endpoint with ID %s", smdClient.Xname)
+					}
+				}
+			}
+
+		}
+	},
+}
+
+func init() {
+	sendCmd.Flags().StringArrayVarP(&sendDataArgs, "data", "d", []string{}, "Set the data to send to specified host (prepend @ for files)")
+	sendCmd.Flags().StringVarP(&sendInputFormat, "format", "F", FORMAT_JSON, "Set the data input format (json|yaml)")
+	sendCmd.Flags().BoolVarP(&forceUpdate, "force-update", "f", false, "Set flag to force update data sent to SMD")
+	sendCmd.Flags().StringVar(&cacertPath, "cacert", "", "Set the path to CA cert file (defaults to system CAs when blank)")
+	rootCmd.AddCommand(sendCmd)
+}
+
+// processDataArgs takes a slice of strings that check for the @ symbol and loads
+// the contents from the file specified in place (which replaces the path).
+//
+// NOTE: The purpose is to make the input arguments uniform for our request. This
+// function is meant to handle data passed with the `-d/--data` flag and positional
+// args from the CLI.
+func processDataArgs(args []string) []map[string]any {
+	// JSON representation
+	type (
+		JSONObject = map[string]any
+		JSONArray  = []JSONObject
+	)
+
+	// load data either from file or directly from args
+	var collection = make(JSONArray, len(args))
+	for i, arg := range args {
+		// if arg is empty string, then skip and continue
+		if len(arg) > 0 {
+			// determine if we're reading from file to load contents
+			if strings.HasPrefix(arg, "@") {
+				var (
+					path     string = strings.TrimLeft(arg, "@")
+					contents []byte
+					data     JSONArray
+					err      error
+				)
+
+				contents, err = os.ReadFile(path)
+				if err != nil {
+					log.Error().Err(err).Str("path", path).Msg("failed to read file")
+					continue
+				}
+
+				// skip empty files
+				if len(contents) == 0 {
+					log.Warn().Str("path", path).Msg("file is empty")
+					continue
+				}
+
+				// convert/validate JSON input format
+				data, err = parseInput(contents)
+				if err != nil {
+					log.Error().Err(err).Str("path", path).Msg("failed to validate input from file")
+				}
+
+				// add loaded data to collection of all data
+				collection = append(collection, data...)
+			} else {
+				// input should be a valid JSON
+				var (
+					data  JSONArray
+					input = []byte(arg)
+					err   error
+				)
+				if !json.Valid(input) {
+					log.Error().Msgf("argument %d not a valid JSON", i)
+					continue
+				}
+				err = json.Unmarshal(input, &data)
+				if err != nil {
+					log.Error().Err(err).Msgf("failed to unmarshal input for argument %d", i)
+				}
+				return data
+			}
+		}
+	}
+	return collection
+}
+
+func handleArgs(args []string) []map[string]any {
+	// JSON representation
+	type (
+		JSONObject = map[string]any
+		JSONArray  = []JSONObject
+	)
+	// no file to load, so we just use the joined args (since each one is a new line)
+	// and then stop
+	var (
+		collection JSONArray
+		data       []byte
+		err        error
+	)
+
+	if len(sendDataArgs) > 0 {
+		return nil
+	}
+	data, err = ReadStdin()
+	if err != nil {
+		log.Error().Err(err).Msg("failed to read from standard input")
+		return nil
+	}
+	if len(data) == 0 {
+		log.Warn().Msg("no data found from standard input")
+		return nil
+	}
+	fmt.Println(string(data))
+	collection, err = parseInput([]byte(data))
+	if err != nil {
+		log.Error().Err(err).Msg("failed to validate input from arg")
+	}
+	return collection
+}
+
+func parseInput(contents []byte) ([]map[string]any, error) {
+	var (
+		data []map[string]any
+		err  error
+	)
+
+	// convert/validate JSON input format
+	switch sendInputFormat {
+	case FORMAT_JSON:
+		err = json.Unmarshal(contents, &data)
+		if err != nil {
+			return nil, fmt.Errorf("failed to unmarshal input in JSON: %v", err)
+		}
+	case FORMAT_YAML:
+		err = yaml.Unmarshal(contents, &data)
+		if err != nil {
+			return nil, fmt.Errorf("failed to unmarshal input in YAML: %v", err)
+		}
+	default:
+		return nil, fmt.Errorf("unrecognized format")
+	}
+	return data, nil
+}
+
+// ReadStdin reads all of standard input and returns the bytes. If an error
+// occurs during scanning, it is returned.
+func ReadStdin() ([]byte, error) {
+	var b []byte
+	input := bufio.NewScanner(os.Stdin)
+	for input.Scan() {
+		b = append(b, input.Bytes()...)
+		b = append(b, byte('\n'))
+		if len(b) == 0 {
+			break
+		}
+	}
+	if err := input.Err(); err != nil {
+		return b, fmt.Errorf("failed to read stdin: %w", err)
+	}
+	return b, nil
+}

example.config.yaml

@@ -1,6 +1,7 @@
 scan:
   hosts:
     - "172.16.1.15"
+    - "10.0.0.1"
   subnets:
     - "172.16.0.0"
     - "172.16.0.0/24"
@@ -13,8 +14,6 @@ scan:
   protocol: "tcp"
   scheme: "https"
 collect:
-  username: "admin"
-  password: "password"
   protocol: "tcp"
   scheme: "https"
   output: "/tmp/magellan/data/"
@@ -24,8 +23,6 @@ collect:
 update:
   host:
   port: 443
-  username: "admin"
-  password: "password"
   transfer-protocol: "https"
   firmware:
     url:
@@ -36,5 +33,5 @@ update:
 concurrency: 1
 timeout: 30
 verbose: true
-db:
+cache:
   path: "/tmp/magellan/magellan.db"

go.mod

@@ -19,6 +19,7 @@ require (
 require (
 	github.com/rs/zerolog v1.33.0
 	golang.org/x/crypto v0.32.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -56,5 +57,4 @@ require (
 	golang.org/x/text v0.21.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

internal/url/url.go

@@ -21,6 +21,14 @@ func Sanitize(uri string) (string, error) {
 	return parsedURI.String(), nil
 }
 
+func TrimScheme(uri string) string {
+	const prefix = "https://"
+	if strings.Contains(uri, prefix) {
+		return strings.TrimPrefix(uri, prefix)
+	}
+	return uri
+}
+
 // FormatHosts() takes a list of hosts and ports and builds full URLs in the
 // form of scheme://host:port. If no scheme is provided, it will use "https" by
 // default.

internal/util/slices.go

@@ -0,0 +1,5 @@
+package util
+
+func IsEmpty[T any](s []T) bool {
+	return len(s) == 0
+}

internal/util/util.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"fmt"
+	"os/user"
 	"time"
 )
 
@@ -25,3 +26,11 @@ func CheckUntil(interval time.Duration, timeout time.Duration, predicate func()
 		}
 	}
 }
+
+func GetCurrentUsername() string {
+	currentUser, _ := user.Current()
+	if currentUser != nil {
+		return currentUser.Username
+	}
+	return ""
+}

pkg/client/client.go

@@ -9,8 +9,6 @@ import (
 	"net/http"
 	"os"
 	"time"
-
-	"github.com/rs/zerolog/log"
 )
 
 type Option[T Client] func(client *T)
@@ -29,56 +27,51 @@ type Client interface {
 	Update(data HTTPBody, headers HTTPHeader) error
 }
 
-// NewClient() creates a new client
+func LoadCertificateFromPath(client Client, path string) error {
-func NewClient[T Client](opts ...func(T)) T {
+	cacert, err := os.ReadFile(path)
-	client := new(T)
-	for _, opt := range opts {
-		opt(*client)
-	}
-	return *client
-}
-
-func WithCertPool[T Client](certPool *x509.CertPool) func(T) {
-	// make sure we have a valid cert pool
-	if certPool == nil {
-		return func(client T) {}
-	}
-	return func(client T) {
-		// make sure that we can access the internal client
-		if client.GetInternalClient() == nil {
-			log.Warn().Any("client", client.GetInternalClient()).Msg("invalid internal HTTP client ()")
-			return
-		}
-		client.GetInternalClient().Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{
-				RootCAs:            certPool,
-				InsecureSkipVerify: true,
-			},
-			DisableKeepAlives: true,
-			Dial: (&net.Dialer{
-				Timeout:   120 * time.Second,
-				KeepAlive: 120 * time.Second,
-			}).Dial,
-			TLSHandshakeTimeout:   120 * time.Second,
-			ResponseHeaderTimeout: 120 * time.Second,
-		}
-	}
-}
-
-func WithSecureTLS[T Client](certPath string) func(T) {
-	cacert, err := os.ReadFile(certPath)
 	if err != nil {
-		return func(client T) {}
+		return fmt.Errorf("failed to read certificate at path: %s", path)
 	}
 	certPool := x509.NewCertPool()
 	certPool.AppendCertsFromPEM(cacert)
-	return WithCertPool[T](certPool)
+	err = LoadCertificateFromPool(client, certPool)
+	if err != nil {
+		return fmt.Errorf("could not initialize certificate from pool: %v", err)
+	}
+	return nil
+}
+
+func LoadCertificateFromPool(client Client, certPool *x509.CertPool) error {
+	// make sure we have a valid cert pool
+	if certPool == nil {
+		return fmt.Errorf("invalid cert pool")
+	}
+
+	// make sure that we can access the internal client
+	internalClient := client.GetInternalClient()
+	if internalClient == nil {
+		return fmt.Errorf("invalid HTTP client")
+	}
+	internalClient.Transport = &http.Transport{
+		TLSClientConfig: &tls.Config{
+			RootCAs:            certPool,
+			InsecureSkipVerify: false,
+		},
+		DisableKeepAlives: true,
+		Dial: (&net.Dialer{
+			Timeout:   120 * time.Second,
+			KeepAlive: 120 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout:   120 * time.Second,
+		ResponseHeaderTimeout: 120 * time.Second,
+	}
+	return nil
 }
 
 // Post() is a simplified wrapper function that packages all of the
 // that marshals a mapper into a JSON-formatted byte array, and then performs
 // a request to the specified URL.
-func (c *MagellanClient) Post(url string, data map[string]any, header HTTPHeader) (*http.Response, HTTPBody, error) {
+func (c *DefaultClient) Post(url string, data map[string]any, header HTTPHeader) (*http.Response, HTTPBody, error) {
 	// serialize data into byte array
 	body, err := json.Marshal(data)
 	if err != nil {

pkg/client/default.go

@@ -5,11 +5,11 @@ import (
 	"net/http"
 )
 
-type MagellanClient struct {
+type DefaultClient struct {
 	*http.Client
 }
 
-func (c *MagellanClient) Name() string {
+func (c *DefaultClient) Name() string {
 	return "default"
 }
 
@@ -18,7 +18,7 @@ func (c *MagellanClient) Name() string {
 // the first argument with no data processing or manipulation. The function sends
 // the data to a set callback URL (which may be changed to use a configurable value
 // instead).
-func (c *MagellanClient) Add(data HTTPBody, headers HTTPHeader) error {
+func (c *DefaultClient) Add(data HTTPBody, headers HTTPHeader) error {
 	if data == nil {
 		return fmt.Errorf("no data found")
 	}
@@ -35,7 +35,7 @@ func (c *MagellanClient) Add(data HTTPBody, headers HTTPHeader) error {
 	return err
 }
 
-func (c *MagellanClient) Update(data HTTPBody, headers HTTPHeader) error {
+func (c *DefaultClient) Update(data HTTPBody, headers HTTPHeader) error {
 	if data == nil {
 		return fmt.Errorf("no data found")
 	}

pkg/client/smd.go

@@ -4,6 +4,7 @@ package client
 //	https://github.com/OpenCHAMI/hms-smd/blob/master/docs/examples.adoc
 //	https://github.com/OpenCHAMI/hms-smd
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 
@@ -16,6 +17,12 @@ type SmdClient struct {
 	Xname string
 }
 
+func NewSmdClient() *SmdClient {
+	return &SmdClient{
+		Client: &http.Client{},
+	}
+}
+
 func (c *SmdClient) Init() {
 	c.Client = &http.Client{}
 }
@@ -44,7 +51,7 @@ func (c *SmdClient) Add(data HTTPBody, headers HTTPHeader) error {
 	url := c.RootEndpoint("/Inventory/RedfishEndpoints")
 	res, body, err := MakeRequest(c.Client, url, http.MethodPost, data, headers)
 	if res != nil {
-		statusOk := res.StatusCode >= 200 && res.StatusCode < 300
+		statusOk := res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusMultipleChoices
 		if !statusOk {
 			if len(body) > 0 {
 				return fmt.Errorf("%d: %s", res.StatusCode, string(body))
@@ -77,3 +84,17 @@ func (c *SmdClient) Update(data HTTPBody, headers HTTPHeader) error {
 	}
 	return err
 }
+
+func (c *SmdClient) SetXnameFromJSON(contents []byte, key string) error {
+	var (
+		data map[string]any
+		err  error
+	)
+
+	err = json.Unmarshal(contents, &data)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal xname: %v", err)
+	}
+	c.Xname = data[key].(string)
+	return nil
+}

pkg/collect.go

@@ -2,12 +2,9 @@
 package magellan
 
 import (
-	"crypto/tls"
-	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"net"
-	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -15,10 +12,12 @@ import (
 	"sync"
 	"time"
 
+	"github.com/OpenCHAMI/magellan/internal/util"
 	"github.com/OpenCHAMI/magellan/pkg/bmc"
 	"github.com/OpenCHAMI/magellan/pkg/client"
 	"github.com/OpenCHAMI/magellan/pkg/crawler"
 	"github.com/OpenCHAMI/magellan/pkg/secrets"
+	"gopkg.in/yaml.v3"
 
 	"github.com/rs/zerolog/log"
 
@@ -38,6 +37,8 @@ type CollectParams struct {
 	CaCertPath  string              // set the cert path with the 'cacert' flag
 	Verbose     bool                // set whether to include verbose output with 'verbose' flag
 	OutputPath  string              // set the path to save output with 'output' flag
+	OutputDir   string              // set the directory path to save output with `output-dir` flag
+	Format      string              // set the output format
 	ForceUpdate bool                // set whether to force updating SMD with 'force-update' flag
 	AccessToken string              // set the access token to include in request with 'access-token' flag
 	SecretStore secrets.SecretStore // set BMC credentials
@@ -66,34 +67,9 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin
 		found      = make([]string, 0, len(*assets))
 		done       = make(chan struct{}, params.Concurrency+1)
 		chanAssets = make(chan RemoteAsset, params.Concurrency+1)
-		outputPath = path.Clean(params.OutputPath)
-		smdClient  = &client.SmdClient{Client: &http.Client{}}
 	)
 
 	// set the client's params from CLI
-	// NOTE: temporary solution until client.NewClient() is fixed
-	smdClient.URI = params.URI
-	if params.CaCertPath != "" {
-		cacert, err := os.ReadFile(params.CaCertPath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read CA cert path: %w", err)
-		}
-		certPool := x509.NewCertPool()
-		certPool.AppendCertsFromPEM(cacert)
-		smdClient.Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{
-				RootCAs:            certPool,
-				InsecureSkipVerify: true,
-			},
-			DisableKeepAlives: true,
-			Dial: (&net.Dialer{
-				Timeout:   120 * time.Second,
-				KeepAlive: 120 * time.Second,
-			}).Dial,
-			TLSHandshakeTimeout:   120 * time.Second,
-			ResponseHeaderTimeout: 120 * time.Second,
-		}
-	}
 	wg.Add(params.Concurrency)
 	for i := 0; i < params.Concurrency; i++ {
 		go func() {
@@ -141,7 +117,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin
 				}
 
 				// we didn't find anything so do not proceed
-				if len(systems) == 0 && len(managers) == 0 {
+				if util.IsEmpty(systems) && util.IsEmpty(managers) {
 					continue
 				}
 
@@ -167,6 +143,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin
 
 				// optionally, add the MACAddr property if we find a matching IP
 				// from the correct ethernet interface
+
 				host := sr.Host
 				str_protocol := "https://"
 				if strings.Contains(host, str_protocol) {
@@ -185,59 +162,9 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin
 				headers.Authorization(params.AccessToken)
 				headers.ContentType("application/json")
 
-				body, err := json.MarshalIndent(data, "", "    ")
-				if err != nil {
-					log.Error().Err(err).Msgf("failed to marshal output to JSON")
-				}
-
-				if params.Verbose {
-					fmt.Printf("%v\n", string(body))
-				}
-
 				// add data output to collections
 				collection = append(collection, data)
 
-				// write JSON data to file if output path is set using hive partitioning strategy
-				if outputPath != "" {
-					var (
-						finalPath = fmt.Sprintf("./%s/%s/%d.json", outputPath, data["ID"], time.Now().Unix())
-						finalDir  = filepath.Dir(finalPath)
-					)
-					// if it doesn't, make the directory and write file
-					err = os.MkdirAll(finalDir, 0o777)
-					if err == nil { // no error
-						err = os.WriteFile(path.Clean(finalPath), body, os.ModePerm)
-						if err != nil {
-							log.Error().Err(err).Msgf("failed to write collect output to file")
-						}
-
-					} else { // error is set
-						log.Error().Err(err).Msg("failed to make directory for collect output")
-					}
-				}
-
-				// add all endpoints to SMD ONLY if a host is provided
-				if smdClient.URI != "" {
-					err = smdClient.Add(body, headers)
-					if err != nil {
-
-						// try updating instead
-						if params.ForceUpdate {
-							smdClient.Xname = data["ID"].(string)
-							err = smdClient.Update(body, headers)
-							if err != nil {
-								log.Error().Err(err).Msgf("failed to forcibly update Redfish endpoint")
-							}
-						} else {
-							log.Error().Err(err).Msgf("failed to add Redfish endpoint")
-						}
-					}
-				} else {
-					if params.Verbose {
-						log.Warn().Msg("no request made (host argument is empty)")
-					}
-				}
-
 				// got host information, so add to list of already probed hosts
 				found = append(found, sr.Host)
 			}
@@ -269,6 +196,64 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) ([]map[strin
 	wg.Wait()
 	close(done)
 
+	var (
+		output []byte
+		err    error
+	)
+
+	// format our output to write to file or standard out
+	switch params.Format {
+	case "json":
+		output, err = json.MarshalIndent(collection, "", "    ")
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to marshal output to JSON")
+		}
+	case "yaml":
+		output, err = yaml.Marshal(collection)
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to marshal output to YAML")
+		}
+	}
+
+	// print the final combined output at the end to write to file
+	if params.Verbose {
+		fmt.Printf("%v\n", string(output))
+	}
+
+	// write data to file in preset directory if output path is set using set format
+	if params.OutputDir != "" {
+		for _, data := range collection {
+			var (
+				finalPath = fmt.Sprintf("./%s/%s/%d.%s", path.Clean(params.OutputDir), data["ID"], time.Now().Unix(), params.Format)
+				finalDir  = filepath.Dir(finalPath)
+			)
+			// if it doesn't, make the directory and write file
+			err = os.MkdirAll(finalDir, 0o777)
+			if err == nil { // no error
+				err = os.WriteFile(path.Clean(finalPath), output, os.ModePerm)
+				if err != nil {
+					log.Error().Err(err).Msgf("failed to write collect output to file")
+				}
+			} else { // error is set
+				log.Error().Err(err).Msg("failed to make directory for collect output")
+			}
+		}
+	}
+
+	// write data to only to the path set (no preset directory structure)
+	if params.OutputPath != "" {
+		// if it doesn't, make the directory and write file
+		err = os.MkdirAll(filepath.Dir(params.OutputPath), 0o777)
+		if err == nil { // no error
+			err = os.WriteFile(path.Clean(params.OutputPath), output, os.ModePerm)
+			if err != nil {
+				log.Error().Err(err).Msgf("failed to write collect output to file")
+			}
+		} else { // error is set
+			log.Error().Err(err).Msg("failed to make directory for collect output")
+		}
+	}
+
 	return collection, nil
 }
 
@@ -345,6 +330,7 @@ func FindMACAddressWithIP(config crawler.CrawlerConfig, targetIP net.IP) (string
 			continue
 		}
 	}
+
 	// no matches found, so return an empty string
 	return "", fmt.Errorf("no ethernet interfaces found with IP address")
 }

pkg/crawler/main.go

@@ -122,17 +122,17 @@ func CrawlBMCForSystems(config CrawlerConfig) ([]InventoryDetail, error) {
 
 	// Obtain the ServiceRoot
 	rf_service := client.GetService()
-	log.Info().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion)
+	log.Debug().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion)
 
 	// Nodes are sometimes only found under Chassis, but they should be found under Systems.
 	rf_chassis, err := rf_service.Chassis()
 	if err == nil {
-		log.Info().Msgf("found %d chassis in ServiceRoot", len(rf_chassis))
+		log.Debug().Msgf("found %d chassis in ServiceRoot", len(rf_chassis))
 		for _, chassis := range rf_chassis {
 			rf_chassis_systems, err := chassis.ComputerSystems()
 			if err == nil {
 				rf_systems = append(rf_systems, rf_chassis_systems...)
-				log.Info().Msgf("found %d systems in chassis %s", len(rf_chassis_systems), chassis.ID)
+				log.Debug().Msgf("found %d systems in chassis %s", len(rf_chassis_systems), chassis.ID)
 			}
 		}
 	}
@@ -140,7 +140,7 @@ func CrawlBMCForSystems(config CrawlerConfig) ([]InventoryDetail, error) {
 	if err != nil {
 		log.Error().Err(err).Msg("failed to get systems from ServiceRoot")
 	}
-	log.Info().Msgf("found %d systems in ServiceRoot", len(rf_root_systems))
+	log.Debug().Msgf("found %d systems in ServiceRoot", len(rf_root_systems))
 	rf_systems = append(rf_systems, rf_root_systems...)
 	return walkSystems(rf_systems, nil, config.URI)
 }
@@ -198,7 +198,7 @@ func CrawlBMCForManagers(config CrawlerConfig) ([]Manager, error) {
 
 	// Obtain the ServiceRoot
 	rf_service := client.GetService()
-	log.Info().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion)
+	log.Debug().Msgf("found ServiceRoot %s. Redfish Version %s", rf_service.ID, rf_service.RedfishVersion)
 
 	rf_managers, err := rf_service.Managers()
 	if err != nil {
@@ -375,7 +375,7 @@ func loadBMCCreds(config CrawlerConfig) (bmc.BMCCredentials, error) {
 		return bmc.BMCCredentials{}, fmt.Errorf("credential store is invalid")
 	}
 	if creds := util.GetBMCCredentials(config.CredentialStore, config.URI); creds == (bmc.BMCCredentials{}) {
-		return creds, fmt.Errorf("%s: credentials blank for BNC", config.URI)
+		return creds, fmt.Errorf("%s: credentials blank for BMC", config.URI)
 	} else {
 		return creds, nil
 	}