diff --git a/Makefile b/Makefile index 8b5f3cb..80292d4 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ mod: ## go mod tidy inst: ## go install tools $(call print-target) go install github.com/client9/misspell/cmd/misspell@v0.3.4 - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.0.1 go install github.com/goreleaser/goreleaser/v2@v2.3.2 go install github.com/cpuguy83/go-md2man/v2@latest @@ -71,7 +71,7 @@ build: ## go build container: ## docker build container: $(call print-target) - docker build . --build-arg REGISTRY_HOST=${REGISTRY_HOST} --no-cache --pull --tag '${NAME}:${VERSION}' + docker build . --build-arg REGISTRY_HOST=${REGISTRY_HOST} --no-cache --pull --tag '${NAME}:${VERSION}' .PHONY: spell spell: ## misspell diff --git a/README.md b/README.md index 2135758..29df9b5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,33 @@ # OpenCHAMI Magellan -The `magellan` CLI tool is a Redfish-based, board management controller (BMC) discovery tool designed to scan networks and is written in Go. The tool collects information from BMC nodes using the provided Redfish RESTful API with [`gofish`](https://github.com/stmcginnis/gofish) and loads the queried data into an [SMD](https://github.com/OpenCHAMI/smd/) instance. The tool strives to be more flexible by implementing multiple methods of discovery to work for a wider range of systems (WIP) and is capable of using independently of other tools or services. +The `magellan` CLI tool is a Redfish-based, board management controller (BMC) discovery tool designed to scan networks and is written in Go. The tool collects information from BMC nodes using the provided Redfish RESTful API with [`gofish`](https://github.com/stmcginnis/gofish) and loads the queried data into an [SMD](https://github.com/OpenCHAMI/smd/) instance. The tool strives to be more flexible by implementing multiple methods of discovery to work for a wider range of systems (WIP) and is capable of being used independently of other tools or services. -**Note: `magellan` v0.1.0 is incompatible with SMD v2.15.3 and earlier.** +> [!NOTE] +> The v0.1.0 version of `magellan` is incompatible with `smd` v2.15.3 and earlier due to `smd` lacking the inventory parsing code used with `magellan`'s output.** + + + + * [Main Features](#main-features) + * [Getting Started](#getting-started) + * [Building the Executable](#building-the-executable) + + [Building on Debian 12 (Bookworm)](#building-on-debian-12-bookworm) + + [Docker](#docker) + + [Arch Linux (AUR)](#arch-linux-aur) + * [Usage](#usage) + + [Checking for Redfish](#checking-for-redfish) + + [Running the Tool](#running-the-tool) + + [Managing Secrets](#managing-secrets) + + [Starting the Emulator](#starting-the-emulator) + + [Updating Firmware](#updating-firmware) + + [Getting an Access Token (WIP)](#getting-an-access-token-wip) + + [Running with Docker](#running-with-docker) + * [How It Works](#how-it-works) + * [TODO](#todo) + * [Copyright](#copyright) + + + + ## Main Features @@ -13,6 +38,7 @@ The `magellan` tool comes packed with a handleful of features for doing discover - Redfish-based firmware updating - Integration with OpenCHAMI SMD - Write inventory data to JSON +- Store and manage BMC secrets See the [TODO](#todo) section for a list of soon-ish goals planned. @@ -25,7 +51,7 @@ See the [TODO](#todo) section for a list of soon-ish goals planned. The `magellan` tool can be built to run on bare metal. Install the required Go tools, clone the repo, and then build the binary in the root directory with the following: ```bash -git clone https://github.com/OpenCHAMI/magellan +git clone https://github.com/OpenCHAMI/magellan cd magellan go mod tidy && go build ``` @@ -40,7 +66,7 @@ Getting the `magellan` tool to work with Go 1.21 on Debian 12 may require instal apt install gcc golang-1.21/bookworm-backport ``` -The binary executable for the `golang-1.21` executable can then be found using `dpkg`. +The binary executable for the `golang-1.21` executable can then be found using `dpkg`.v2.0.1 ```bash dpkg -L golang-1.21-go @@ -49,7 +75,7 @@ dpkg -L golang-1.21-go Using the correct binary, set the `CGO_ENABLED` environment variable and build the executable with `cgo` enabled: ```bash -export GOBIN=/usr/bin/golang-1.21/bin/go +export GOBIN=/usr/bin/golang-1.21/bin/go go env -w CGO_ENABLED=1 go mod tidy && go build ``` @@ -66,6 +92,17 @@ docker pull ghcr.io/openchami/magellan:latest See the ["Running with Docker"](#running-with-docker) section below about running with the Docker container. + +### Arch Linux (AUR) + +The `magellan` tool is in the AUR as a binary package and can be installed via your favorite AUR helper. + +```bash +yay -S magellan-bin +``` +> [!NOTE] +> The AUR package may not always be in sync with the latest release. It is recommended to install `magellan` from source for the latest version. + ## Usage The sections below assume that the BMC nodes have an IP address available to query Redfish. Currently, `magellan` does not support discovery with MAC addresses although that may change in the future. @@ -173,14 +210,95 @@ This will initiate a crawler that will find as much inventory data as possible. Note: If the `cache` flag is not set, `magellan` will use `/tmp/$USER/magellan.db` by default. +### Managing Secrets + +When connecting to an array of BMC nodes, some nodes may have different secret credentials than the rest. These secrets can be stored and used automatically by `magellan` when performing a `collect` or a `crawl`. All secrets are encrypted and are only accessible using the same `MASTER_KEY` as when stored originally. + +To store secrets using `magellan`: + +1. Set the `MASTER_KEY` environment variable. This can be generated using `magellan secrets generatekey`. + +```bash +export MASTER_KEY=$(magellan secrets generatekey) +``` + +2. Store secret credentials for hosts shown by `magellan list`: + +```bash +export bmc_host=https://172.16.0.105:443 +magellan secrets store $bmc_host $bmc_username:$bmc_password +``` + +There should be no output unless an error occurred. + +3. Print the list of hosts to confirm secrets are stored. + +```bash +magellan secrets list +``` + +If you see your `bmc_host` listed in the output, that means that your secrets were stored successfully. + +Additionally, if you want to see the actually contents, make sure the `MASTER_KEY` environment variable is correctly set and do the following: + +```bash +magellan secrets retrieve $bmc_host +``` + +4. Run either a `crawl` or `collect` and `magellan` should be a do find the credentials for each host. + +```bash +magellan crawl -i $bmc_host +magellan collect \ + --username $default_bmc_username \ + --password $default_bmc_password +``` + +If you pass agruments with the `--username/--password` flags, they will be used as a fallback if no credentials are found in the store. However, the secret store credentials are always used first if they exists. + +> [!NOTE] +> Make sure that the `secretID` is EXACTLY as show with `magellan list`. Otherwise, `magellan` will not be able to do the lookup from the secret store correctly. + +### Starting the Emulator + +This repository includes a quick and dirty way to test `magellan` using a Redfish emulator with little to no effort to get running. + +1. Make sure you have `docker` with Docker compose and optionally `make`. + +2. Run the `emulator/setup.sh` script or alternatively `make emulator`. + +This will start a flask server that you can make requests to using `curl`. + +```bash +export emulator_host=https://172.21.0.2:5000 +export emulator_username=root # set in the `rf_emulator.yml` file +export emulator_password=root_password # set in the `rf_emulator.yml` file +curl -k $emulator_host/redfish/v1 -u $emulator_username:$emulator_password +``` + +...or with `magellan` using the secret store... + +```bash +magellan scan --subnet 172.21.0.0/24 +magellan secrets store \ + $emulator_host \ + $emulator_username:$emulator_password +magellan collect --host https://smd.openchami.cluster +``` + +This example should work just like running on real hardware. + +> [!NOTE] +> The emulator host may be different from the one in the README. Make sure to double-check the host! + ### Updating Firmware The `magellan` tool is capable of updating firmware with using the `update` subcommand via the Redfish API. This may sometimes necessary if some of the `collect` output is missing or is not including what is expected. The subcommand expects there to be a running HTTP/HTTPS server running that has an accessible URL path to the firmware download. Specify the URL with the `--firmware-path` flag and the firmware type with the `--component` flag (optional) with all the other usual arguments like in the example below: ```bash ./magellan update 172.16.0.108:443 \ - --username $USERNAME \ - --password $PASSWORD \ + --username $bmc_username \ + --password $bmc_password \ --firmware-path http://172.16.0.255:8005/firmware/bios/image.RBU \ --component BIOS ``` @@ -188,9 +306,12 @@ The `magellan` tool is capable of updating firmware with using the `update` subc Then, the update status can be viewed by including the `--status` flag along with the other usual arguments or with the `watch` command: ```bash -./magellan update 172.16.0.110 --status --username $USERNAME --pass $PASSWORD | jq '.' +./magellan update 172.16.0.110 \ + --status \ + --username $bmc_username \ + --password $bmc_password | jq '.' # ...or... -watch -n 1 "./magellan update 172.16.0.110 --status --username $USERNAME --password $PASSWORD | jq '.'" +watch -n 1 "./magellan update 172.16.0.110 --status --username $bmc_username --password $bmc_password | jq '.'" ``` ### Getting an Access Token (WIP) @@ -219,7 +340,6 @@ The `magellan` tool can be ran in a Docker container after pulling the latest im ```bash docker pull ghcr.io/openchami/magellan:latest - ``` Then, run either with the helper script found in `bin/magellan.sh` or the binary in the container: @@ -259,8 +379,10 @@ See the [issue list](https://github.com/OpenCHAMI/magellan/issues) for plans for * [ ] Separate `collect` subcommand with making request to endpoint * [X] Support logging in with `opaal` to get access token * [X] Support using CA certificates with HTTP requests to SMD -* [ ] Add tests for the regressions and compatibility +* [X] Add tests for the regressions and compatibility * [X] Clean up, remove unused, and tidy code (first round) +* [X] Add `secrets` command to manage secret credentials +* [ ] Add server component to make `magellan` a micro-service ## Copyright diff --git a/cmd/collect.go b/cmd/collect.go index c304beb..7223a75 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -25,7 +25,11 @@ var CollectCmd = &cobra.Command{ "See the 'scan' command on how to perform a scan.\n\n" + "Examples:\n" + " magellan collect --cache ./assets.db --output ./logs --timeout 30 --cacert cecert.pem\n" + - " magellan collect --host smd.example.com --port 27779 --username username --password password", + " magellan collect --host smd.example.com --port 27779 --username $username --password $password\n\n" + + // example using `collect` + " export MASTER_KEY=$(magellan secrets generatekey)\n" + + " magellan secrets store $node_creds_json -f nodes.json" + + " magellan collect --host openchami.cluster --username $username --password $password \\\n", Run: func(cmd *cobra.Command, args []string) { // get probe states stored in db from scan scannedResults, err := sqlite.GetScannedAssets(cachePath) @@ -48,17 +52,13 @@ var CollectCmd = &cobra.Command{ } } - if verbose { - log.Debug().Str("Access Token", accessToken) - } - - // + // set the minimum/maximum number of concurrent processes if concurrency <= 0 { concurrency = mathutil.Clamp(len(scannedResults), 1, 10000) } - // Create a StaticSecretStore to hold the username and password - secrets := secrets.NewStaticStore(username, password) - _, err = magellan.CollectInventory(&scannedResults, &magellan.CollectParams{ + + // set the collect parameters from CLI params + params := &magellan.CollectParams{ URI: host, Timeout: timeout, Concurrency: concurrency, @@ -67,9 +67,28 @@ var CollectCmd = &cobra.Command{ OutputPath: outputPath, ForceUpdate: forceUpdate, AccessToken: accessToken, - }, secrets) + SecretsFile: secretsFile, + Username: username, + Password: password, + } + + // show all of the 'collect' parameters being set from CLI if verbose + if verbose { + log.Debug().Any("params", params) + } + + // load the secrets file to get node credentials by ID (i.e. the BMC node's URI) + store, err := secrets.OpenStore(params.SecretsFile) if err != nil { - log.Error().Err(err).Msgf("failed to collect data") + // Something went wrong with the store so try using + // Create a StaticSecretStore to hold the username and password + log.Warn().Err(err).Msg("failed to open local store") + store = secrets.NewStaticStore(username, password) + } + + _, err = magellan.CollectInventory(&scannedResults, params, store) + if err != nil { + log.Error().Err(err).Msg("failed to collect data") } }, } @@ -77,13 +96,14 @@ var CollectCmd = &cobra.Command{ func init() { currentUser, _ = user.Current() CollectCmd.PersistentFlags().StringVar(&host, "host", "", "Set the URI to the SMD root endpoint") - CollectCmd.PersistentFlags().StringVar(&username, "username", "", "Set the BMC user") - CollectCmd.PersistentFlags().StringVar(&password, "password", "", "Set the BMC password") - CollectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the scheme used to query") + CollectCmd.PersistentFlags().StringVarP(&username, "username", "u", "", "Set the master BMC username") + CollectCmd.PersistentFlags().StringVarP(&password, "password", "p", "", "Set the master BMC password") + CollectCmd.PersistentFlags().StringVar(&secretsFile, "secrets-file", "", "Set path to the node secrets file") + CollectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the default scheme used to query when not included in URI") CollectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "Set the protocol used to query") CollectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "Set the path to store collection data") CollectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "Set flag to force update data sent to SMD") - CollectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "Path to CA cert. (defaults to system CAs)") + CollectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "Set the path to CA cert file. (defaults to system CAs when blank)") // set flags to only be used together CollectCmd.MarkFlagsRequiredTogether("username", "password") diff --git a/cmd/crawl.go b/cmd/crawl.go index e9e91bd..0e2984d 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -3,7 +3,8 @@ package cmd import ( "encoding/json" "fmt" - "log" + + "github.com/rs/zerolog/log" urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/OpenCHAMI/magellan/pkg/crawler" @@ -18,8 +19,8 @@ import ( var CrawlCmd = &cobra.Command{ Use: "crawl [uri]", Short: "Crawl a single BMC for inventory information", - Long: "Crawl a single BMC for inventory information. This command does NOT store information\n" + - "about the scan into cache after completion. To do so, use the 'collect' command instead\n\n" + + Long: "Crawl a single BMC for inventory information with URI. This command does NOT scan subnets nor store scan information\n" + + "in cache after completion. To do so, use the 'collect' command instead\n\n" + "Examples:\n" + " magellan crawl https://bmc.example.com\n" + " magellan crawl https://bmc.example.com -i -u username -p password", @@ -36,22 +37,37 @@ var CrawlCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { - staticStore := &secrets.StaticStore{ - Username: viper.GetString("crawl.username"), - Password: viper.GetString("crawl.password"), + var ( + uri = args[0] + store secrets.SecretStore + err error + ) + // try and load credentials from local store first + store, err = secrets.OpenStore(secretsFile) + if err != nil { + log.Warn().Err(err).Msg("failed to open local store...falling back to default provided arguments") + // try and use the `username` and `password` arguments instead + store = secrets.NewStaticStore(username, password) } + + // found the store so try to load the creds + _, err = store.GetSecretByID(uri) + if err != nil { + store = secrets.NewStaticStore(username, password) + } + systems, err := crawler.CrawlBMCForSystems(crawler.CrawlerConfig{ - URI: args[0], - CredentialStore: staticStore, - Insecure: cmd.Flag("insecure").Value.String() == "true", + URI: uri, + CredentialStore: store, + Insecure: insecure, }) if err != nil { - log.Fatalf("Error crawling BMC: %v", err) + log.Error().Err(err).Msg("failed to crawl BMC") } // Marshal the inventory details to JSON jsonData, err := json.MarshalIndent(systems, "", " ") if err != nil { - fmt.Println("Error marshalling to JSON:", err) + log.Error().Err(err).Msg("failed to marshal JSON") return } @@ -61,9 +77,10 @@ var CrawlCmd = &cobra.Command{ } func init() { - CrawlCmd.Flags().StringP("username", "u", "", "Set the username for the BMC") - CrawlCmd.Flags().StringP("password", "p", "", "Set the password for the BMC") - CrawlCmd.Flags().BoolP("insecure", "i", false, "Ignore SSL errors") + CrawlCmd.Flags().StringVarP(&username, "username", "u", "", "Set the username for the BMC") + CrawlCmd.Flags().StringVarP(&password, "password", "p", "", "Set the password for the BMC") + CrawlCmd.Flags().BoolVarP(&insecure, "insecure", "i", false, "Ignore SSL errors") + CrawlCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "set the secrets file with BMC credentials") checkBindFlagError(viper.BindPFlag("crawl.username", CrawlCmd.Flags().Lookup("username"))) checkBindFlagError(viper.BindPFlag("crawl.password", CrawlCmd.Flags().Lookup("password"))) diff --git a/cmd/list.go b/cmd/list.go index d760501..9fbc361 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -21,6 +21,7 @@ var ( // is what is consumed by the `collect` command with the --cache flag. var ListCmd = &cobra.Command{ Use: "list", + Args: cobra.ExactArgs(0), Short: "List information stored in cache from a scan", Long: "Prints all of the host and associated data found from performing a scan.\n" + "See the 'scan' command on how to perform a scan.\n\n" + diff --git a/cmd/root.go b/cmd/root.go index 3b0d4f0..ae43a53 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -44,6 +44,7 @@ var ( verbose bool debug bool forceUpdate bool + insecure bool ) // The `root` command doesn't do anything on it's own except display diff --git a/cmd/secrets.go b/cmd/secrets.go new file mode 100644 index 0000000..cb6a191 --- /dev/null +++ b/cmd/secrets.go @@ -0,0 +1,273 @@ +package cmd + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/OpenCHAMI/magellan/pkg/secrets" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + secretsFile string + secretsStoreFormat string + secretsStoreInputFile string +) + +var secretsCmd = &cobra.Command{ + Use: "secrets", + Short: "Manage credentials for BMC nodes", + Long: "Manage credentials for BMC nodes to for querying information through redfish. This requires generating a key and setting the 'MASTER_KEY' environment variable for the secrets store.\n" + + "Examples:\n\n" + + " export MASTER_KEY=$(magellan secrets generatekey)\n" + + // store specific BMC node creds for `collect` and `crawl` in default secrets store (`--file/-f`` flag not set) + " magellan secrets store $bmc_host $bmc_creds" + + // retrieve creds from secrets store + " magellan secrets retrieve $bmc_host -f nodes.json" + + // list creds from specific secrets + " magellan secrets list -f nodes.json", + Run: func(cmd *cobra.Command, args []string) { + // show command help and exit + if len(args) < 1 { + cmd.Help() + os.Exit(0) + } + }, +} + +var secretsGenerateKeyCmd = &cobra.Command{ + Use: "generatekey", + Args: cobra.NoArgs, + Short: "Generates a new 32-byte master key (in hex).", + Run: func(cmd *cobra.Command, args []string) { + key, err := secrets.GenerateMasterKey() + if err != nil { + fmt.Printf("Error generating master key: %v\n", err) + os.Exit(1) + } + fmt.Printf("%s\n", key) + }, +} + +var secretsStoreCmd = &cobra.Command{ + Use: "store secretID ", + Args: cobra.MinimumNArgs(1), + Short: "Stores the given string value under secretID.", + Run: func(cmd *cobra.Command, args []string) { + var ( + secretID = args[0] + secretValue string + store secrets.SecretStore + inputFileBytes []byte + err error + ) + + // require either the args or input file + if len(args) < 1 && secretsStoreInputFile == "" { + log.Error().Msg("no input data or file") + os.Exit(1) + } else if len(args) > 1 && secretsStoreInputFile == "" { + // use args[1] here because args[0] is the secretID + secretValue = args[1] + } + + // handle input file format + switch secretsStoreFormat { + case "basic": // format: $username:$password + var ( + values []string + username string + password string + ) + // seperate username and password provided + values = strings.Split(secretValue, ":") + if len(values) != 2 { + log.Error().Msgf("expected 2 arguments in [username:password] format but got %d", len(values)) + os.Exit(1) + } + + // open secret store to save credentials + store, err = secrets.OpenStore(secretsFile) + if err != nil { + log.Error().Err(err).Msg("failed to open secrets store") + os.Exit(1) + } + + // extract username/password from input (for clarity) + username = values[0] + password = values[1] + + // create JSON formatted string from input + secretValue = fmt.Sprintf("{\"username\": \"%s\", \"password\": \"%s\"}", username, password) + + case "base64": // format: ($encoded_base64_string) + decoded, err := base64.StdEncoding.DecodeString(secretValue) + if err != nil { + log.Error().Err(err).Msg("error decoding base64 data") + os.Exit(1) + } + + // check the decoded string if it's a valid JSON and has creds + if !isValidCredsJSON(string(decoded)) { + log.Error().Err(err).Msg("value is not a valid JSON or is missing credentials") + os.Exit(1) + } + + store, err = secrets.OpenStore(secretsFile) + if err != nil { + log.Error().Err(err).Msg("failed to open secrets store") + os.Exit(1) + } + secretValue = string(decoded) + case "json": // format: {"username": $username, "password": $password} + // read input from file if set and override + if secretsStoreInputFile != "" { + if secretValue != "" { + log.Error().Msg("cannot use -i/--input-file with positional argument") + os.Exit(1) + } + inputFileBytes, err = os.ReadFile(secretsStoreInputFile) + if err != nil { + log.Error().Err(err).Msg("failed to read input file") + os.Exit(1) + } + secretValue = string(inputFileBytes) + } + + // make sure we have valid JSON with "username" and "password" properties + if !isValidCredsJSON(string(secretValue)) { + log.Error().Err(err).Msg("not a valid JSON or creds") + os.Exit(1) + } + store, err = secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + default: + log.Error().Msg("no input format set") + os.Exit(1) + } + + if err := store.StoreSecretByID(secretID, secretValue); err != nil { + fmt.Printf("Error storing secret: %v\n", err) + os.Exit(1) + } + }, +} + +func isValidCredsJSON(val string) bool { + var ( + valid = !json.Valid([]byte(val)) + creds map[string]string + err error + ) + err = json.Unmarshal([]byte(val), &creds) + if err != nil { + return false + } + _, valid = creds["username"] + _, valid = creds["password"] + return valid +} + +var secretsRetrieveCmd = &cobra.Command{ + Use: "retrieve secretID", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + var ( + secretID = args[0] + secretValue string + store secrets.SecretStore + err error + ) + + store, err = secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + secretValue, err = store.GetSecretByID(secretID) + if err != nil { + fmt.Printf("Error retrieving secret: %v\n", err) + os.Exit(1) + } + fmt.Printf("Secret for %s: %s\n", secretID, secretValue) + }, +} + +var secretsListCmd = &cobra.Command{ + Use: "list", + Args: cobra.ExactArgs(0), + Short: "Lists all the secret IDs and their values.", + Run: func(cmd *cobra.Command, args []string) { + store, err := secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + secrets, err := store.ListSecrets() + if err != nil { + fmt.Printf("Error listing secrets: %v\n", err) + os.Exit(1) + } + + for key := range secrets { + fmt.Printf("%s\n", key) + } + }, +} + +var secretsRemoveCmd = &cobra.Command{ + Use: "remove secretIDs...", + Args: cobra.MinimumNArgs(1), + Short: "Remove secrets by IDs from secret store.", + Run: func(cmd *cobra.Command, args []string) { + for _, secretID := range args { + // open secret store from file + store, err := secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + // remove secret from store by it's ID + err = store.RemoveSecretByID(secretID) + if err != nil { + fmt.Println("failed to remove secret: ", err) + os.Exit(1) + } + + // update store by saving to original file + secrets.SaveSecrets(secretsFile, store.(*secrets.LocalSecretStore).Secrets) + } + }, +} + +func init() { + secretsCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "set the secrets file with BMC credentials") + secretsStoreCmd.Flags().StringVarP(&secretsStoreFormat, "format", "F", "basic", "set the input format for the secrets file (basic|json|base64)") + secretsStoreCmd.Flags().StringVarP(&secretsStoreInputFile, "input-file", "i", "", "set the file to read as input") + + secretsCmd.AddCommand(secretsGenerateKeyCmd) + secretsCmd.AddCommand(secretsStoreCmd) + secretsCmd.AddCommand(secretsRetrieveCmd) + secretsCmd.AddCommand(secretsListCmd) + secretsCmd.AddCommand(secretsRemoveCmd) + + rootCmd.AddCommand(secretsCmd) + + checkBindFlagError(viper.BindPFlags(secretsCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsStoreCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags())) +} diff --git a/go.mod b/go.mod index 9b83a07..01bcf2b 100644 --- a/go.mod +++ b/go.mod @@ -52,8 +52,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sys v0.28.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index cce3350..be886fb 100644 --- a/go.sum +++ b/go.sum @@ -123,10 +123,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= @@ -155,12 +151,6 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -176,8 +166,6 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/collect.go b/pkg/collect.go index ccb1a67..40ec851 100644 --- a/pkg/collect.go +++ b/pkg/collect.go @@ -41,6 +41,7 @@ type CollectParams struct { OutputPath string // set the path to save output with 'output' flag ForceUpdate bool // set whether to force updating SMD with 'force-update' flag AccessToken string // set the access token to include in request with 'access-token' flag + SecretsFile string // set the path to secrets file } // This is the main function used to collect information from the BMC nodes via Redfish. @@ -49,7 +50,7 @@ type CollectParams struct { // // Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency // property value between 1 and 10000. -func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secrets.SecretStore) ([]map[string]any, error) { +func CollectInventory(assets *[]RemoteAsset, params *CollectParams, localStore secrets.SecretStore) ([]map[string]any, error) { // check for available remote assets found from scan if assets == nil { return nil, fmt.Errorf("no assets found") @@ -69,6 +70,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret outputPath = path.Clean(params.OutputPath) smdClient = &client.SmdClient{Client: &http.Client{}} ) + // set the client's params from CLI // NOTE: temporary solution until client.NewClient() is fixed smdClient.URI = params.URI @@ -79,7 +81,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret } certPool := x509.NewCertPool() certPool.AppendCertsFromPEM(cacert) - smdClient.Client.Transport = &http.Transport{ + smdClient.Transport = &http.Transport{ TLSClientConfig: &tls.Config{ RootCAs: certPool, InsecureSkipVerify: true, @@ -105,25 +107,47 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret // generate custom xnames for bmcs // TODO: add xname customization via CLI - node := xnames.Node{ - Cabinet: 1000, - Chassis: 1, - ComputeModule: 7, - NodeBMC: offset, - } + var ( + uri = fmt.Sprintf("%s:%d", sr.Host, sr.Port) + node = xnames.Node{ + Cabinet: 1000, + Chassis: 1, + ComputeModule: 7, + NodeBMC: offset, + } + ) offset += 1 // crawl BMC node to fetch inventory data via Redfish var ( - systems []crawler.InventoryDetail - managers []crawler.Manager - config = crawler.CrawlerConfig{ - URI: fmt.Sprintf("%s:%d", sr.Host, sr.Port), - CredentialStore: store, + fallbackStore = secrets.NewStaticStore(params.Username, params.Password) + systems []crawler.InventoryDetail + managers []crawler.Manager + config = crawler.CrawlerConfig{ + URI: uri, + CredentialStore: localStore, Insecure: true, } + err error ) - systems, err := crawler.CrawlBMCForSystems(config) + + // determine if local store exists and has credentials for + // the provided secretID... + // if it does not, use the fallback static store instead with + // the username and password provided as arguments + if localStore != nil { + _, err := localStore.GetSecretByID(uri) + if err != nil { + log.Warn().Err(err).Msgf("could not retrieve secrets for %s...falling back to default provided credentials for user '%s'", uri, params.Username) + config.CredentialStore = fallbackStore + } + } else { + log.Warn().Msgf("invalid store for %s...falling back to default provided credentials for user '%s'", uri, params.Username) + config.CredentialStore = fallbackStore + } + + // crawl for node and BMC information + systems, err = crawler.CrawlBMCForSystems(config) if err != nil { log.Error().Err(err).Msg("failed to crawl BMC for systems") } diff --git a/pkg/crawler/main.go b/pkg/crawler/main.go index ddf6357..bd29fcb 100644 --- a/pkg/crawler/main.go +++ b/pkg/crawler/main.go @@ -148,7 +148,6 @@ func CrawlBMCForSystems(config CrawlerConfig) ([]InventoryDetail, error) { return walkSystems(rf_systems, nil, config.URI) } -// CrawlBMCForSystems pulls BMC manager information. // CrawlBMCForManagers connects to a BMC (Baseboard Management Controller) using the provided configuration, // retrieves the ServiceRoot, and then fetches the list of managers from the ServiceRoot. // @@ -374,6 +373,10 @@ func walkManagers(rf_managers []*redfish.Manager, baseURI string) ([]Manager, er } func loadBMCCreds(config CrawlerConfig) (BMCUsernamePassword, error) { + // NOTE: it is possible for the SecretStore to be nil, so we need a check + if config.CredentialStore == nil { + return BMCUsernamePassword{}, fmt.Errorf("credential store is invalid") + } creds, err := config.CredentialStore.GetSecretByID(config.URI) if err != nil { event := log.Error() diff --git a/pkg/secrets/example/main.go b/pkg/secrets/example/main.go new file mode 100644 index 0000000..52ab649 --- /dev/null +++ b/pkg/secrets/example/main.go @@ -0,0 +1,212 @@ +package main + +// This example demonstrates the usage of the LocalSecretStore to store and retrieve secrets. +// It provides a command-line interface to generate a master key, store secrets, and retrieve them. +// The master key is assumed to be stored in the environment variable MASTER_KEY and while it can +// anything you want, we recommend a 32 bit key for AES-256 encryption. The master key is used +// as part of a Key Derivation Function (KDF) to generate a unique AES key for each secret. +// The algorithm of choice is HMAC-based Extract-and-Expand Key Derivation Function (HKDF). +// Each secret is separately encrypted using AES-GCM and stored in a JSON file. +// The JSON file is loaded into memory when the LocalSecretStore is created and saved back to the file +// when a secret is stored or removed. +// + +import ( + "encoding/base64" + "fmt" + "os" + + "github.com/OpenCHAMI/magellan/pkg/secrets" +) + +func usage() { + fmt.Println("Usage:") + fmt.Println(" go run main.go generatekey") + fmt.Println(" - Generates a new 32-byte master key (in hex).") + fmt.Println() + fmt.Println(" Export MASTER_KEY= to use the same key in the next commands.") + fmt.Println() + fmt.Println(" go run main.go store [filename]") + fmt.Println(" - Stores the given string value under secretID.") + fmt.Println() + fmt.Println(" go run main.go storebase64 [filename]") + fmt.Println(" - Decodes the base64-encoded string before storing.") + fmt.Println() + fmt.Println(" go run main.go storejson [filename]") + fmt.Println(" - Stores the provided JSON for the specified secretID.") + fmt.Println() + fmt.Println(" go run main.go retrieve [filename]") + fmt.Println(" - Retrieves and prints the secret value for the given secretID.") + fmt.Println() + fmt.Println(" go run main.go list [filename]") + fmt.Println(" - Lists all the secret IDs and their values.") + fmt.Println() +} + +// openStore tries to create or open the LocalSecretStore based on the environment +// variable MASTER_KEY. If not found, it prints an error. +func openStore(filename string) (*secrets.LocalSecretStore, error) { + masterKey := os.Getenv("MASTER_KEY") + if masterKey == "" { + return nil, fmt.Errorf("MASTER_KEY environment variable not set") + } + + store, err := secrets.NewLocalSecretStore(masterKey, filename, true) + if err != nil { + return nil, fmt.Errorf("cannot open secrets store: %v", err) + } + return store, nil +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(1) + } + + cmd := os.Args[1] + + switch cmd { + case "generatekey": + key, err := secrets.GenerateMasterKey() + if err != nil { + fmt.Printf("Error generating master key: %v\n", err) + os.Exit(1) + } + fmt.Printf("%s\n", key) + + case "store": + if len(os.Args) < 4 { + fmt.Println("Not enough arguments. Usage: go run main.go store [filename]") + os.Exit(1) + } + secretID := os.Args[2] + secretValue := os.Args[3] + filename := "mysecrets.json" + if len(os.Args) == 5 { + filename = os.Args[4] + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := store.StoreSecretByID(secretID, secretValue); err != nil { + fmt.Printf("Error storing secret: %v\n", err) + os.Exit(1) + } + fmt.Println("Secret stored successfully.") + + case "storebase64": + if len(os.Args) < 4 { + fmt.Println("Not enough arguments. Usage: go run main.go storebase64 [filename]") + os.Exit(1) + } + secretID := os.Args[2] + base64Value := os.Args[3] + filename := "mysecrets.json" + if len(os.Args) == 5 { + filename = os.Args[4] + } + + decoded, err := base64.StdEncoding.DecodeString(base64Value) + if err != nil { + fmt.Printf("Error decoding base64 data: %v\n", err) + os.Exit(1) + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := store.StoreSecretByID(secretID, string(decoded)); err != nil { + fmt.Printf("Error storing base64-decoded secret: %v\n", err) + os.Exit(1) + } + fmt.Println("Base64-decoded secret stored successfully.") + + case "storejson": + if len(os.Args) < 4 { + fmt.Println(`Not enough arguments. Usage: go run main.go storejson '{"key":"value"}' [filename]`) + os.Exit(1) + } + secretID := os.Args[2] + jsonValue := os.Args[3] + filename := "mysecrets.json" + if len(os.Args) == 5 { + filename = os.Args[4] + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := store.StoreSecretByID(secretID, jsonValue); err != nil { + fmt.Printf("Error storing JSON secret: %v\n", err) + os.Exit(1) + } + fmt.Println("JSON secret stored successfully.") + + case "retrieve": + if len(os.Args) < 3 { + fmt.Println("Not enough arguments. Usage: go run main.go retrieve [filename]") + os.Exit(1) + } + secretID := os.Args[2] + filename := "mysecrets.json" + if len(os.Args) == 4 { + filename = os.Args[3] + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + secretValue, err := store.GetSecretByID(secretID) + if err != nil { + fmt.Printf("Error retrieving secret: %v\n", err) + os.Exit(1) + } + fmt.Printf("Secret for %s: %s\n", secretID, secretValue) + + case "list": + if len(os.Args) < 2 { + fmt.Println("Not enough arguments. Usage: go run main.go list [filename]") + os.Exit(1) + } + + filename := "mysecrets.json" + if len(os.Args) == 3 { + filename = os.Args[2] + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + secrets, err := store.ListSecrets() + if err != nil { + fmt.Printf("Error listing secrets: %v\n", err) + os.Exit(1) + } + + fmt.Println("Secrets:") + for key, value := range secrets { + fmt.Printf("%s: %s\n", key, value) + } + + default: + usage() + } + +} diff --git a/pkg/secrets/localstore.go b/pkg/secrets/localstore.go index 76fd136..f426320 100644 --- a/pkg/secrets/localstore.go +++ b/pkg/secrets/localstore.go @@ -84,7 +84,7 @@ func (l *LocalSecretStore) StoreSecretByID(secretID, secret string) error { l.mu.Lock() l.Secrets[secretID] = encryptedSecret - err = saveSecrets(l.filename, l.Secrets) + err = SaveSecrets(l.filename, l.Secrets) l.mu.Unlock() return err } @@ -101,8 +101,40 @@ func (l *LocalSecretStore) ListSecrets() (map[string]string, error) { return secretsCopy, nil } +// RemoveSecretByID removes the specified secretID stored locally +func (l *LocalSecretStore) RemoveSecretByID(secretID string) error { + l.mu.RLock() + // Let user know if there was nothing to delete + _, err := l.GetSecretByID(secretID) + if err != nil { + return err + } + delete(l.Secrets, secretID) + l.mu.RUnlock() + return nil +} + +// openStore tries to create or open the LocalSecretStore based on the environment +// variable MASTER_KEY. If not found, it prints an error. +func OpenStore(filename string) (SecretStore, error) { + if filename == "" { + return nil, fmt.Errorf("path to secret store required") + } + + masterKey := os.Getenv("MASTER_KEY") + if masterKey == "" { + return nil, fmt.Errorf("MASTER_KEY environment variable not set") + } + + store, err := NewLocalSecretStore(masterKey, filename, true) + if err != nil { + return nil, fmt.Errorf("failed to create new local secret store: %v", err) + } + return store, nil +} + // Saves secrets back to the JSON file -func saveSecrets(jsonFile string, store map[string]string) error { +func SaveSecrets(jsonFile string, store map[string]string) error { file, err := os.OpenFile(jsonFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err diff --git a/pkg/secrets/main.go b/pkg/secrets/main.go index 5925d53..5cb7f95 100644 --- a/pkg/secrets/main.go +++ b/pkg/secrets/main.go @@ -4,4 +4,5 @@ type SecretStore interface { GetSecretByID(secretID string) (string, error) StoreSecretByID(secretID, secret string) error ListSecrets() (map[string]string, error) + RemoveSecretByID(secretID string) error } diff --git a/pkg/secrets/staticstore.go b/pkg/secrets/staticstore.go index 3e77870..40d9049 100644 --- a/pkg/secrets/staticstore.go +++ b/pkg/secrets/staticstore.go @@ -18,11 +18,18 @@ func NewStaticStore(username, password string) *StaticStore { func (s *StaticStore) GetSecretByID(secretID string) (string, error) { return fmt.Sprintf(`{"username":"%s","password":"%s"}`, s.Username, s.Password), nil } + func (s *StaticStore) StoreSecretByID(secretID, secret string) error { return nil } + func (s *StaticStore) ListSecrets() (map[string]string, error) { return map[string]string{ "static_creds": fmt.Sprintf(`{"username":"%s","password":"%s"}`, s.Username, s.Password), }, nil } + +func (s *StaticStore) RemoveSecretByID(secretID string) error { + // Nothing to do here, since nothing is being stored. With different implementations, we could return an error when no secret is found for a specific ID. + return nil +} diff --git a/tests/compatibility_test.go b/tests/compatibility_test.go index dfcc5e5..f428f6a 100644 --- a/tests/compatibility_test.go +++ b/tests/compatibility_test.go @@ -78,7 +78,7 @@ func TestRedfishV1ServiceRootAvailability(t *testing.T) { // Simple test to ensure an expected Redfish version minimum requirement. func TestRedfishV1Version(t *testing.T) { var ( - url string = fmt.Sprintf("%s/redfish/v1/", *host) + url = fmt.Sprintf("%s/redfish/v1/", *host) body client.HTTPBody = []byte{} headers client.HTTPHeader = map[string]string{} testClient = &http.Client{