From a1276360fe624508fcaf3a6e58ae4d8cd4873095 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 14 Mar 2025 13:28:47 -0600 Subject: [PATCH 01/34] refactor: added function to open secrets store by checking env var --- pkg/secrets/example/main.go | 212 ++++++++++++++++++++++++++++++++++++ pkg/secrets/localstore.go | 15 +++ pkg/secrets/staticstore.go | 2 + 3 files changed, 229 insertions(+) create mode 100644 pkg/secrets/example/main.go diff --git a/pkg/secrets/example/main.go b/pkg/secrets/example/main.go new file mode 100644 index 0000000..52ab649 --- /dev/null +++ b/pkg/secrets/example/main.go @@ -0,0 +1,212 @@ +package main + +// This example demonstrates the usage of the LocalSecretStore to store and retrieve secrets. +// It provides a command-line interface to generate a master key, store secrets, and retrieve them. +// The master key is assumed to be stored in the environment variable MASTER_KEY and while it can +// anything you want, we recommend a 32 bit key for AES-256 encryption. The master key is used +// as part of a Key Derivation Function (KDF) to generate a unique AES key for each secret. +// The algorithm of choice is HMAC-based Extract-and-Expand Key Derivation Function (HKDF). +// Each secret is separately encrypted using AES-GCM and stored in a JSON file. +// The JSON file is loaded into memory when the LocalSecretStore is created and saved back to the file +// when a secret is stored or removed. +// + +import ( + "encoding/base64" + "fmt" + "os" + + "github.com/OpenCHAMI/magellan/pkg/secrets" +) + +func usage() { + fmt.Println("Usage:") + fmt.Println(" go run main.go generatekey") + fmt.Println(" - Generates a new 32-byte master key (in hex).") + fmt.Println() + fmt.Println(" Export MASTER_KEY= to use the same key in the next commands.") + fmt.Println() + fmt.Println(" go run main.go store [filename]") + fmt.Println(" - Stores the given string value under secretID.") + fmt.Println() + fmt.Println(" go run main.go storebase64 [filename]") + fmt.Println(" - Decodes the base64-encoded string before storing.") + fmt.Println() + fmt.Println(" go run main.go storejson [filename]") + fmt.Println(" - Stores the provided JSON for the specified secretID.") + fmt.Println() + fmt.Println(" go run main.go retrieve [filename]") + fmt.Println(" - Retrieves and prints the secret value for the given secretID.") + fmt.Println() + fmt.Println(" go run main.go list [filename]") + fmt.Println(" - Lists all the secret IDs and their values.") + fmt.Println() +} + +// openStore tries to create or open the LocalSecretStore based on the environment +// variable MASTER_KEY. If not found, it prints an error. +func openStore(filename string) (*secrets.LocalSecretStore, error) { + masterKey := os.Getenv("MASTER_KEY") + if masterKey == "" { + return nil, fmt.Errorf("MASTER_KEY environment variable not set") + } + + store, err := secrets.NewLocalSecretStore(masterKey, filename, true) + if err != nil { + return nil, fmt.Errorf("cannot open secrets store: %v", err) + } + return store, nil +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(1) + } + + cmd := os.Args[1] + + switch cmd { + case "generatekey": + key, err := secrets.GenerateMasterKey() + if err != nil { + fmt.Printf("Error generating master key: %v\n", err) + os.Exit(1) + } + fmt.Printf("%s\n", key) + + case "store": + if len(os.Args) < 4 { + fmt.Println("Not enough arguments. Usage: go run main.go store [filename]") + os.Exit(1) + } + secretID := os.Args[2] + secretValue := os.Args[3] + filename := "mysecrets.json" + if len(os.Args) == 5 { + filename = os.Args[4] + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := store.StoreSecretByID(secretID, secretValue); err != nil { + fmt.Printf("Error storing secret: %v\n", err) + os.Exit(1) + } + fmt.Println("Secret stored successfully.") + + case "storebase64": + if len(os.Args) < 4 { + fmt.Println("Not enough arguments. Usage: go run main.go storebase64 [filename]") + os.Exit(1) + } + secretID := os.Args[2] + base64Value := os.Args[3] + filename := "mysecrets.json" + if len(os.Args) == 5 { + filename = os.Args[4] + } + + decoded, err := base64.StdEncoding.DecodeString(base64Value) + if err != nil { + fmt.Printf("Error decoding base64 data: %v\n", err) + os.Exit(1) + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := store.StoreSecretByID(secretID, string(decoded)); err != nil { + fmt.Printf("Error storing base64-decoded secret: %v\n", err) + os.Exit(1) + } + fmt.Println("Base64-decoded secret stored successfully.") + + case "storejson": + if len(os.Args) < 4 { + fmt.Println(`Not enough arguments. Usage: go run main.go storejson '{"key":"value"}' [filename]`) + os.Exit(1) + } + secretID := os.Args[2] + jsonValue := os.Args[3] + filename := "mysecrets.json" + if len(os.Args) == 5 { + filename = os.Args[4] + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := store.StoreSecretByID(secretID, jsonValue); err != nil { + fmt.Printf("Error storing JSON secret: %v\n", err) + os.Exit(1) + } + fmt.Println("JSON secret stored successfully.") + + case "retrieve": + if len(os.Args) < 3 { + fmt.Println("Not enough arguments. Usage: go run main.go retrieve [filename]") + os.Exit(1) + } + secretID := os.Args[2] + filename := "mysecrets.json" + if len(os.Args) == 4 { + filename = os.Args[3] + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + secretValue, err := store.GetSecretByID(secretID) + if err != nil { + fmt.Printf("Error retrieving secret: %v\n", err) + os.Exit(1) + } + fmt.Printf("Secret for %s: %s\n", secretID, secretValue) + + case "list": + if len(os.Args) < 2 { + fmt.Println("Not enough arguments. Usage: go run main.go list [filename]") + os.Exit(1) + } + + filename := "mysecrets.json" + if len(os.Args) == 3 { + filename = os.Args[2] + } + + store, err := openStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + secrets, err := store.ListSecrets() + if err != nil { + fmt.Printf("Error listing secrets: %v\n", err) + os.Exit(1) + } + + fmt.Println("Secrets:") + for key, value := range secrets { + fmt.Printf("%s: %s\n", key, value) + } + + default: + usage() + } + +} diff --git a/pkg/secrets/localstore.go b/pkg/secrets/localstore.go index 76fd136..553a63a 100644 --- a/pkg/secrets/localstore.go +++ b/pkg/secrets/localstore.go @@ -101,6 +101,21 @@ func (l *LocalSecretStore) ListSecrets() (map[string]string, error) { return secretsCopy, nil } +// openStore tries to create or open the LocalSecretStore based on the environment +// variable MASTER_KEY. If not found, it prints an error. +func OpenStore(filename string) (SecretStore, error) { + masterKey := os.Getenv("MASTER_KEY") + if masterKey == "" { + return nil, fmt.Errorf("MASTER_KEY environment variable not set") + } + + store, err := NewLocalSecretStore(masterKey, filename, true) + if err != nil { + return nil, fmt.Errorf("cannot open secrets store: %v", err) + } + return store, nil +} + // Saves secrets back to the JSON file func saveSecrets(jsonFile string, store map[string]string) error { file, err := os.OpenFile(jsonFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) diff --git a/pkg/secrets/staticstore.go b/pkg/secrets/staticstore.go index 3e77870..85c2d68 100644 --- a/pkg/secrets/staticstore.go +++ b/pkg/secrets/staticstore.go @@ -18,9 +18,11 @@ func NewStaticStore(username, password string) *StaticStore { func (s *StaticStore) GetSecretByID(secretID string) (string, error) { return fmt.Sprintf(`{"username":"%s","password":"%s"}`, s.Username, s.Password), nil } + func (s *StaticStore) StoreSecretByID(secretID, secret string) error { return nil } + func (s *StaticStore) ListSecrets() (map[string]string, error) { return map[string]string{ "static_creds": fmt.Sprintf(`{"username":"%s","password":"%s"}`, s.Username, s.Password), From c3a7ebf975624ab6a227a78b5a61187afc8773a4 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 17 Mar 2025 10:14:36 -0600 Subject: [PATCH 02/34] chore: added pre-condition guards for secrets --- pkg/crawler/main.go | 5 ++++- pkg/secrets/localstore.go | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/crawler/main.go b/pkg/crawler/main.go index ddf6357..bd29fcb 100644 --- a/pkg/crawler/main.go +++ b/pkg/crawler/main.go @@ -148,7 +148,6 @@ func CrawlBMCForSystems(config CrawlerConfig) ([]InventoryDetail, error) { return walkSystems(rf_systems, nil, config.URI) } -// CrawlBMCForSystems pulls BMC manager information. // CrawlBMCForManagers connects to a BMC (Baseboard Management Controller) using the provided configuration, // retrieves the ServiceRoot, and then fetches the list of managers from the ServiceRoot. // @@ -374,6 +373,10 @@ func walkManagers(rf_managers []*redfish.Manager, baseURI string) ([]Manager, er } func loadBMCCreds(config CrawlerConfig) (BMCUsernamePassword, error) { + // NOTE: it is possible for the SecretStore to be nil, so we need a check + if config.CredentialStore == nil { + return BMCUsernamePassword{}, fmt.Errorf("credential store is invalid") + } creds, err := config.CredentialStore.GetSecretByID(config.URI) if err != nil { event := log.Error() diff --git a/pkg/secrets/localstore.go b/pkg/secrets/localstore.go index 553a63a..1cf862a 100644 --- a/pkg/secrets/localstore.go +++ b/pkg/secrets/localstore.go @@ -104,6 +104,10 @@ func (l *LocalSecretStore) ListSecrets() (map[string]string, error) { // openStore tries to create or open the LocalSecretStore based on the environment // variable MASTER_KEY. If not found, it prints an error. func OpenStore(filename string) (SecretStore, error) { + if filename == "" { + return nil, fmt.Errorf("no path to secret store provided") + } + masterKey := os.Getenv("MASTER_KEY") if masterKey == "" { return nil, fmt.Errorf("MASTER_KEY environment variable not set") From 5ecc051fef344a3cac5ec8637bf228649e846dac Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 17 Mar 2025 10:34:53 -0600 Subject: [PATCH 03/34] refactor: added optional secrets file parameter and lookup in collect --- pkg/collect.go | 55 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/pkg/collect.go b/pkg/collect.go index ccb1a67..07591d5 100644 --- a/pkg/collect.go +++ b/pkg/collect.go @@ -41,6 +41,7 @@ type CollectParams struct { OutputPath string // set the path to save output with 'output' flag ForceUpdate bool // set whether to force updating SMD with 'force-update' flag AccessToken string // set the access token to include in request with 'access-token' flag + SecretsFile string // set the path to secrets file } // This is the main function used to collect information from the BMC nodes via Redfish. @@ -60,15 +61,18 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret // collect bmc information asynchronously var ( - offset = 0 - wg sync.WaitGroup - collection = make([]map[string]any, 0) - found = make([]string, 0, len(*assets)) - done = make(chan struct{}, params.Concurrency+1) - chanAssets = make(chan RemoteAsset, params.Concurrency+1) - outputPath = path.Clean(params.OutputPath) - smdClient = &client.SmdClient{Client: &http.Client{}} + offset = 0 + wg sync.WaitGroup + collection = make([]map[string]any, 0) + found = make([]string, 0, len(*assets)) + done = make(chan struct{}, params.Concurrency+1) + chanAssets = make(chan RemoteAsset, params.Concurrency+1) + outputPath = path.Clean(params.OutputPath) + smdClient = &client.SmdClient{Client: &http.Client{}} + initialStore secrets.SecretStore = store + err error ) + // set the client's params from CLI // NOTE: temporary solution until client.NewClient() is fixed smdClient.URI = params.URI @@ -103,27 +107,46 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret return } + // use initial store to check for creds for specific node + store = initialStore + // generate custom xnames for bmcs // TODO: add xname customization via CLI - node := xnames.Node{ - Cabinet: 1000, - Chassis: 1, - ComputeModule: 7, - NodeBMC: offset, - } + var ( + uri = fmt.Sprintf("%s:%d", sr.Host, sr.Port) + node = xnames.Node{ + Cabinet: 1000, + Chassis: 1, + ComputeModule: 7, + NodeBMC: offset, + } + ) offset += 1 + // determine if local store exists and has credentials for + // the provided secretID... + // if it does not, create a static store and use the username + // and password provided instead + _, err = store.GetSecretByID(uri) + if store == nil || err != nil { + log.Warn().Err(err).Msgf("could not retrieve secrets for %s...falling back to default provided credentials", uri) + store = secrets.NewStaticStore(params.Username, params.Password) + } + // crawl BMC node to fetch inventory data via Redfish var ( systems []crawler.InventoryDetail managers []crawler.Manager config = crawler.CrawlerConfig{ - URI: fmt.Sprintf("%s:%d", sr.Host, sr.Port), + URI: uri, CredentialStore: store, Insecure: true, } + err error ) - systems, err := crawler.CrawlBMCForSystems(config) + + // crawl for node and BMC information + systems, err = crawler.CrawlBMCForSystems(config) if err != nil { log.Error().Err(err).Msg("failed to crawl BMC for systems") } From f18d279468d18351820be5dfa3b8adc23e244297 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 17 Mar 2025 10:37:04 -0600 Subject: [PATCH 04/34] refactor: updated description/example and added 'secrets-file' flag to cmd --- cmd/collect.go | 48 +++++++++----- cmd/crawl.go | 4 +- cmd/root.go | 1 + cmd/secrets.go | 174 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 17 deletions(-) create mode 100644 cmd/secrets.go diff --git a/cmd/collect.go b/cmd/collect.go index c304beb..ed80ea1 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -25,7 +25,11 @@ var CollectCmd = &cobra.Command{ "See the 'scan' command on how to perform a scan.\n\n" + "Examples:\n" + " magellan collect --cache ./assets.db --output ./logs --timeout 30 --cacert cecert.pem\n" + - " magellan collect --host smd.example.com --port 27779 --username username --password password", + " magellan collect --host smd.example.com --port 27779 --username $username --password $password\n\n" + + // example using `collect` + " export MASTER_KEY=$(magellan secrets generatekey)\n" + + " magellan secrets store $node_creds_json -f nodes.json" + + " magellan collect --host openchami.cluster --username $username --password $password \\\n", Run: func(cmd *cobra.Command, args []string) { // get probe states stored in db from scan scannedResults, err := sqlite.GetScannedAssets(cachePath) @@ -48,17 +52,13 @@ var CollectCmd = &cobra.Command{ } } - if verbose { - log.Debug().Str("Access Token", accessToken) - } - - // + // set the minimum/maximum number of concurrent processes if concurrency <= 0 { concurrency = mathutil.Clamp(len(scannedResults), 1, 10000) } - // Create a StaticSecretStore to hold the username and password - secrets := secrets.NewStaticStore(username, password) - _, err = magellan.CollectInventory(&scannedResults, &magellan.CollectParams{ + + // set the collect parameters from CLI params + params := &magellan.CollectParams{ URI: host, Timeout: timeout, Concurrency: concurrency, @@ -67,9 +67,26 @@ var CollectCmd = &cobra.Command{ OutputPath: outputPath, ForceUpdate: forceUpdate, AccessToken: accessToken, - }, secrets) + } + + // show all of the 'collect' parameters being set from CLI if verbose + if verbose { + log.Debug().Any("params", params) + } + + // load the secrets file to get node credentials by ID (i.e. the BMC node's URI) + store, err := secrets.OpenStore(params.SecretsFile) if err != nil { - log.Error().Err(err).Msgf("failed to collect data") + // Something went wrong with the store so try using + // Create a StaticSecretStore to hold the username and password + fmt.Println(err) + store = secrets.NewStaticStore(username, password) + } else { + } + + _, err = magellan.CollectInventory(&scannedResults, params, store) + if err != nil { + log.Error().Err(err).Msg("failed to collect data") } }, } @@ -77,13 +94,14 @@ var CollectCmd = &cobra.Command{ func init() { currentUser, _ = user.Current() CollectCmd.PersistentFlags().StringVar(&host, "host", "", "Set the URI to the SMD root endpoint") - CollectCmd.PersistentFlags().StringVar(&username, "username", "", "Set the BMC user") - CollectCmd.PersistentFlags().StringVar(&password, "password", "", "Set the BMC password") - CollectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the scheme used to query") + CollectCmd.PersistentFlags().StringVar(&username, "username", "", "Set the master BMC username") + CollectCmd.PersistentFlags().StringVar(&password, "password", "", "Set the master BMC password") + CollectCmd.PersistentFlags().StringVar(&secretsFile, "secrets-file", "", "Set path to the node secrets file") + CollectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the default scheme used to query when not included in URI") CollectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "Set the protocol used to query") CollectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "Set the path to store collection data") CollectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "Set flag to force update data sent to SMD") - CollectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "Path to CA cert. (defaults to system CAs)") + CollectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "Set the path to CA cert file. (defaults to system CAs when blank)") // set flags to only be used together CollectCmd.MarkFlagsRequiredTogether("username", "password") diff --git a/cmd/crawl.go b/cmd/crawl.go index e9e91bd..55757ce 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -18,8 +18,8 @@ import ( var CrawlCmd = &cobra.Command{ Use: "crawl [uri]", Short: "Crawl a single BMC for inventory information", - Long: "Crawl a single BMC for inventory information. This command does NOT store information\n" + - "about the scan into cache after completion. To do so, use the 'collect' command instead\n\n" + + Long: "Crawl a single BMC for inventory information with URI. This command does NOT scan subnets nor store scan information\n" + + "in cache after completion. To do so, use the 'collect' command instead\n\n" + "Examples:\n" + " magellan crawl https://bmc.example.com\n" + " magellan crawl https://bmc.example.com -i -u username -p password", diff --git a/cmd/root.go b/cmd/root.go index 3b0d4f0..6ef162b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -38,6 +38,7 @@ var ( cacertPath string username string password string + secretsFile string cachePath string outputPath string configPath string diff --git a/cmd/secrets.go b/cmd/secrets.go new file mode 100644 index 0000000..2ca718c --- /dev/null +++ b/cmd/secrets.go @@ -0,0 +1,174 @@ +package cmd + +import ( + "encoding/base64" + "fmt" + "os" + + "github.com/OpenCHAMI/magellan/pkg/secrets" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var secretsCmd = &cobra.Command{ + Use: "secrets", + Short: "Manage credentials for BMC nodes", + Long: "Manage credentials for BMC nodes to for querying information through redfish. This requires generating a key and setting the 'MASTER_KEY' environment variable for the secrets store.\n" + + "Examples:\n\n" + + " export MASTER_KEY=$(magellan secrets generatekey)\n" + + // store specific BMC node creds for `collect` and `crawl` in default secrets store (`--file/-f`` flag not set) + " magellan secrets store $bmc_host $bmc_creds" + + // retrieve creds from secrets store + " magellan secrets retrieve $bmc_host -f nodes.json" + + // list creds from specific secrets + " magellan secrets list -f nodes.json", + Run: func(cmd *cobra.Command, args []string) { + // show command help and exit + if len(args) < 1 { + cmd.Help() + os.Exit(0) + } + }, +} + +var secretsGenerateKeyCmd = &cobra.Command{ + Use: "generatekey", + Short: "Generates a new 32-byte master key (in hex).", + Run: func(cmd *cobra.Command, args []string) { + key, err := secrets.GenerateMasterKey() + if err != nil { + fmt.Printf("Error generating master key: %v\n", err) + os.Exit(1) + } + fmt.Printf("%s\n", key) + }, +} + +var secretsStoreCmd = &cobra.Command{ + Use: "store secretID secretValue", + Args: cobra.ExactArgs(2), + Short: "Stores the given string value under secretID.", + Run: func(cmd *cobra.Command, args []string) { + var ( + secretID = args[0] + secretValue = args[1] + ) + + store, err := secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := store.StoreSecretByID(secretID, secretValue); err != nil { + fmt.Printf("Error storing secret: %v\n", err) + os.Exit(1) + } + fmt.Println("Secret stored successfully.") + }, +} + +var secretsStoreBase64Cmd = &cobra.Command{ + Use: "storebase64 base64String", + Args: cobra.ExactArgs(1), + Short: "Decodes the base64-encoded string before storing.", + Run: func(cmd *cobra.Command, args []string) { + if len(os.Args) < 4 { + fmt.Println("Not enough arguments. Usage: go run main.go storebase64 [filename]") + os.Exit(1) + } + secretID := os.Args[2] + base64Value := os.Args[3] + filename := "mysecrets.json" + if len(os.Args) == 5 { + filename = os.Args[4] + } + + decoded, err := base64.StdEncoding.DecodeString(base64Value) + if err != nil { + fmt.Printf("Error decoding base64 data: %v\n", err) + os.Exit(1) + } + + store, err := secrets.OpenStore(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := store.StoreSecretByID(secretID, string(decoded)); err != nil { + fmt.Printf("Error storing base64-decoded secret: %v\n", err) + os.Exit(1) + } + fmt.Println("Base64-decoded secret stored successfully.") + }, +} + +var secretsRetrieveCmd = &cobra.Command{ + Use: "retrieve secretID", + Run: func(cmd *cobra.Command, args []string) { + if len(os.Args) < 3 { + fmt.Println("Not enough arguments. Usage: go run main.go retrieve [filename]") + os.Exit(1) + } + secretID := os.Args[2] + + store, err := secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + secretValue, err := store.GetSecretByID(secretID) + if err != nil { + fmt.Printf("Error retrieving secret: %v\n", err) + os.Exit(1) + } + fmt.Printf("Secret for %s: %s\n", secretID, secretValue) + }, +} + +var secretsListCmd = &cobra.Command{ + Use: "list", + Short: "Lists all the secret IDs and their values.", + Run: func(cmd *cobra.Command, args []string) { + if len(args) < 2 { + fmt.Println("Not enough arguments. Usage: go run main.go list [filename]") + os.Exit(1) + } + + store, err := secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + secrets, err := store.ListSecrets() + if err != nil { + fmt.Printf("Error listing secrets: %v\n", err) + os.Exit(1) + } + + fmt.Println("Secrets:") + for key, value := range secrets { + fmt.Printf("%s: %s\n", key, value) + } + }, +} + +func init() { + secretsCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "") + + secretsCmd.AddCommand(secretsGenerateKeyCmd) + secretsCmd.AddCommand(secretsStoreCmd) + secretsCmd.AddCommand(secretsStoreBase64Cmd) + secretsCmd.AddCommand(secretsRetrieveCmd) + secretsCmd.AddCommand(secretsListCmd) + + checkBindFlagError(viper.BindPFlags(secretsCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsStoreCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags())) + checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags())) +} From 73c3323cc82c4c8a733701be8c2282def2e84473 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 17 Mar 2025 10:37:18 -0600 Subject: [PATCH 05/34] chore: updated go deps --- go.mod | 2 -- go.sum | 12 ------------ 2 files changed, 14 deletions(-) diff --git a/go.mod b/go.mod index 9b83a07..01bcf2b 100644 --- a/go.mod +++ b/go.mod @@ -52,8 +52,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sys v0.28.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index cce3350..be886fb 100644 --- a/go.sum +++ b/go.sum @@ -123,10 +123,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= @@ -155,12 +151,6 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -176,8 +166,6 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 23bd31a1aaf941419c95da55964a3d549661b8f0 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 17 Mar 2025 11:39:23 -0600 Subject: [PATCH 06/34] feat: add 'secrets' command to root --- cmd/secrets.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/secrets.go b/cmd/secrets.go index 2ca718c..9fc05c4 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -165,6 +165,8 @@ func init() { secretsCmd.AddCommand(secretsRetrieveCmd) secretsCmd.AddCommand(secretsListCmd) + rootCmd.AddCommand(secretsCmd) + checkBindFlagError(viper.BindPFlags(secretsCmd.Flags())) checkBindFlagError(viper.BindPFlags(secretsGenerateKeyCmd.Flags())) checkBindFlagError(viper.BindPFlags(secretsStoreCmd.Flags())) From 5b79031afac317c2586111a751fb552a049be384 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 17 Mar 2025 18:34:08 -0600 Subject: [PATCH 07/34] refactor: updated secrets cmd implementation --- cmd/root.go | 1 - cmd/secrets.go | 152 +++++++++++++++++++++++++++++++------------------ 2 files changed, 97 insertions(+), 56 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 6ef162b..3b0d4f0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -38,7 +38,6 @@ var ( cacertPath string username string password string - secretsFile string cachePath string outputPath string configPath string diff --git a/cmd/secrets.go b/cmd/secrets.go index 9fc05c4..581c332 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -2,14 +2,22 @@ package cmd import ( "encoding/base64" + "encoding/json" "fmt" "os" "github.com/OpenCHAMI/magellan/pkg/secrets" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) +var ( + secretsFile string + secretsStoreFormat string + secretsStoreInputFile string +) + var secretsCmd = &cobra.Command{ Use: "secrets", Short: "Manage credentials for BMC nodes", @@ -33,6 +41,7 @@ var secretsCmd = &cobra.Command{ var secretsGenerateKeyCmd = &cobra.Command{ Use: "generatekey", + Args: cobra.NoArgs, Short: "Generates a new 32-byte master key (in hex).", Run: func(cmd *cobra.Command, args []string) { key, err := secrets.GenerateMasterKey() @@ -45,18 +54,73 @@ var secretsGenerateKeyCmd = &cobra.Command{ } var secretsStoreCmd = &cobra.Command{ - Use: "store secretID secretValue", - Args: cobra.ExactArgs(2), + Use: "store secretID ", + Args: cobra.MinimumNArgs(1), Short: "Stores the given string value under secretID.", Run: func(cmd *cobra.Command, args []string) { var ( - secretID = args[0] - secretValue = args[1] + secretID string = args[0] + secretValue string + store secrets.SecretStore + inputFileBytes []byte + err error ) - store, err := secrets.OpenStore(secretsFile) - if err != nil { - fmt.Println(err) + // require either the args or input file + if len(args) < 1 && secretsStoreInputFile == "" { + log.Error().Msg("no input data or file") + os.Exit(1) + } else if len(args) > 1 && secretsStoreInputFile == "" { + secretValue = args[1] + } + + switch secretsStoreFormat { + case "base64": + decoded, err := base64.StdEncoding.DecodeString(secretValue) + if err != nil { + fmt.Printf("Error decoding base64 data: %v\n", err) + os.Exit(1) + } + + // check the decoded string if it's a valid JSON and has creds + if !isValidCredsJSON(string(decoded)) { + log.Error().Msg("value is not a valid JSON or is missing credentials") + os.Exit(1) + } + + store, err = secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + secretValue = string(decoded) + case "json": + // read input from file if set and override + if secretsStoreInputFile != "" { + if secretValue != "" { + log.Error().Msg("cannot use -i/--input-file with positional argument") + os.Exit(1) + } + inputFileBytes, err = os.ReadFile(secretsStoreInputFile) + if err != nil { + log.Error().Err(err).Msg("failed to read input file") + os.Exit(1) + } + secretValue = string(inputFileBytes) + } + + // make sure we have valid JSON with "username" and "password" properties + if !isValidCredsJSON(string(secretValue)) { + log.Error().Err(err).Msg("not a valid JSON or creds") + os.Exit(1) + } + store, err = secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + default: + log.Error().Msg("no input format set") os.Exit(1) } @@ -68,58 +132,39 @@ var secretsStoreCmd = &cobra.Command{ }, } -var secretsStoreBase64Cmd = &cobra.Command{ - Use: "storebase64 base64String", - Args: cobra.ExactArgs(1), - Short: "Decodes the base64-encoded string before storing.", - Run: func(cmd *cobra.Command, args []string) { - if len(os.Args) < 4 { - fmt.Println("Not enough arguments. Usage: go run main.go storebase64 [filename]") - os.Exit(1) - } - secretID := os.Args[2] - base64Value := os.Args[3] - filename := "mysecrets.json" - if len(os.Args) == 5 { - filename = os.Args[4] - } - - decoded, err := base64.StdEncoding.DecodeString(base64Value) - if err != nil { - fmt.Printf("Error decoding base64 data: %v\n", err) - os.Exit(1) - } - - store, err := secrets.OpenStore(filename) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - if err := store.StoreSecretByID(secretID, string(decoded)); err != nil { - fmt.Printf("Error storing base64-decoded secret: %v\n", err) - os.Exit(1) - } - fmt.Println("Base64-decoded secret stored successfully.") - }, +func isValidCredsJSON(val string) bool { + var ( + valid bool = !json.Valid([]byte(val)) + creds map[string]string + err error + ) + err = json.Unmarshal([]byte(val), &creds) + if err != nil { + return false + } + _, valid = creds["username"] + _, valid = creds["password"] + return valid } var secretsRetrieveCmd = &cobra.Command{ - Use: "retrieve secretID", + Use: "retrieve secretID", + Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - if len(os.Args) < 3 { - fmt.Println("Not enough arguments. Usage: go run main.go retrieve [filename]") - os.Exit(1) - } - secretID := os.Args[2] + var ( + secretID = args[0] + secretValue string + store secrets.SecretStore + err error + ) - store, err := secrets.OpenStore(secretsFile) + store, err = secrets.OpenStore(secretsFile) if err != nil { fmt.Println(err) os.Exit(1) } - secretValue, err := store.GetSecretByID(secretID) + secretValue, err = store.GetSecretByID(secretID) if err != nil { fmt.Printf("Error retrieving secret: %v\n", err) os.Exit(1) @@ -130,13 +175,9 @@ var secretsRetrieveCmd = &cobra.Command{ var secretsListCmd = &cobra.Command{ Use: "list", + Args: cobra.MinimumNArgs(1), Short: "Lists all the secret IDs and their values.", Run: func(cmd *cobra.Command, args []string) { - if len(args) < 2 { - fmt.Println("Not enough arguments. Usage: go run main.go list [filename]") - os.Exit(1) - } - store, err := secrets.OpenStore(secretsFile) if err != nil { fmt.Println(err) @@ -158,10 +199,11 @@ var secretsListCmd = &cobra.Command{ func init() { secretsCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "") + secretsStoreCmd.Flags().StringVar(&secretsStoreFormat, "format", "json", "set the input format for the secrets file (json|base64)") + secretsStoreCmd.Flags().StringVarP(&secretsStoreInputFile, "input-file", "i", "", "set the file to read as input") secretsCmd.AddCommand(secretsGenerateKeyCmd) secretsCmd.AddCommand(secretsStoreCmd) - secretsCmd.AddCommand(secretsStoreBase64Cmd) secretsCmd.AddCommand(secretsRetrieveCmd) secretsCmd.AddCommand(secretsListCmd) From 932daeafe1b7bf0f6e4fe64d22964d3fb6c081f1 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 19 Mar 2025 11:10:26 -0600 Subject: [PATCH 08/34] refactor: added func to remove secrets from store --- cmd/secrets.go | 18 ++++++++++++++++++ pkg/secrets/localstore.go | 5 +++++ pkg/secrets/main.go | 1 + 3 files changed, 24 insertions(+) diff --git a/cmd/secrets.go b/cmd/secrets.go index 581c332..ab838a2 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -197,6 +197,23 @@ var secretsListCmd = &cobra.Command{ }, } +var secretsRemoveCmd = &cobra.Command{ + Use: "remove secretIDs...", + Args: cobra.MinimumNArgs(2), + Short: "Remove secrets by IDs from secret store.", + Run: func(cmd *cobra.Command, args []string) { + for _, secretID := range args { + store, err := secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + store.RemoveSecretByID(secretID) + } + }, +} + func init() { secretsCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "") secretsStoreCmd.Flags().StringVar(&secretsStoreFormat, "format", "json", "set the input format for the secrets file (json|base64)") @@ -206,6 +223,7 @@ func init() { secretsCmd.AddCommand(secretsStoreCmd) secretsCmd.AddCommand(secretsRetrieveCmd) secretsCmd.AddCommand(secretsListCmd) + secretsCmd.AddCommand(secretsRemoveCmd) rootCmd.AddCommand(secretsCmd) diff --git a/pkg/secrets/localstore.go b/pkg/secrets/localstore.go index 1cf862a..e87a7cc 100644 --- a/pkg/secrets/localstore.go +++ b/pkg/secrets/localstore.go @@ -101,6 +101,11 @@ func (l *LocalSecretStore) ListSecrets() (map[string]string, error) { return secretsCopy, nil } +// RemoveSecretByID removes the specified secretID stored locally +func (l *LocalSecretStore) RemoveSecretByID(secretID string) { + delete(l.Secrets, secretID) +} + // openStore tries to create or open the LocalSecretStore based on the environment // variable MASTER_KEY. If not found, it prints an error. func OpenStore(filename string) (SecretStore, error) { diff --git a/pkg/secrets/main.go b/pkg/secrets/main.go index 5925d53..6d27976 100644 --- a/pkg/secrets/main.go +++ b/pkg/secrets/main.go @@ -4,4 +4,5 @@ type SecretStore interface { GetSecretByID(secretID string) (string, error) StoreSecretByID(secretID, secret string) error ListSecrets() (map[string]string, error) + RemoveSecretByID(secretID string) } From a8f0c125054fd051e15d6bc1a77f303e002a1977 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 19 Mar 2025 11:36:29 -0600 Subject: [PATCH 09/34] fix: added missing funcs for secret store implementations --- pkg/secrets/localstore.go | 2 ++ pkg/secrets/staticstore.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/pkg/secrets/localstore.go b/pkg/secrets/localstore.go index e87a7cc..1ae9b91 100644 --- a/pkg/secrets/localstore.go +++ b/pkg/secrets/localstore.go @@ -103,7 +103,9 @@ func (l *LocalSecretStore) ListSecrets() (map[string]string, error) { // RemoveSecretByID removes the specified secretID stored locally func (l *LocalSecretStore) RemoveSecretByID(secretID string) { + l.mu.RLock() delete(l.Secrets, secretID) + l.mu.RUnlock() } // openStore tries to create or open the LocalSecretStore based on the environment diff --git a/pkg/secrets/staticstore.go b/pkg/secrets/staticstore.go index 85c2d68..405f58b 100644 --- a/pkg/secrets/staticstore.go +++ b/pkg/secrets/staticstore.go @@ -28,3 +28,7 @@ func (s *StaticStore) ListSecrets() (map[string]string, error) { "static_creds": fmt.Sprintf(`{"username":"%s","password":"%s"}`, s.Username, s.Password), }, nil } + +func (s *StaticStore) RemoveSecretByID(secretID string) { + // Nothing to do here, since nothing is being stored +} From ceeaa5d891ffc8b10d47657674b973b6e75d8a6b Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 19 Mar 2025 15:31:24 -0600 Subject: [PATCH 10/34] refactor: minor changes to store in collect --- pkg/collect.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/collect.go b/pkg/collect.go index 07591d5..a83c730 100644 --- a/pkg/collect.go +++ b/pkg/collect.go @@ -70,7 +70,6 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret outputPath = path.Clean(params.OutputPath) smdClient = &client.SmdClient{Client: &http.Client{}} initialStore secrets.SecretStore = store - err error ) // set the client's params from CLI @@ -127,9 +126,14 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret // the provided secretID... // if it does not, create a static store and use the username // and password provided instead - _, err = store.GetSecretByID(uri) - if store == nil || err != nil { - log.Warn().Err(err).Msgf("could not retrieve secrets for %s...falling back to default provided credentials", uri) + if store != nil { + _, err := store.GetSecretByID(uri) + if err != nil { + log.Warn().Err(err).Msgf("could not retrieve secrets for %s...falling back to default provided credentials", uri) + store = secrets.NewStaticStore(params.Username, params.Password) + } + } else { + log.Warn().Msgf("invalid store...falling back to default provided credentials for %s", uri) store = secrets.NewStaticStore(params.Username, params.Password) } From 053773d90cb8022035afe41d9360570c8d7512fa Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 19 Mar 2025 15:46:53 -0600 Subject: [PATCH 11/34] fix: added secrets file path to collect parameters --- cmd/collect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/collect.go b/cmd/collect.go index ed80ea1..f643636 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -67,6 +67,7 @@ var CollectCmd = &cobra.Command{ OutputPath: outputPath, ForceUpdate: forceUpdate, AccessToken: accessToken, + SecretsFile: secretsFile, } // show all of the 'collect' parameters being set from CLI if verbose @@ -81,7 +82,6 @@ var CollectCmd = &cobra.Command{ // Create a StaticSecretStore to hold the username and password fmt.Println(err) store = secrets.NewStaticStore(username, password) - } else { } _, err = magellan.CollectInventory(&scannedResults, params, store) From d428dbfd27f68ff1e6acbcea24e475eb3a3fae45 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 20 Mar 2025 08:58:06 -0600 Subject: [PATCH 12/34] refactor: added exact number of args to list cmd --- cmd/list.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/list.go b/cmd/list.go index d760501..9fbc361 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -21,6 +21,7 @@ var ( // is what is consumed by the `collect` command with the --cache flag. var ListCmd = &cobra.Command{ Use: "list", + Args: cobra.ExactArgs(0), Short: "List information stored in cache from a scan", Long: "Prints all of the host and associated data found from performing a scan.\n" + "See the 'scan' command on how to perform a scan.\n\n" + From df77e075ef9e896e78ecd0c8049a0ea30347dcb0 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 20 Mar 2025 08:59:15 -0600 Subject: [PATCH 13/34] refactor: changed required number of args for secrets list --- cmd/secrets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/secrets.go b/cmd/secrets.go index ab838a2..b75e4e7 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -175,7 +175,7 @@ var secretsRetrieveCmd = &cobra.Command{ var secretsListCmd = &cobra.Command{ Use: "list", - Args: cobra.MinimumNArgs(1), + Args: cobra.ExactArgs(0), Short: "Lists all the secret IDs and their values.", Run: func(cmd *cobra.Command, args []string) { store, err := secrets.OpenStore(secretsFile) From 41346aebbb9d7b54a27c6e9d137c24ea5b248fad Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 20 Mar 2025 08:59:48 -0600 Subject: [PATCH 14/34] refactor: changed to use local store with static store fallback --- cmd/crawl.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/crawl.go b/cmd/crawl.go index 55757ce..3a3c4e5 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -36,13 +36,17 @@ var CrawlCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { - staticStore := &secrets.StaticStore{ - Username: viper.GetString("crawl.username"), - Password: viper.GetString("crawl.password"), + store, err := secrets.OpenStore(secretsFile) + if err != nil { + fmt.Println(err) + store = &secrets.StaticStore{ + Username: viper.GetString("crawl.username"), + Password: viper.GetString("crawl.password"), + } } systems, err := crawler.CrawlBMCForSystems(crawler.CrawlerConfig{ URI: args[0], - CredentialStore: staticStore, + CredentialStore: store, Insecure: cmd.Flag("insecure").Value.String() == "true", }) if err != nil { From 7990ec097df824f544a70f2be6673fe3faf81a29 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 20 Mar 2025 09:11:21 -0600 Subject: [PATCH 15/34] refactor: added flag to set secrets file for crawl --- cmd/crawl.go | 1 + cmd/secrets.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/crawl.go b/cmd/crawl.go index 3a3c4e5..12203eb 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -68,6 +68,7 @@ func init() { CrawlCmd.Flags().StringP("username", "u", "", "Set the username for the BMC") CrawlCmd.Flags().StringP("password", "p", "", "Set the password for the BMC") CrawlCmd.Flags().BoolP("insecure", "i", false, "Ignore SSL errors") + CrawlCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "set the secrets file with BMC credentials") checkBindFlagError(viper.BindPFlag("crawl.username", CrawlCmd.Flags().Lookup("username"))) checkBindFlagError(viper.BindPFlag("crawl.password", CrawlCmd.Flags().Lookup("password"))) diff --git a/cmd/secrets.go b/cmd/secrets.go index b75e4e7..b4aaa43 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -215,7 +215,7 @@ var secretsRemoveCmd = &cobra.Command{ } func init() { - secretsCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "") + secretsCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "set the secrets file with BMC credentials") secretsStoreCmd.Flags().StringVar(&secretsStoreFormat, "format", "json", "set the input format for the secrets file (json|base64)") secretsStoreCmd.Flags().StringVarP(&secretsStoreInputFile, "input-file", "i", "", "set the file to read as input") From 17350ab99b9ea96f08b1464023378bfcfe52e82d Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 20 Mar 2025 09:27:39 -0600 Subject: [PATCH 16/34] fix: changed number of minimum args for secrets list --- cmd/secrets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/secrets.go b/cmd/secrets.go index b4aaa43..3ab09c5 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -199,7 +199,7 @@ var secretsListCmd = &cobra.Command{ var secretsRemoveCmd = &cobra.Command{ Use: "remove secretIDs...", - Args: cobra.MinimumNArgs(2), + Args: cobra.MinimumNArgs(1), Short: "Remove secrets by IDs from secret store.", Run: func(cmd *cobra.Command, args []string) { for _, secretID := range args { From e38402edc3e0d1cb11b1e79b6501d8070b3e4670 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 20 Mar 2025 09:28:24 -0600 Subject: [PATCH 17/34] refactor: changed removing secret from store returns error --- pkg/secrets/localstore.go | 8 +++++++- pkg/secrets/main.go | 2 +- pkg/secrets/staticstore.go | 5 +++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/secrets/localstore.go b/pkg/secrets/localstore.go index 1ae9b91..08acd6c 100644 --- a/pkg/secrets/localstore.go +++ b/pkg/secrets/localstore.go @@ -102,10 +102,16 @@ func (l *LocalSecretStore) ListSecrets() (map[string]string, error) { } // RemoveSecretByID removes the specified secretID stored locally -func (l *LocalSecretStore) RemoveSecretByID(secretID string) { +func (l *LocalSecretStore) RemoveSecretByID(secretID string) error { l.mu.RLock() + // Let user know if there was nothing to delete + _, err := l.GetSecretByID(secretID) + if err != nil { + return err + } delete(l.Secrets, secretID) l.mu.RUnlock() + return nil } // openStore tries to create or open the LocalSecretStore based on the environment diff --git a/pkg/secrets/main.go b/pkg/secrets/main.go index 6d27976..5cb7f95 100644 --- a/pkg/secrets/main.go +++ b/pkg/secrets/main.go @@ -4,5 +4,5 @@ type SecretStore interface { GetSecretByID(secretID string) (string, error) StoreSecretByID(secretID, secret string) error ListSecrets() (map[string]string, error) - RemoveSecretByID(secretID string) + RemoveSecretByID(secretID string) error } diff --git a/pkg/secrets/staticstore.go b/pkg/secrets/staticstore.go index 405f58b..40d9049 100644 --- a/pkg/secrets/staticstore.go +++ b/pkg/secrets/staticstore.go @@ -29,6 +29,7 @@ func (s *StaticStore) ListSecrets() (map[string]string, error) { }, nil } -func (s *StaticStore) RemoveSecretByID(secretID string) { - // Nothing to do here, since nothing is being stored +func (s *StaticStore) RemoveSecretByID(secretID string) error { + // Nothing to do here, since nothing is being stored. With different implementations, we could return an error when no secret is found for a specific ID. + return nil } From 01a88beb97529aa16beabc53c7cb6de72940ee11 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 20 Mar 2025 10:17:33 -0600 Subject: [PATCH 18/34] fix: secrets remove not updating local store and return error when not found --- cmd/secrets.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/secrets.go b/cmd/secrets.go index 3ab09c5..65f03db 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -203,13 +203,22 @@ var secretsRemoveCmd = &cobra.Command{ Short: "Remove secrets by IDs from secret store.", Run: func(cmd *cobra.Command, args []string) { for _, secretID := range args { + // open secret store from file store, err := secrets.OpenStore(secretsFile) if err != nil { fmt.Println(err) os.Exit(1) } - store.RemoveSecretByID(secretID) + // remove secret from store by it's ID + err = store.RemoveSecretByID(secretID) + if err != nil { + fmt.Println("failed to remove secret: ", err) + os.Exit(1) + } + + // update store by saving to original file + secrets.SaveSecrets(secretsFile, store.(*secrets.LocalSecretStore).Secrets) } }, } From 6c5fc993b0c1feab45bcd10f6809bc011eb71758 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 20 Mar 2025 10:18:22 -0600 Subject: [PATCH 19/34] refactor: export function to save JSON secrets --- pkg/secrets/localstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/secrets/localstore.go b/pkg/secrets/localstore.go index 08acd6c..5013d52 100644 --- a/pkg/secrets/localstore.go +++ b/pkg/secrets/localstore.go @@ -84,7 +84,7 @@ func (l *LocalSecretStore) StoreSecretByID(secretID, secret string) error { l.mu.Lock() l.Secrets[secretID] = encryptedSecret - err = saveSecrets(l.filename, l.Secrets) + err = SaveSecrets(l.filename, l.Secrets) l.mu.Unlock() return err } @@ -134,7 +134,7 @@ func OpenStore(filename string) (SecretStore, error) { } // Saves secrets back to the JSON file -func saveSecrets(jsonFile string, store map[string]string) error { +func SaveSecrets(jsonFile string, store map[string]string) error { file, err := os.OpenFile(jsonFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err From 34af75c1d23be01d2d68047d69fb4e1ce97304df Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 24 Mar 2025 11:35:37 -0600 Subject: [PATCH 20/34] refactor: change error message to warning --- cmd/collect.go | 2 +- cmd/secrets.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/collect.go b/cmd/collect.go index f643636..d222fb5 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -80,7 +80,7 @@ var CollectCmd = &cobra.Command{ if err != nil { // Something went wrong with the store so try using // Create a StaticSecretStore to hold the username and password - fmt.Println(err) + log.Warn().Err(err).Msg("failed to open local store") store = secrets.NewStaticStore(username, password) } diff --git a/cmd/secrets.go b/cmd/secrets.go index 65f03db..7e86170 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -74,6 +74,7 @@ var secretsStoreCmd = &cobra.Command{ secretValue = args[1] } + // handle input file format switch secretsStoreFormat { case "base64": decoded, err := base64.StdEncoding.DecodeString(secretValue) From 7cd927d50327d372fab04f969c1f1410142a2f6c Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 24 Mar 2025 11:36:53 -0600 Subject: [PATCH 21/34] refactor: minor changes to error messages --- pkg/secrets/localstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/secrets/localstore.go b/pkg/secrets/localstore.go index 5013d52..f426320 100644 --- a/pkg/secrets/localstore.go +++ b/pkg/secrets/localstore.go @@ -118,7 +118,7 @@ func (l *LocalSecretStore) RemoveSecretByID(secretID string) error { // variable MASTER_KEY. If not found, it prints an error. func OpenStore(filename string) (SecretStore, error) { if filename == "" { - return nil, fmt.Errorf("no path to secret store provided") + return nil, fmt.Errorf("path to secret store required") } masterKey := os.Getenv("MASTER_KEY") @@ -128,7 +128,7 @@ func OpenStore(filename string) (SecretStore, error) { store, err := NewLocalSecretStore(masterKey, filename, true) if err != nil { - return nil, fmt.Errorf("cannot open secrets store: %v", err) + return nil, fmt.Errorf("failed to create new local secret store: %v", err) } return store, nil } From daa7a32b144dab78fd8ed0144824b2861e2b1508 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 24 Mar 2025 13:36:02 -0600 Subject: [PATCH 22/34] refactor: added basic input format and cleanup --- cmd/secrets.go | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/cmd/secrets.go b/cmd/secrets.go index 7e86170..c6e1972 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "strings" "github.com/OpenCHAMI/magellan/pkg/secrets" "github.com/rs/zerolog/log" @@ -54,7 +55,7 @@ var secretsGenerateKeyCmd = &cobra.Command{ } var secretsStoreCmd = &cobra.Command{ - Use: "store secretID ", + Use: "store secretID ", Args: cobra.MinimumNArgs(1), Short: "Stores the given string value under secretID.", Run: func(cmd *cobra.Command, args []string) { @@ -71,31 +72,59 @@ var secretsStoreCmd = &cobra.Command{ log.Error().Msg("no input data or file") os.Exit(1) } else if len(args) > 1 && secretsStoreInputFile == "" { + // use args[1] here because args[0] is the secretID secretValue = args[1] } // handle input file format switch secretsStoreFormat { - case "base64": + case "basic": // format: $username:$password + var ( + values []string + username string + password string + ) + // seperate username and password provided + values = strings.Split(secretValue, ":") + if len(values) != 2 { + log.Error().Msgf("expected 2 arguments in [username:password] format but got %d", len(values)) + os.Exit(1) + } + + // open secret store to save credentials + store, err = secrets.OpenStore(secretsFile) + if err != nil { + log.Error().Err(err).Msg("failed to open secrets store") + os.Exit(1) + } + + // extract username/password from input (for clarity) + username = values[0] + password = values[1] + + // create JSON formatted string from input + secretValue = fmt.Sprintf("{\"username\": \"%s\", \"password\": \"%s\"}", username, password) + + case "base64": // format: ($encoded_base64_string) decoded, err := base64.StdEncoding.DecodeString(secretValue) if err != nil { - fmt.Printf("Error decoding base64 data: %v\n", err) + log.Error().Err(err).Msg("error decoding base64 data") os.Exit(1) } // check the decoded string if it's a valid JSON and has creds if !isValidCredsJSON(string(decoded)) { - log.Error().Msg("value is not a valid JSON or is missing credentials") + log.Error().Err(err).Msg("value is not a valid JSON or is missing credentials") os.Exit(1) } store, err = secrets.OpenStore(secretsFile) if err != nil { - fmt.Println(err) + log.Error().Err(err).Msg("failed to open secrets store") os.Exit(1) } secretValue = string(decoded) - case "json": + case "json": // format: {"username": $username, "password": $password} // read input from file if set and override if secretsStoreInputFile != "" { if secretValue != "" { @@ -129,7 +158,6 @@ var secretsStoreCmd = &cobra.Command{ fmt.Printf("Error storing secret: %v\n", err) os.Exit(1) } - fmt.Println("Secret stored successfully.") }, } @@ -191,7 +219,6 @@ var secretsListCmd = &cobra.Command{ os.Exit(1) } - fmt.Println("Secrets:") for key, value := range secrets { fmt.Printf("%s: %s\n", key, value) } @@ -225,8 +252,8 @@ var secretsRemoveCmd = &cobra.Command{ } func init() { - secretsCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "set the secrets file with BMC credentials") - secretsStoreCmd.Flags().StringVar(&secretsStoreFormat, "format", "json", "set the input format for the secrets file (json|base64)") + secretsCmd.Flags().StringVarP(&secretsFile, "output-file", "o", "nodes.json", "set the secrets file with BMC credentials") + secretsStoreCmd.Flags().StringVarP(&secretsStoreFormat, "format", "f", "basic", "set the input format for the secrets file (basic|json|base64)") secretsStoreCmd.Flags().StringVarP(&secretsStoreInputFile, "input-file", "i", "", "set the file to read as input") secretsCmd.AddCommand(secretsGenerateKeyCmd) From 8f96a2f686bbeb9aaf848f94c973105c5af234cc Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 24 Mar 2025 14:29:16 -0600 Subject: [PATCH 23/34] refactor: changed short opts for secret store --- cmd/secrets.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/secrets.go b/cmd/secrets.go index c6e1972..fb61168 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -252,8 +252,8 @@ var secretsRemoveCmd = &cobra.Command{ } func init() { - secretsCmd.Flags().StringVarP(&secretsFile, "output-file", "o", "nodes.json", "set the secrets file with BMC credentials") - secretsStoreCmd.Flags().StringVarP(&secretsStoreFormat, "format", "f", "basic", "set the input format for the secrets file (basic|json|base64)") + secretsCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "set the secrets file with BMC credentials") + secretsStoreCmd.Flags().StringVarP(&secretsStoreFormat, "format", "F", "basic", "set the input format for the secrets file (basic|json|base64)") secretsStoreCmd.Flags().StringVarP(&secretsStoreInputFile, "input-file", "i", "", "set the file to read as input") secretsCmd.AddCommand(secretsGenerateKeyCmd) From 10b3f55b5354acafaa21bc1fdbb8b37c13c12df3 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 24 Mar 2025 14:29:47 -0600 Subject: [PATCH 24/34] refactor: use vars for cred flags --- cmd/crawl.go | 10 +++++----- cmd/root.go | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/crawl.go b/cmd/crawl.go index 12203eb..db9821c 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -40,8 +40,8 @@ var CrawlCmd = &cobra.Command{ if err != nil { fmt.Println(err) store = &secrets.StaticStore{ - Username: viper.GetString("crawl.username"), - Password: viper.GetString("crawl.password"), + Username: username, + Password: password, } } systems, err := crawler.CrawlBMCForSystems(crawler.CrawlerConfig{ @@ -65,9 +65,9 @@ var CrawlCmd = &cobra.Command{ } func init() { - CrawlCmd.Flags().StringP("username", "u", "", "Set the username for the BMC") - CrawlCmd.Flags().StringP("password", "p", "", "Set the password for the BMC") - CrawlCmd.Flags().BoolP("insecure", "i", false, "Ignore SSL errors") + CrawlCmd.Flags().StringVarP(&username, "username", "u", "", "Set the username for the BMC") + CrawlCmd.Flags().StringVarP(&password, "password", "p", "", "Set the password for the BMC") + CrawlCmd.Flags().BoolVarP(&insecure, "insecure", "i", false, "Ignore SSL errors") CrawlCmd.Flags().StringVarP(&secretsFile, "file", "f", "nodes.json", "set the secrets file with BMC credentials") checkBindFlagError(viper.BindPFlag("crawl.username", CrawlCmd.Flags().Lookup("username"))) diff --git a/cmd/root.go b/cmd/root.go index 3b0d4f0..ae43a53 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -44,6 +44,7 @@ var ( verbose bool debug bool forceUpdate bool + insecure bool ) // The `root` command doesn't do anything on it's own except display From 67e2d40606e30e7cb74eb865d290831b8a3df907 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 24 Mar 2025 14:43:34 -0600 Subject: [PATCH 25/34] refactor: changed logging to use consistent JSON format --- cmd/crawl.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/crawl.go b/cmd/crawl.go index db9821c..9192d59 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -3,7 +3,8 @@ package cmd import ( "encoding/json" "fmt" - "log" + + "github.com/rs/zerolog/log" urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/OpenCHAMI/magellan/pkg/crawler" @@ -36,9 +37,11 @@ var CrawlCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { + // try and load credentials from local store first store, err := secrets.OpenStore(secretsFile) if err != nil { - fmt.Println(err) + log.Error().Err(err).Msg("failed to open store") + // try and use the `username` and `password` arguments instead store = &secrets.StaticStore{ Username: username, Password: password, @@ -47,15 +50,15 @@ var CrawlCmd = &cobra.Command{ systems, err := crawler.CrawlBMCForSystems(crawler.CrawlerConfig{ URI: args[0], CredentialStore: store, - Insecure: cmd.Flag("insecure").Value.String() == "true", + Insecure: insecure, }) if err != nil { - log.Fatalf("Error crawling BMC: %v", err) + log.Error().Err(err).Msg("error crawling BMC") } // Marshal the inventory details to JSON jsonData, err := json.MarshalIndent(systems, "", " ") if err != nil { - fmt.Println("Error marshalling to JSON:", err) + log.Error().Err(err).Msg("error marshalling to JSON:") return } From c88a29be005173c22c76ec01fc34f51952ba009c Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 24 Mar 2025 15:32:47 -0600 Subject: [PATCH 26/34] refactor: added check for secretID in secrets store --- cmd/crawl.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/cmd/crawl.go b/cmd/crawl.go index 9192d59..0e2984d 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -37,28 +37,37 @@ var CrawlCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { + var ( + uri = args[0] + store secrets.SecretStore + err error + ) // try and load credentials from local store first - store, err := secrets.OpenStore(secretsFile) + store, err = secrets.OpenStore(secretsFile) if err != nil { - log.Error().Err(err).Msg("failed to open store") + log.Warn().Err(err).Msg("failed to open local store...falling back to default provided arguments") // try and use the `username` and `password` arguments instead - store = &secrets.StaticStore{ - Username: username, - Password: password, - } + store = secrets.NewStaticStore(username, password) } + + // found the store so try to load the creds + _, err = store.GetSecretByID(uri) + if err != nil { + store = secrets.NewStaticStore(username, password) + } + systems, err := crawler.CrawlBMCForSystems(crawler.CrawlerConfig{ - URI: args[0], + URI: uri, CredentialStore: store, Insecure: insecure, }) if err != nil { - log.Error().Err(err).Msg("error crawling BMC") + log.Error().Err(err).Msg("failed to crawl BMC") } // Marshal the inventory details to JSON jsonData, err := json.MarshalIndent(systems, "", " ") if err != nil { - log.Error().Err(err).Msg("error marshalling to JSON:") + log.Error().Err(err).Msg("failed to marshal JSON") return } From d4d0bc8a2c10f7e966273d7d4c62796a94929034 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 25 Mar 2025 09:53:58 -0600 Subject: [PATCH 27/34] fix: collect not falling back to CLI args correctly --- pkg/collect.go | 57 ++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/pkg/collect.go b/pkg/collect.go index a83c730..1a2552e 100644 --- a/pkg/collect.go +++ b/pkg/collect.go @@ -61,15 +61,14 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret // collect bmc information asynchronously var ( - offset = 0 - wg sync.WaitGroup - collection = make([]map[string]any, 0) - found = make([]string, 0, len(*assets)) - done = make(chan struct{}, params.Concurrency+1) - chanAssets = make(chan RemoteAsset, params.Concurrency+1) - outputPath = path.Clean(params.OutputPath) - smdClient = &client.SmdClient{Client: &http.Client{}} - initialStore secrets.SecretStore = store + offset = 0 + wg sync.WaitGroup + collection = make([]map[string]any, 0) + found = make([]string, 0, len(*assets)) + done = make(chan struct{}, params.Concurrency+1) + chanAssets = make(chan RemoteAsset, params.Concurrency+1) + outputPath = path.Clean(params.OutputPath) + smdClient = &client.SmdClient{Client: &http.Client{}} ) // set the client's params from CLI @@ -106,9 +105,6 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret return } - // use initial store to check for creds for specific node - store = initialStore - // generate custom xnames for bmcs // TODO: add xname customization via CLI var ( @@ -122,26 +118,12 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret ) offset += 1 - // determine if local store exists and has credentials for - // the provided secretID... - // if it does not, create a static store and use the username - // and password provided instead - if store != nil { - _, err := store.GetSecretByID(uri) - if err != nil { - log.Warn().Err(err).Msgf("could not retrieve secrets for %s...falling back to default provided credentials", uri) - store = secrets.NewStaticStore(params.Username, params.Password) - } - } else { - log.Warn().Msgf("invalid store...falling back to default provided credentials for %s", uri) - store = secrets.NewStaticStore(params.Username, params.Password) - } - // crawl BMC node to fetch inventory data via Redfish var ( - systems []crawler.InventoryDetail - managers []crawler.Manager - config = crawler.CrawlerConfig{ + fallbackStore = secrets.NewStaticStore(params.Username, params.Password) + systems []crawler.InventoryDetail + managers []crawler.Manager + config = crawler.CrawlerConfig{ URI: uri, CredentialStore: store, Insecure: true, @@ -149,6 +131,21 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret err error ) + // determine if local store exists and has credentials for + // the provided secretID... + // if it does not, use the fallback static store instead with + // the username and password provided as arguments + if store != nil { + _, err := store.GetSecretByID(uri) + if err != nil { + log.Warn().Err(err).Msgf("could not retrieve secrets for %s...falling back to default provided credentials", uri) + config.CredentialStore = fallbackStore + } + } else { + log.Warn().Msgf("invalid store...falling back to default provided credentials for %s", uri) + config.CredentialStore = fallbackStore + } + // crawl for node and BMC information systems, err = crawler.CrawlBMCForSystems(config) if err != nil { From 3a1fc3fe38772198ccaec4d58a00161325bf9e43 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 25 Mar 2025 14:15:39 -0600 Subject: [PATCH 28/34] fix: added username/password to collect params --- cmd/collect.go | 2 ++ cmd/secrets.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index d222fb5..2764b0d 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -68,6 +68,8 @@ var CollectCmd = &cobra.Command{ ForceUpdate: forceUpdate, AccessToken: accessToken, SecretsFile: secretsFile, + Username: username, + Password: password, } // show all of the 'collect' parameters being set from CLI if verbose diff --git a/cmd/secrets.go b/cmd/secrets.go index fb61168..3766de9 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -219,8 +219,8 @@ var secretsListCmd = &cobra.Command{ os.Exit(1) } - for key, value := range secrets { - fmt.Printf("%s: %s\n", key, value) + for key := range secrets { + fmt.Printf("%s\n", key) } }, } From 841a97dce423a6baefbfe7c147484e7ec201c66b Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 25 Mar 2025 14:16:19 -0600 Subject: [PATCH 29/34] refactor: changed var name for clarity and added logging details --- pkg/collect.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/collect.go b/pkg/collect.go index 1a2552e..31288fc 100644 --- a/pkg/collect.go +++ b/pkg/collect.go @@ -50,7 +50,7 @@ type CollectParams struct { // // Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency // property value between 1 and 10000. -func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secrets.SecretStore) ([]map[string]any, error) { +func CollectInventory(assets *[]RemoteAsset, params *CollectParams, localStore secrets.SecretStore) ([]map[string]any, error) { // check for available remote assets found from scan if assets == nil { return nil, fmt.Errorf("no assets found") @@ -125,7 +125,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret managers []crawler.Manager config = crawler.CrawlerConfig{ URI: uri, - CredentialStore: store, + CredentialStore: localStore, Insecure: true, } err error @@ -135,14 +135,14 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, store secret // the provided secretID... // if it does not, use the fallback static store instead with // the username and password provided as arguments - if store != nil { - _, err := store.GetSecretByID(uri) + if localStore != nil { + _, err := localStore.GetSecretByID(uri) if err != nil { - log.Warn().Err(err).Msgf("could not retrieve secrets for %s...falling back to default provided credentials", uri) + log.Warn().Err(err).Msgf("could not retrieve secrets for %s...falling back to default provided credentials for user '%s'", uri, params.Username) config.CredentialStore = fallbackStore } } else { - log.Warn().Msgf("invalid store...falling back to default provided credentials for %s", uri) + log.Warn().Msgf("invalid store for %s...falling back to default provided credentials for user '%s'", uri, params.Username) config.CredentialStore = fallbackStore } From 0c53fa7495efa62e39a9be2ee60147d4c6aa4c63 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 25 Mar 2025 14:52:58 -0600 Subject: [PATCH 30/34] makefile: updated golangci-lint version --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8b5f3cb..e2062ad 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ mod: ## go mod tidy inst: ## go install tools $(call print-target) go install github.com/client9/misspell/cmd/misspell@v0.3.4 - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v2.0.1 go install github.com/goreleaser/goreleaser/v2@v2.3.2 go install github.com/cpuguy83/go-md2man/v2@latest @@ -71,7 +71,7 @@ build: ## go build container: ## docker build container: $(call print-target) - docker build . --build-arg REGISTRY_HOST=${REGISTRY_HOST} --no-cache --pull --tag '${NAME}:${VERSION}' + docker build . --build-arg REGISTRY_HOST=${REGISTRY_HOST} --no-cache --pull --tag '${NAME}:${VERSION}' .PHONY: spell spell: ## misspell From 667fd39213cfc8b79785e2b0fe00605f0735ba68 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 25 Mar 2025 14:54:09 -0600 Subject: [PATCH 31/34] lint: apply changes from golint --- cmd/secrets.go | 4 ++-- pkg/collect.go | 2 +- tests/compatibility_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/secrets.go b/cmd/secrets.go index 3766de9..cb6a191 100644 --- a/cmd/secrets.go +++ b/cmd/secrets.go @@ -60,7 +60,7 @@ var secretsStoreCmd = &cobra.Command{ Short: "Stores the given string value under secretID.", Run: func(cmd *cobra.Command, args []string) { var ( - secretID string = args[0] + secretID = args[0] secretValue string store secrets.SecretStore inputFileBytes []byte @@ -163,7 +163,7 @@ var secretsStoreCmd = &cobra.Command{ func isValidCredsJSON(val string) bool { var ( - valid bool = !json.Valid([]byte(val)) + valid = !json.Valid([]byte(val)) creds map[string]string err error ) diff --git a/pkg/collect.go b/pkg/collect.go index 31288fc..40ec851 100644 --- a/pkg/collect.go +++ b/pkg/collect.go @@ -81,7 +81,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams, localStore s } certPool := x509.NewCertPool() certPool.AppendCertsFromPEM(cacert) - smdClient.Client.Transport = &http.Transport{ + smdClient.Transport = &http.Transport{ TLSClientConfig: &tls.Config{ RootCAs: certPool, InsecureSkipVerify: true, diff --git a/tests/compatibility_test.go b/tests/compatibility_test.go index dfcc5e5..f428f6a 100644 --- a/tests/compatibility_test.go +++ b/tests/compatibility_test.go @@ -78,7 +78,7 @@ func TestRedfishV1ServiceRootAvailability(t *testing.T) { // Simple test to ensure an expected Redfish version minimum requirement. func TestRedfishV1Version(t *testing.T) { var ( - url string = fmt.Sprintf("%s/redfish/v1/", *host) + url = fmt.Sprintf("%s/redfish/v1/", *host) body client.HTTPBody = []byte{} headers client.HTTPHeader = map[string]string{} testClient = &http.Client{ From 6d68bbd28fd32cd35bb8e2b9c8d608a7d008793b Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 25 Mar 2025 15:26:14 -0600 Subject: [PATCH 32/34] makefile: corrected golangci-lint install string --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e2062ad..80292d4 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ mod: ## go mod tidy inst: ## go install tools $(call print-target) go install github.com/client9/misspell/cmd/misspell@v0.3.4 - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v2.0.1 + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.0.1 go install github.com/goreleaser/goreleaser/v2@v2.3.2 go install github.com/cpuguy83/go-md2man/v2@latest From a47a71b03940a1aa0f4fa7b54846790254ecca6b Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 25 Mar 2025 16:34:04 -0600 Subject: [PATCH 33/34] cmd: allow short opts for username/password --- cmd/collect.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 2764b0d..7223a75 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -96,8 +96,8 @@ var CollectCmd = &cobra.Command{ func init() { currentUser, _ = user.Current() CollectCmd.PersistentFlags().StringVar(&host, "host", "", "Set the URI to the SMD root endpoint") - CollectCmd.PersistentFlags().StringVar(&username, "username", "", "Set the master BMC username") - CollectCmd.PersistentFlags().StringVar(&password, "password", "", "Set the master BMC password") + CollectCmd.PersistentFlags().StringVarP(&username, "username", "u", "", "Set the master BMC username") + CollectCmd.PersistentFlags().StringVarP(&password, "password", "p", "", "Set the master BMC password") CollectCmd.PersistentFlags().StringVar(&secretsFile, "secrets-file", "", "Set path to the node secrets file") CollectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the default scheme used to query when not included in URI") CollectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "Set the protocol used to query") From 414f8d34f271b1124b4217a6aa2c2589b5d4a9bc Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 25 Mar 2025 16:48:41 -0600 Subject: [PATCH 34/34] readme: update with secrets and emulator sections --- README.md | 144 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 133 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2135758..29df9b5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,33 @@ # OpenCHAMI Magellan -The `magellan` CLI tool is a Redfish-based, board management controller (BMC) discovery tool designed to scan networks and is written in Go. The tool collects information from BMC nodes using the provided Redfish RESTful API with [`gofish`](https://github.com/stmcginnis/gofish) and loads the queried data into an [SMD](https://github.com/OpenCHAMI/smd/) instance. The tool strives to be more flexible by implementing multiple methods of discovery to work for a wider range of systems (WIP) and is capable of using independently of other tools or services. +The `magellan` CLI tool is a Redfish-based, board management controller (BMC) discovery tool designed to scan networks and is written in Go. The tool collects information from BMC nodes using the provided Redfish RESTful API with [`gofish`](https://github.com/stmcginnis/gofish) and loads the queried data into an [SMD](https://github.com/OpenCHAMI/smd/) instance. The tool strives to be more flexible by implementing multiple methods of discovery to work for a wider range of systems (WIP) and is capable of being used independently of other tools or services. -**Note: `magellan` v0.1.0 is incompatible with SMD v2.15.3 and earlier.** +> [!NOTE] +> The v0.1.0 version of `magellan` is incompatible with `smd` v2.15.3 and earlier due to `smd` lacking the inventory parsing code used with `magellan`'s output.** + + + + * [Main Features](#main-features) + * [Getting Started](#getting-started) + * [Building the Executable](#building-the-executable) + + [Building on Debian 12 (Bookworm)](#building-on-debian-12-bookworm) + + [Docker](#docker) + + [Arch Linux (AUR)](#arch-linux-aur) + * [Usage](#usage) + + [Checking for Redfish](#checking-for-redfish) + + [Running the Tool](#running-the-tool) + + [Managing Secrets](#managing-secrets) + + [Starting the Emulator](#starting-the-emulator) + + [Updating Firmware](#updating-firmware) + + [Getting an Access Token (WIP)](#getting-an-access-token-wip) + + [Running with Docker](#running-with-docker) + * [How It Works](#how-it-works) + * [TODO](#todo) + * [Copyright](#copyright) + + + + ## Main Features @@ -13,6 +38,7 @@ The `magellan` tool comes packed with a handleful of features for doing discover - Redfish-based firmware updating - Integration with OpenCHAMI SMD - Write inventory data to JSON +- Store and manage BMC secrets See the [TODO](#todo) section for a list of soon-ish goals planned. @@ -25,7 +51,7 @@ See the [TODO](#todo) section for a list of soon-ish goals planned. The `magellan` tool can be built to run on bare metal. Install the required Go tools, clone the repo, and then build the binary in the root directory with the following: ```bash -git clone https://github.com/OpenCHAMI/magellan +git clone https://github.com/OpenCHAMI/magellan cd magellan go mod tidy && go build ``` @@ -40,7 +66,7 @@ Getting the `magellan` tool to work with Go 1.21 on Debian 12 may require instal apt install gcc golang-1.21/bookworm-backport ``` -The binary executable for the `golang-1.21` executable can then be found using `dpkg`. +The binary executable for the `golang-1.21` executable can then be found using `dpkg`.v2.0.1 ```bash dpkg -L golang-1.21-go @@ -49,7 +75,7 @@ dpkg -L golang-1.21-go Using the correct binary, set the `CGO_ENABLED` environment variable and build the executable with `cgo` enabled: ```bash -export GOBIN=/usr/bin/golang-1.21/bin/go +export GOBIN=/usr/bin/golang-1.21/bin/go go env -w CGO_ENABLED=1 go mod tidy && go build ``` @@ -66,6 +92,17 @@ docker pull ghcr.io/openchami/magellan:latest See the ["Running with Docker"](#running-with-docker) section below about running with the Docker container. + +### Arch Linux (AUR) + +The `magellan` tool is in the AUR as a binary package and can be installed via your favorite AUR helper. + +```bash +yay -S magellan-bin +``` +> [!NOTE] +> The AUR package may not always be in sync with the latest release. It is recommended to install `magellan` from source for the latest version. + ## Usage The sections below assume that the BMC nodes have an IP address available to query Redfish. Currently, `magellan` does not support discovery with MAC addresses although that may change in the future. @@ -173,14 +210,95 @@ This will initiate a crawler that will find as much inventory data as possible. Note: If the `cache` flag is not set, `magellan` will use `/tmp/$USER/magellan.db` by default. +### Managing Secrets + +When connecting to an array of BMC nodes, some nodes may have different secret credentials than the rest. These secrets can be stored and used automatically by `magellan` when performing a `collect` or a `crawl`. All secrets are encrypted and are only accessible using the same `MASTER_KEY` as when stored originally. + +To store secrets using `magellan`: + +1. Set the `MASTER_KEY` environment variable. This can be generated using `magellan secrets generatekey`. + +```bash +export MASTER_KEY=$(magellan secrets generatekey) +``` + +2. Store secret credentials for hosts shown by `magellan list`: + +```bash +export bmc_host=https://172.16.0.105:443 +magellan secrets store $bmc_host $bmc_username:$bmc_password +``` + +There should be no output unless an error occurred. + +3. Print the list of hosts to confirm secrets are stored. + +```bash +magellan secrets list +``` + +If you see your `bmc_host` listed in the output, that means that your secrets were stored successfully. + +Additionally, if you want to see the actually contents, make sure the `MASTER_KEY` environment variable is correctly set and do the following: + +```bash +magellan secrets retrieve $bmc_host +``` + +4. Run either a `crawl` or `collect` and `magellan` should be a do find the credentials for each host. + +```bash +magellan crawl -i $bmc_host +magellan collect \ + --username $default_bmc_username \ + --password $default_bmc_password +``` + +If you pass agruments with the `--username/--password` flags, they will be used as a fallback if no credentials are found in the store. However, the secret store credentials are always used first if they exists. + +> [!NOTE] +> Make sure that the `secretID` is EXACTLY as show with `magellan list`. Otherwise, `magellan` will not be able to do the lookup from the secret store correctly. + +### Starting the Emulator + +This repository includes a quick and dirty way to test `magellan` using a Redfish emulator with little to no effort to get running. + +1. Make sure you have `docker` with Docker compose and optionally `make`. + +2. Run the `emulator/setup.sh` script or alternatively `make emulator`. + +This will start a flask server that you can make requests to using `curl`. + +```bash +export emulator_host=https://172.21.0.2:5000 +export emulator_username=root # set in the `rf_emulator.yml` file +export emulator_password=root_password # set in the `rf_emulator.yml` file +curl -k $emulator_host/redfish/v1 -u $emulator_username:$emulator_password +``` + +...or with `magellan` using the secret store... + +```bash +magellan scan --subnet 172.21.0.0/24 +magellan secrets store \ + $emulator_host \ + $emulator_username:$emulator_password +magellan collect --host https://smd.openchami.cluster +``` + +This example should work just like running on real hardware. + +> [!NOTE] +> The emulator host may be different from the one in the README. Make sure to double-check the host! + ### Updating Firmware The `magellan` tool is capable of updating firmware with using the `update` subcommand via the Redfish API. This may sometimes necessary if some of the `collect` output is missing or is not including what is expected. The subcommand expects there to be a running HTTP/HTTPS server running that has an accessible URL path to the firmware download. Specify the URL with the `--firmware-path` flag and the firmware type with the `--component` flag (optional) with all the other usual arguments like in the example below: ```bash ./magellan update 172.16.0.108:443 \ - --username $USERNAME \ - --password $PASSWORD \ + --username $bmc_username \ + --password $bmc_password \ --firmware-path http://172.16.0.255:8005/firmware/bios/image.RBU \ --component BIOS ``` @@ -188,9 +306,12 @@ The `magellan` tool is capable of updating firmware with using the `update` subc Then, the update status can be viewed by including the `--status` flag along with the other usual arguments or with the `watch` command: ```bash -./magellan update 172.16.0.110 --status --username $USERNAME --pass $PASSWORD | jq '.' +./magellan update 172.16.0.110 \ + --status \ + --username $bmc_username \ + --password $bmc_password | jq '.' # ...or... -watch -n 1 "./magellan update 172.16.0.110 --status --username $USERNAME --password $PASSWORD | jq '.'" +watch -n 1 "./magellan update 172.16.0.110 --status --username $bmc_username --password $bmc_password | jq '.'" ``` ### Getting an Access Token (WIP) @@ -219,7 +340,6 @@ The `magellan` tool can be ran in a Docker container after pulling the latest im ```bash docker pull ghcr.io/openchami/magellan:latest - ``` Then, run either with the helper script found in `bin/magellan.sh` or the binary in the container: @@ -259,8 +379,10 @@ See the [issue list](https://github.com/OpenCHAMI/magellan/issues) for plans for * [ ] Separate `collect` subcommand with making request to endpoint * [X] Support logging in with `opaal` to get access token * [X] Support using CA certificates with HTTP requests to SMD -* [ ] Add tests for the regressions and compatibility +* [X] Add tests for the regressions and compatibility * [X] Clean up, remove unused, and tidy code (first round) +* [X] Add `secrets` command to manage secret credentials +* [ ] Add server component to make `magellan` a micro-service ## Copyright