From 2edb9fdbb0fc86d3e3432783d24b53ef6da0003f Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 5 Sep 2023 14:26:53 -0600 Subject: [PATCH] Refactored flags and added xnames to add endpoint to `hms-smd` --- README.md | 100 ++++++++++++++++---------- api/dora/dora.go | 2 +- api/smd/smd.go | 8 +-- api/util.go | 5 +- bin/magellan.sh | 6 +- cmd/collect.go | 101 +++++++++++++++++---------- cmd/list.go | 4 +- cmd/root.go | 16 ++--- cmd/scan.go | 5 +- go.mod | 6 +- go.sum | 12 +--- internal/db/postgresql/postgresql.go | 1 + internal/db/sqlite/sqlite.go | 60 ++++++++++++++++ internal/logger.go | 26 +++++++ internal/magellan.go | 17 +++-- internal/scan.go | 63 ++--------------- migrations/probe_states.down.sql | 3 + migrations/probe_states.up.sql | 10 +++ 18 files changed, 270 insertions(+), 175 deletions(-) create mode 100644 internal/db/postgresql/postgresql.go create mode 100644 internal/db/sqlite/sqlite.go create mode 100644 internal/logger.go create mode 100644 migrations/probe_states.down.sql create mode 100644 migrations/probe_states.up.sql diff --git a/README.md b/README.md index dec619c..48c028c 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,86 @@ # Magellan -Magellan is a small tool designed to collect BMC information and load the data -into `hms-smd`. It is able to probe hosts for specific open ports using the `dora` -API or it's own simplier, built-in scanner and query BMC information via `bmclib`. -Once the data is received, it is then stored into `hms-smd` using its API. +Magellan is a small tool designed to scan a network and collect BMC information +to load the data into an [`hms-smd`](https://github.com/alexlovelltroy/hms-smd/tree/master) instance. + +## How It Works + +Magellan is designed to do three things: + +1. Scan for BMC nodes in cluster available on a network +2. Query information about each BMC node +3. Store queried information into a database + +Magellan first tries to probe for specified hosts using the [`dora`](https://github.com/bmc-toolbox/dora) +API. If that fails, it then tries to use its own built-in, simpler scanner as a fallback. +Next, it tries to query information about the BMC node using `bmclib` functions, but requires +access to a redfish interface on the node to work. Once the BMC information is received, +it is then stored into `hms-smd` using its API. + +In summary, `magellan` needs at minimum the following configured to work on each node: + +1. Available redfish interface with its host and port +2. A running instance of `hms-smd` with its host and port +3. Additional dependencies for `bmclib` such as `ipmitool` ## Building -To build the project, run the following: +Install Go, clone the repo, and then run the following in the project root: ```bash +clone https://github.com/bikeshack/magellan +cd magellan go mod tidy && go build ``` +This should find and download all of the required dependencies. Although other +versions of Go may work, the project has only been tested with v1.20. + ## Usage -Magellan's main goal is to load inventory components into `hms-smd` using redfish -or IMPI interfaces. It can scan subnets or specific hosts to find interfaces to query and stores into a local database. +There are three main commands to use with the tool: `scan`, `list`, and `collect`. +To scan a network for BMC nodes, use the `scan` command. If not port is specified, +`magellan` will probe ports 623, 22, 442, and 5000 by default similar to `dora`: ```bash -./magellan --help -Usage of ./magellan: - --cert-pool string path to an file containing x509 CAs. An empty string uses the system CAs. Only takes effect when --secure-tls=true - --driver strings set the BMC driver to use (default [redfish]) - --host strings set additional hosts - --pass string set the BMC pass (default "root_password") - --port ints set the ports to scan - --secure-tls enable secure TLS - --subnet strings set additional subnets (default [127.0.0.0]) - --threads int set the number of threads (default -1) - --timeout int set the timeout (default 1) - --user string set the BMC user (default "root") - -# example usage -./magellan scan \ - --subnet 127.0.0.0 \ # add a subnet of hosts - --host 127.0.0.1 \ # add an additional host - --port 5000 \ # port to scan for - --timeout 10 \ # timeout for response - --threads 255 \ # number of simutaneoulsy jobs - --dbpath data/assets.db # path to store scan results - -./magellan collect \ - --host 127.0.0.1 \ # host of hms-smd API - --port 27777 \ # port of hms-smd API - --dbpath data/assets.db # path of stored scan results +./magellan scan --subnet 192.168.0.0 --db.path data/assets.db --port 623 ``` +This with scan the `192.168.0.0` network returning the found nodes for port 623 +and store the results in database with path `data/assets.db`. Additional flags can +be set such as `host` to add additional hosts to scan, `timeout` to set how long +to wait for a responds from the BMC node, or `threads` to set the number of requests +to make concurrently. Try using `./magellan help scan` for a complete set of options. + +To see the available BMC nodes found from the scan, use the `list` command. Make +sure to point to the same database used before: + +```bash +./magellan list --db.path data/assets.db +``` + +This will print a list of IP address and ports found and stored from the scan. +Finally, run the `collect` command to store BMC info into `hms-smd`: + +```bash +./magellan collect --db.path data/assets.db --driver ipmi --timeout 5 --user admin --pass password +``` + +This uses the info store in the database above to request information about each +BMC node if possible. It uses the driver specified by the `driver` flag which is +passed to and set in `bmclib`. Like with the scan, the time to wait for a response +can be set with the `timeout` flag as well. This command also requires the `user` +and `pass/password` flag to be set to use `ipmitool` (which must installed as well). +Additionally, it may be necessary to set the `host` and `port` flags for `magellan` +to find the `hms-smd` API. + ## TODO -List of things left to fix or do... +List of things left to fix, do, or ideas... * [ ] Switch to internal scanner if `dora` fails +* [ ] Set default port automatically depending on the driver used to scan * [X] Test using different `bmclib` supported drivers (mainly 'redfish') * [ ] Confirm loading different components into `hms-smd` -* [ ] Clean up and tidy code -* [ ] Add unit tests +* [ ] Add unit tests for `scan`, `list`, and `collect` commands +* [ ] Clean up, remove unused, and tidy code diff --git a/api/dora/dora.go b/api/dora/dora.go index d571c3b..eba8548 100644 --- a/api/dora/dora.go +++ b/api/dora/dora.go @@ -42,7 +42,7 @@ func ScanForAssets() error { func QueryScannedPorts() error { // Perform scan and collect from dora server url := makeEndpointUrl("/scanned_ports") - _, body, err := api.MakeRequest(url, "GET", nil) + _, body, err := api.MakeRequest(url, "GET", nil, nil) if err != nil { return fmt.Errorf("could not discover assets: %v", err) } diff --git a/api/smd/smd.go b/api/smd/smd.go index 96125d5..4d612af 100644 --- a/api/smd/smd.go +++ b/api/smd/smd.go @@ -22,7 +22,7 @@ func makeEndpointUrl(endpoint string) string { func GetRedfishEndpoints() error { url := makeEndpointUrl("/Inventory/RedfishEndpoints") - _, body, err := api.MakeRequest(url, "GET", nil) + _, body, err := api.MakeRequest(url, "GET", nil, nil) if err != nil { return fmt.Errorf("could not get endpoint: %v", err) } @@ -33,7 +33,7 @@ func GetRedfishEndpoints() error { func GetComponentEndpoint(xname string) error { url := makeEndpointUrl("/Inventory/ComponentsEndpoints/" + xname) - res, body, err := api.MakeRequest(url, "GET", nil) + res, body, err := api.MakeRequest(url, "GET", nil, nil) if err != nil { return fmt.Errorf("could not get endpoint: %v", err) } @@ -42,7 +42,7 @@ func GetComponentEndpoint(xname string) error { return nil } -func AddRedfishEndpoint(data []byte) error { +func AddRedfishEndpoint(data []byte, headers map[string]string) error { if data == nil { return fmt.Errorf("could not add redfish endpoint: no data found") } @@ -51,7 +51,7 @@ func AddRedfishEndpoint(data []byte) error { // _ = ep // Add redfish endpoint via POST `/hsm/v2/Inventory/RedfishEndpoints` endpoint url := makeEndpointUrl("/Inventory/RedfishEndpoints") - res, body, _ := api.MakeRequest(url, "POST", data) + res, body, _ := api.MakeRequest(url, "POST", data, headers) fmt.Println("smd url: ", url) fmt.Println("res: ", res) fmt.Println("body: ", string(body)) diff --git a/api/util.go b/api/util.go index 730d01b..74d5693 100644 --- a/api/util.go +++ b/api/util.go @@ -8,10 +8,13 @@ import ( ) -func MakeRequest(url string, httpMethod string, body []byte) (*http.Response, []byte, error) { +func MakeRequest(url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) { // url := getSmdEndpointUrl(endpoint) req, _ := http.NewRequest(httpMethod, url, bytes.NewBuffer(body)) req.Header.Add("User-Agent", "magellan") + for k, v := range headers { + req.Header.Add(k, v) + } res, err := http.DefaultClient.Do(req) if err != nil { return nil, nil, fmt.Errorf("could not make request: %v", err) diff --git a/bin/magellan.sh b/bin/magellan.sh index c6c10b4..a422071 100644 --- a/bin/magellan.sh +++ b/bin/magellan.sh @@ -4,14 +4,14 @@ function build(){ } function scan() { - ./magellan scan --subnet 172.16.0.0 --dbpath data/assets.db --driver ipmi --port 623 + ./magellan scan --subnet 172.16.0.0 --db.path data/assets.db --port 623 } function list(){ - ./magellan list --dbpath data/assets.db + ./magellan list --db.path data/assets.db } function collect() { - ./magellan collect --dbpath data/assets.db --driver ipmi --timeout 5 --user admin --pass password + ./magellan collect --db.path data/assets.db --driver ipmi --timeout 5 --user admin --pass password } diff --git a/cmd/collect.go b/cmd/collect.go index c76787f..58f5b99 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -1,9 +1,12 @@ package cmd import ( + "davidallendj/magellan/api/smd" magellan "davidallendj/magellan/internal" + "davidallendj/magellan/internal/db/sqlite" + "fmt" - "github.com/bombsimon/logrusr/v2" + "github.com/Cray-HPE/hms-xname/xnames" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -13,19 +16,26 @@ var collectCmd = &cobra.Command{ Use: "collect", Short: "Query information about BMC", Run: func(cmd *cobra.Command, args []string) { - // make application logger - l := logrus.New() - l.Level = logrus.DebugLevel - logger := logrusr.New(l) + + // make application logger + l := magellan.NewLogger(logrus.New(), logrus.DebugLevel) // get probe states stored in db from scan - probeStates, err := magellan.GetStates(dbpath) + probeStates, err := sqlite.GetProbeResults(dbpath) if err != nil { - l.Errorf("could not get states: %v", err) + l.Log.Errorf("could not get states: %v", err) + } + + // generate custom xnames for bmcs + node := xnames.Node{ + Cabinet: 1000, + Chassis: 1, + ComputeModule: 7, + NodeBMC: 1, + Node: 0, } // use the found results to query bmc information - inventories := [][]byte{} users := [][]byte{} for _, ps := range probeStates { if !ps.State { @@ -43,61 +53,80 @@ var collectCmd = &cobra.Command{ WithSecureTLS: withSecureTLS, } - client, err := magellan.NewClient(&logger, &q) + client, err := magellan.NewClient(l, &q) if err != nil { - l.Errorf("could not make client: %v", err) + l.Log.Errorf("could not make client: %v", err) return } // metadata - _, err = magellan.QueryMetadata(client, &logger, &q) + _, err = magellan.QueryMetadata(client, l, &q) if err != nil { - l.Errorf("could not query metadata: %v\n", err) + l.Log.Errorf("could not query metadata: %v\n", err) } // inventories - inventory, err := magellan.QueryInventory(client, &logger, &q) + inventory, err := magellan.QueryInventory(client, l, &q) // inventory, err := magellan.QueryInventoryV2(q.Host, q.Port, q.User, q.Pass) if err != nil { - l.Errorf("could not query inventory: %v\n", err) + l.Log.Errorf("could not query inventory: %v\n", err) + } + + node.NodeBMC += 1 + + data := make(map[string]any) + data["ID"] = fmt.Sprintf("%v", node) + data["FQDN"] = ps.Host + data["RediscoverOnUpdate"] = false + + headers := make(map[string]string) + headers["Content-Type"] = "application/json" + + // add all endpoints to smd + err = smd.AddRedfishEndpoint(inventory, headers) + if err != nil { + logrus.Errorf("could not add redfish endpoint: %v", err) + } + + // confirm the inventories were added + err = smd.GetRedfishEndpoints() + if err != nil { + logrus.Errorf("could not get redfish endpoints: %v\n", err) } - inventories = append(inventories, inventory) // users - user, err := magellan.QueryUsers(client, &logger, &q) + user, err := magellan.QueryUsers(client, l, &q) if err != nil { - l.Errorf("could not query users: %v\n", err) + l.Log.Errorf("could not query users: %v\n", err) } users = append(users, user) - // // bios - _, err = magellan.QueryBios(client, &logger, &q) + // bios + _, err = magellan.QueryBios(client, l, &q) if err != nil { - l.Errorf("could not query bios: %v\n", err) + l.Log.Errorf("could not query bios: %v\n", err) } - _, err = magellan.QueryPowerState(client, &logger, &q) + _, err = magellan.QueryPowerState(client, l, &q) if err != nil { - l.Errorf("could not query power state: %v\n", err) + l.Log.Errorf("could not query power state: %v\n", err) } + } - - // add all endpoints to smd - // for _, inventory := range inventories { - // err := smd.AddRedfishEndpoint(inventory) - // if err != nil { - // logrus.Errorf("could not add redfish endpoint: %v", err) - // } - // } - - // confirm the inventories were added - // err = smd.GetRedfishEndpoints() - // if err != nil { - // logrus.Errorf("could not get redfish endpoints: %v\n", err) - // } + }, } func init(){ + collectCmd.PersistentFlags().StringSliceVar(&drivers, "driver", []string{"redfish"}, "set the driver(s) and fallback drivers to use") + collectCmd.PersistentFlags().StringVar(&smd.Host, "host", smd.Host, "set the host to the smd API") + collectCmd.PersistentFlags().IntVar(&smd.Port, "port", smd.Port, "set the port to the smd API") + collectCmd.PersistentFlags().StringVar(&user, "user", "", "set the BMC user") + collectCmd.PersistentFlags().StringVar(&pass, "pass", "", "set the BMC password") + collectCmd.PersistentFlags().StringVar(&pass, "password", "", "set the BMC password") + collectCmd.PersistentFlags().StringVar(&preferredDriver, "preferred-driver", "ipmi", "set the preferred driver to use") + collectCmd.PersistentFlags().StringVar(&ipmitoolPath, "ipmitool.path", "/usr/bin/ipmitool", "set the path for ipmitool") + collectCmd.PersistentFlags().BoolVar(&withSecureTLS, "secure-tls", false, "enable secure TLS") + collectCmd.PersistentFlags().StringVar(&certPoolFile, "cert-pool", "", "path to CA cert. (defaults to system CAs; used with --secure-tls=true)") rootCmd.AddCommand(collectCmd) } \ No newline at end of file diff --git a/cmd/list.go b/cmd/list.go index a539f43..fa69c41 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -1,7 +1,7 @@ package cmd import ( - magellan "davidallendj/magellan/internal" + "davidallendj/magellan/internal/db/sqlite" "fmt" "github.com/sirupsen/logrus" @@ -13,7 +13,7 @@ var listCmd = &cobra.Command{ Use: "list", Short: "List information from scan", Run: func(cmd *cobra.Command, args []string) { - probeResults, err := magellan.GetStates(dbpath) + probeResults, err := sqlite.GetProbeResults(dbpath) if err != nil { logrus.Errorf("could not get probe results: %v\n", err) } diff --git a/cmd/root.go b/cmd/root.go index 86ea350..b4d20e8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,7 +1,6 @@ package cmd import ( - "davidallendj/magellan/api/smd" "fmt" "os" @@ -26,6 +25,7 @@ var ( // TODO: discover bmc's on network (dora) // TODO: query bmc component information and store in db (?) // TODO: send bmc component information to smd +// TODO: set ports to scan automatically with set driver var rootCmd = &cobra.Command{ Use: "magellan", @@ -47,17 +47,9 @@ func Execute() { } func init() { - rootCmd.PersistentFlags().StringVar(&user, "user", "", "set the BMC user") - rootCmd.PersistentFlags().StringVar(&pass, "pass", "", "set the BMC pass") - rootCmd.PersistentFlags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts") - rootCmd.PersistentFlags().StringVar(&smd.Host, "smd-host", "localhost", "set the host to the hms-smd API") rootCmd.PersistentFlags().IntVar(&threads, "threads", -1, "set the number of threads") rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 10, "set the timeout") - rootCmd.PersistentFlags().IntSliceVar(&ports, "port", []int{}, "set the ports to scan") - rootCmd.PersistentFlags().StringSliceVar(&drivers, "driver", []string{"redfish"}, "set the driver(s) and fallback drivers to use") - rootCmd.PersistentFlags().StringVar(&preferredDriver, "preferred-driver", "ipmi", "set the preferred driver to use") - rootCmd.PersistentFlags().StringVar(&dbpath, "dbpath", ":memory:", "set the probe storage path") - rootCmd.PersistentFlags().StringVar(&ipmitoolPath, "ipmitool", "/usr/bin/ipmitool", "set the path for ipmitool") - rootCmd.PersistentFlags().BoolVar(&withSecureTLS, "secure-tls", false, "enable secure TLS") - rootCmd.PersistentFlags().StringVar(&certPoolFile, "cert-pool", "", "path to CA cert. (defaults to system CAs; used with --secure-tls=true)") + + rootCmd.PersistentFlags().StringVar(&dbpath, "db.path", "/tmp/magellan.db", "set the probe storage path") + } diff --git a/cmd/scan.go b/cmd/scan.go index 278b0e7..e86835d 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -2,6 +2,7 @@ package cmd import ( magellan "davidallendj/magellan/internal" + "davidallendj/magellan/internal/db/sqlite" "fmt" "github.com/cznic/mathutil" @@ -42,11 +43,13 @@ var scanCmd = &cobra.Command{ } probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, threads, timeout) fmt.Printf("probe states: %v\n", probeStates) - magellan.StoreStates(dbpath, &probeStates) + sqlite.InsertProbeResults(dbpath, &probeStates) }, } func init() { + scanCmd.PersistentFlags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan") + scanCmd.PersistentFlags().IntSliceVar(&ports, "port", []int{}, "set the ports to scan") scanCmd.Flags().Uint8Var(&begin, "begin", 0, "set the starting point for range of IP addresses") scanCmd.Flags().Uint8Var(&end, "end", 255, "set the ending point for range of IP addresses") scanCmd.Flags().StringSliceVar(&subnets, "subnet", []string{}, "set additional subnets") diff --git a/go.mod b/go.mod index 3efcdf6..482dcf6 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,7 @@ replace github.com/bmc-toolbox/dora => ../../dora require ( github.com/bmc-toolbox/bmclib/v2 v2.0.1-0.20230714152943-a1b87e2ff47f - github.com/bombsimon/logrusr/v2 v2.0.1 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 - github.com/go-logr/logr v1.2.4 github.com/jacobweinstock/registrar v0.4.7 github.com/jmoiron/sqlx v1.3.5 github.com/mattn/go-sqlite3 v1.14.6 @@ -18,13 +16,16 @@ require ( ) require ( + github.com/Cray-HPE/hms-xname v1.3.0 // indirect github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230 // indirect github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22 // indirect github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jacobweinstock/iamt v0.0.0-20230502042727-d7cdbe67d9ef // indirect + github.com/kr/text v0.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/satori/go.uuid v1.2.0 // indirect @@ -33,4 +34,5 @@ require ( golang.org/x/exp v0.0.0-20230127130021-4ca2cb1a16b7 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/sys v0.11.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 66a5905..8d5d7f4 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Cray-HPE/hms-xname v1.3.0 h1:DQmetMniubqcaL6Cxarz9+7KFfWGSEizIhfPHIgC3Gw= +github.com/Cray-HPE/hms-xname v1.3.0/go.mod h1:XKdjQSzoTps5KDOE8yWojBTAWASGaS6LfRrVDxwTQO8= github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230 h1:t95Grn2mOPfb3+kPDWsNnj4dlNcxnvuR72IjY8eYjfQ= github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230/go.mod h1:t2EzW1qybnPDQ3LR/GgeF0GOzHUXT5IVMLP2gkW1cmc= github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22 h1:a0MBqYm44o0NcthLKCljZHe1mxlN6oahCQHHThnSwB4= @@ -7,7 +9,6 @@ github.com/bmc-toolbox/bmclib/v2 v2.0.1-0.20230714152943-a1b87e2ff47f/go.mod h1: github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a h1:SjtoU9dE3bYfYnPXODCunMztjoDgnE3DVJCPLBqwz6Q= github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a/go.mod h1:SY//n1PJjZfbFbmAsB6GvEKbc7UXz3d30s3kWxfJQ/c= github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM= -github.com/bombsimon/logrusr/v2 v2.0.1/go.mod h1:ByVAX+vHdLGAfdroiMg6q0zgq2FODY2lc5YJvzmOJio= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= @@ -15,7 +16,6 @@ github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.0.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= @@ -34,7 +34,6 @@ github.com/jacobweinstock/registrar v0.4.7 h1:s4dOExccgD+Pc7rJC+f3Mc3D+NXHcXUaOi github.com/jacobweinstock/registrar v0.4.7/go.mod h1:PWmkdGFG5/ZdCqgMo7pvB3pXABOLHc5l8oQ0sgmBNDU= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -54,7 +53,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= @@ -64,8 +62,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stmcginnis/gofish v0.14.0 h1:geECNAiG33JDB2x2xDkerpOOuXFqxp5YP3EFE3vd5iM= github.com/stmcginnis/gofish v0.14.0/go.mod h1:BLDSFTp8pDlf/xDbLZa+F7f7eW0E/CHCboggsu8CznI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -74,8 +70,6 @@ golang.org/x/exp v0.0.0-20230127130021-4ca2cb1a16b7 h1:o7Ps2IYdzLRolS9/nadqeMSHp golang.org/x/exp v0.0.0-20230127130021-4ca2cb1a16b7/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210608053332-aa57babbf139/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -83,8 +77,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/db/postgresql/postgresql.go b/internal/db/postgresql/postgresql.go new file mode 100644 index 0000000..090c4e2 --- /dev/null +++ b/internal/db/postgresql/postgresql.go @@ -0,0 +1 @@ +package postgresql \ No newline at end of file diff --git a/internal/db/sqlite/sqlite.go b/internal/db/sqlite/sqlite.go new file mode 100644 index 0000000..a5ec3fb --- /dev/null +++ b/internal/db/sqlite/sqlite.go @@ -0,0 +1,60 @@ +package sqlite + +import ( + "fmt" + + magellan "davidallendj/magellan/internal" + + "github.com/jmoiron/sqlx" +) + +func InsertProbeResults(path string, states *[]magellan.BMCProbeResult) error { + if states == nil { + return fmt.Errorf("states == nil") + } + + // create database if it doesn't already exist + schema := ` + CREATE TABLE IF NOT EXISTS magellan_scanned_ports ( + host TEXT PRIMARY KEY NOT NULL, + port INTEGER, + protocol TEXT, + state INTEGER + ); + ` + db, err := sqlx.Open("sqlite3", path) + if err != nil { + return fmt.Errorf("could not open database: %v", err) + } + db.MustExec(schema) + + // insert all probe states into db + tx := db.MustBegin() + for _, state := range *states { + sql := `INSERT OR REPLACE INTO magellan_scanned_ports (host, port, protocol, state) + VALUES (:host, :port, :protocol, :state);` + _, err := tx.NamedExec(sql, &state) + if err != nil { + fmt.Printf("could not execute transaction: %v\n", err) + } + } + err = tx.Commit() + if err != nil { + return fmt.Errorf("could not commit transaction: %v", err) + } + return nil +} + +func GetProbeResults(path string) ([]magellan.BMCProbeResult, error) { + db, err := sqlx.Open("sqlite3", path) + if err != nil { + return nil, fmt.Errorf("could not open database: %v", err) + } + + results := []magellan.BMCProbeResult{} + err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC") + if err != nil { + return nil, fmt.Errorf("could not retrieve probes: %v", err) + } + return results, nil +} \ No newline at end of file diff --git a/internal/logger.go b/internal/logger.go new file mode 100644 index 0000000..d821d60 --- /dev/null +++ b/internal/logger.go @@ -0,0 +1,26 @@ +package magellan + +import ( + "github.com/sirupsen/logrus" +) + + +type Logger struct { + Log *logrus.Logger + Path string +} + + +func NewLogger(l *logrus.Logger, level logrus.Level) *Logger { + l.SetLevel(level) + return &Logger{ + Log: logrus.New(), + Path: "", + } +} + + +func (l *Logger)WriteFile(path string) { + +} + diff --git a/internal/magellan.go b/internal/magellan.go index 7271628..f19f691 100644 --- a/internal/magellan.go +++ b/internal/magellan.go @@ -9,7 +9,6 @@ import ( "time" bmclib "github.com/bmc-toolbox/bmclib/v2" - "github.com/go-logr/logr" "github.com/jacobweinstock/registrar" _ "github.com/mattn/go-sqlite3" _ "github.com/stmcginnis/gofish" @@ -22,7 +21,7 @@ const ( REDFISH_PORT = 5000 ) -type bmcProbeResult struct { +type BMCProbeResult struct { Host string `json:"host"` Port int `json:"port"` Protocol string `json:"protocol"` @@ -44,7 +43,7 @@ type QueryParams struct { IpmitoolPath string } -func NewClient(l *logr.Logger, q *QueryParams) (*bmclib.Client, error) { +func NewClient(l *Logger, q *QueryParams) (*bmclib.Client, error) { // NOTE: bmclib.NewClient(host, port, user, pass) // ...seems like the `port` params doesn't work like expected depending on interface @@ -59,7 +58,7 @@ func NewClient(l *logr.Logger, q *QueryParams) (*bmclib.Client, error) { clientOpts := []bmclib.Option{ // bmclib.WithSecureTLS(), // bmclib.WithHTTPClient(&httpClient), - bmclib.WithLogger(*l), + // bmclib.WithLogger(), // bmclib.WithRedfishHTTPClient(&httpClient), bmclib.WithRedfishPort(fmt.Sprint(q.Port)), bmclib.WithRedfishUseBasicAuth(true), @@ -105,7 +104,7 @@ func NewClient(l *logr.Logger, q *QueryParams) (*bmclib.Client, error) { return client, nil } -func QueryMetadata(client *bmclib.Client, l *logr.Logger, q *QueryParams) ([]byte, error) { +func QueryMetadata(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { // client, err := NewClient(l, q) // open BMC session and update driver registry @@ -139,7 +138,7 @@ func QueryMetadata(client *bmclib.Client, l *logr.Logger, q *QueryParams) ([]byt return []byte(b), nil } -func QueryInventory(client *bmclib.Client, l *logr.Logger, q *QueryParams) ([]byte, error) { +func QueryInventory(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { // discover.ScanAndConnect(url, user, pass, clientOpts) // open BMC session and update driver registry @@ -172,7 +171,7 @@ func QueryInventory(client *bmclib.Client, l *logr.Logger, q *QueryParams) ([]by return []byte(b), nil } -func QueryPowerState(client *bmclib.Client, l *logr.Logger, q *QueryParams) ([]byte, error) { +func QueryPowerState(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout)) client.Registry.FilterForCompatible(ctx) err := client.PreferProvider(q.Preferred).Open(ctx) @@ -203,7 +202,7 @@ func QueryPowerState(client *bmclib.Client, l *logr.Logger, q *QueryParams) ([]b } -func QueryUsers(client *bmclib.Client, l *logr.Logger, q *QueryParams) ([]byte, error) { +func QueryUsers(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { // discover.ScanAndConnect(url, user, pass, clientOpts) // client, err := NewClient(l, q) // if err != nil { @@ -242,7 +241,7 @@ func QueryUsers(client *bmclib.Client, l *logr.Logger, q *QueryParams) ([]byte, return []byte(b), nil } -func QueryBios(client *bmclib.Client, l *logr.Logger, q *QueryParams) ([]byte, error) { +func QueryBios(client *bmclib.Client, l *Logger, q *QueryParams) ([]byte, error) { // client, err := NewClient(l, q) // if err != nil { // return nil, fmt.Errorf("could not make query: %v", err) diff --git a/internal/scan.go b/internal/scan.go index c12bcae..d2e66ca 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -5,15 +5,13 @@ import ( "net" "sync" "time" - - "github.com/jmoiron/sqlx" ) -func rawConnect(host string, ports []int, timeout int, keepOpenOnly bool) []bmcProbeResult { - results := []bmcProbeResult{} +func rawConnect(host string, ports []int, timeout int, keepOpenOnly bool) []BMCProbeResult { + results := []BMCProbeResult{} for _, p := range ports { - result := bmcProbeResult{ + result := BMCProbeResult{ Host: host, Port: p, Protocol: "tcp", @@ -53,8 +51,8 @@ func GenerateHosts(subnet string, begin uint8, end uint8) []string { return hosts } -func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []bmcProbeResult { - states := make([]bmcProbeResult, 0, len(hosts)) +func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []BMCProbeResult { + states := make([]BMCProbeResult, 0, len(hosts)) done := make(chan struct{}, threads+1) chanHost := make(chan string, threads+1) // chanPort := make(chan int, threads+1) @@ -92,57 +90,6 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int) []bmcP return states } -func StoreStates(path string, states *[]bmcProbeResult) error { - if states == nil { - return fmt.Errorf("states == nil") - } - - // create database if it doesn't already exist - schema := ` - CREATE TABLE IF NOT EXISTS magellan_scanned_ports ( - host TEXT PRIMARY KEY NOT NULL, - port INTEGER, - protocol TEXT, - state INTEGER - ); - ` - db, err := sqlx.Open("sqlite3", path) - if err != nil { - return fmt.Errorf("could not open database: %v", err) - } - db.MustExec(schema) - - // insert all probe states into db - tx := db.MustBegin() - for _, state := range *states { - sql := `INSERT OR REPLACE INTO magellan_scanned_ports (host, port, protocol, state) - VALUES (:host, :port, :protocol, :state);` - _, err := tx.NamedExec(sql, &state) - if err != nil { - fmt.Printf("could not execute transaction: %v\n", err) - } - } - err = tx.Commit() - if err != nil { - return fmt.Errorf("could not commit transaction: %v", err) - } - return nil -} - -func GetStates(path string) ([]bmcProbeResult, error) { - db, err := sqlx.Open("sqlite3", path) - if err != nil { - return nil, fmt.Errorf("could not open database: %v", err) - } - - results := []bmcProbeResult{} - err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC") - if err != nil { - return nil, fmt.Errorf("could not retrieve probes: %v", err) - } - return results, nil -} - func GetDefaultPorts() []int { return []int{SSH_PORT, TLS_PORT, IPMI_PORT, REDFISH_PORT} } \ No newline at end of file diff --git a/migrations/probe_states.down.sql b/migrations/probe_states.down.sql new file mode 100644 index 0000000..84ef1e5 --- /dev/null +++ b/migrations/probe_states.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS magellan_probe_states; +DROP INDEX IF EXISTS magellan_probe_states_index_host; +DROP INDEX IF EXISTS magellan_probe_states_index_state; \ No newline at end of file diff --git a/migrations/probe_states.up.sql b/migrations/probe_states.up.sql new file mode 100644 index 0000000..703771e --- /dev/null +++ b/migrations/probe_states.up.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS magellan_probe_states ( + host TEXT PRIMARY KEY NOT NULL, + port INTEGER, + protocol TEXT, + state INTEGER, + updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS magellan_probe_states_index_host ON magellan_probe_states (host); +CREATE INDEX IF NOT EXISTS magellan_probe_states_index_state ON magellan_proble_states (state); \ No newline at end of file