From 4597f63d1264b8d336510d9e5dc6f8f0f2226fdc Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 14 Aug 2024 10:57:30 -0600 Subject: [PATCH] Fixed issue with host string and added internal url package --- cmd/collect.go | 8 +-- cmd/crawl.go | 14 ++--- cmd/scan.go | 6 +-- internal/cache/cache.go | 1 + internal/scan.go | 3 +- internal/url/url.go | 116 ++++++++++++++++++++++++++++++++++++++++ pkg/client/net.go | 98 --------------------------------- 7 files changed, 131 insertions(+), 115 deletions(-) create mode 100644 internal/url/url.go diff --git a/cmd/collect.go b/cmd/collect.go index f33a52c..1f3288a 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -3,10 +3,10 @@ package cmd import ( "fmt" "os/user" - "strings" magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/cache/sqlite" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/OpenCHAMI/magellan/pkg/auth" "github.com/cznic/mathutil" "github.com/rs/zerolog/log" @@ -33,8 +33,10 @@ var collectCmd = &cobra.Command{ } // URL sanitanization for host argument - host = strings.TrimSuffix(host, "/") - host = strings.ReplaceAll(host, "//", "/") + host, err = urlx.Sanitize(host) + if err != nil { + log.Error().Err(err).Msg("failed to sanitize host") + } // try to load access token either from env var, file, or config if var not set if accessToken == "" { diff --git a/cmd/crawl.go b/cmd/crawl.go index ed70e5b..8069c9b 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -4,9 +4,8 @@ import ( "encoding/json" "fmt" "log" - "net/url" - "strings" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/OpenCHAMI/magellan/pkg/crawler" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -25,19 +24,14 @@ var crawlCmd = &cobra.Command{ " magellan crawl https://bmc.example.com -i -u username -p password", Args: func(cmd *cobra.Command, args []string) error { // Validate that the only argument is a valid URI + var err error if err := cobra.ExactArgs(1)(cmd, args); err != nil { return err } - parsedURI, err := url.ParseRequestURI(args[0]) + args[0], err = urlx.Sanitize(args[0]) if err != nil { - return fmt.Errorf("invalid URI specified: %s", args[0]) + return fmt.Errorf("failed to sanitize URI: %w", err) } - // Remove any trailing slashes - parsedURI.Path = strings.TrimSuffix(parsedURI.Path, "/") - // Collapse any doubled slashes - parsedURI.Path = strings.ReplaceAll(parsedURI.Path, "//", "/") - // Update the URI in the args slice - args[0] = parsedURI.String() return nil }, Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/scan.go b/cmd/scan.go index b0ed529..fedc691 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -9,9 +9,9 @@ import ( magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/cache/sqlite" - "github.com/OpenCHAMI/magellan/pkg/client" "github.com/rs/zerolog/log" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/cznic/mathutil" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -72,8 +72,8 @@ var scanCmd = &cobra.Command{ } // format and combine flag and positional args - targetHosts = append(targetHosts, client.FormatHostUrls(args, ports, scheme, verbose)...) - targetHosts = append(targetHosts, client.FormatHostUrls(hosts, ports, scheme, verbose)...) + targetHosts = append(targetHosts, urlx.FormatHosts(args, ports, scheme, verbose)...) + targetHosts = append(targetHosts, urlx.FormatHosts(hosts, ports, scheme, verbose)...) // add more hosts specified with `--subnet` flag if debug { diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 96513de..7b4eea1 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -4,6 +4,7 @@ import ( "database/sql/driver" ) +// TODO: implement extendable storage drivers using cache interface (sqlite, duckdb, etc.) type Cache[T any] interface { CreateIfNotExists(path string) (driver.Connector, error) Insert(path string, data ...T) error diff --git a/internal/scan.go b/internal/scan.go index 455749d..a88116d 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -10,6 +10,7 @@ import ( "sync" "time" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/OpenCHAMI/magellan/pkg/client" "github.com/rs/zerolog/log" ) @@ -164,7 +165,7 @@ func GenerateHostsWithSubnet(subnet string, subnetMask *net.IPMask, additionalPo // generate new IPs from subnet and format to full URL subnetIps := generateIPsWithSubnet(&subnetIp, subnetMask) - return client.FormatIPUrls(subnetIps, additionalPorts, defaultScheme, false) + return urlx.FormatIPs(subnetIps, additionalPorts, defaultScheme, false) } // GetDefaultPorts() returns a list of default ports. The only reason to have diff --git a/internal/url/url.go b/internal/url/url.go new file mode 100644 index 0000000..ba8ad3e --- /dev/null +++ b/internal/url/url.go @@ -0,0 +1,116 @@ +package url + +import ( + "fmt" + "net/url" + "strings" + + "github.com/rs/zerolog/log" +) + +func Sanitize(uri string) (string, error) { + // URL sanitanization for host argument + parsedURI, err := url.ParseRequestURI(uri) + if err != nil { + return "", fmt.Errorf("failed to parse URI: %w", err) + } + // Remove any trailing slashes + parsedURI.Path = strings.TrimSuffix(uri, "/") + // Collapse any doubled slashes + parsedURI.Path = strings.ReplaceAll(uri, "//", "/") + return parsedURI.String(), nil +} + +// FormatHosts() takes a list of hosts and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatHosts(hosts []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, host := range hosts { + uri, err := url.ParseRequestURI(host) + if err != nil { + if verbose { + log.Warn().Msgf("invalid URI parsed: %s", host) + } + continue + } + + // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) + if uri.Scheme == "" { + if scheme != "" { + uri.Scheme = scheme + } else { + // hardcoded assumption + uri.Scheme = "https" + } + } + + // tidy up slashes and update arg with new value + uri.Path = strings.TrimSuffix(uri.Path, "/") + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + var tmp []string + for _, port := range ports { + uri.Host += fmt.Sprintf(":%d", port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} + +// FormatIPs() takes a list of IP addresses and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatIPs(ips []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, ip := range ips { + if scheme == "" { + scheme = "https" + } + // make an entirely new object since we're expecting just IPs + uri := &url.URL{ + Scheme: scheme, + Host: ip, + } + + // tidy up slashes and update arg with new value + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + uri.Path = strings.TrimSuffix(uri.Path, "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + if len(ports) == 0 { + ports = append(ports, 443) + } + var tmp []string + for _, port := range ports { + uri.Host += fmt.Sprintf(":%d", port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} diff --git a/pkg/client/net.go b/pkg/client/net.go index af69a53..9a41d77 100644 --- a/pkg/client/net.go +++ b/pkg/client/net.go @@ -7,10 +7,6 @@ import ( "io" "net" "net/http" - "net/url" - "strings" - - "github.com/rs/zerolog/log" ) // HTTP aliases for readibility @@ -82,97 +78,3 @@ func MakeRequest(client *http.Client, url string, httpMethod string, body HTTPBo } return res, b, err } - -// FormatHostUrls() takes a list of hosts and ports and builds full URLs in the -// form of scheme://host:port. If no scheme is provided, it will use "https" by -// default. -// -// Returns a 2D string slice where each slice contains URL host strings for each -// port. The intention is to have all of the URLs for a single host combined into -// a single slice to initiate one goroutine per host, but making request to multiple -// ports. -func FormatHostUrls(hosts []string, ports []int, scheme string, verbose bool) [][]string { - // format each positional arg as a complete URL - var formattedHosts [][]string - for _, host := range hosts { - uri, err := url.ParseRequestURI(host) - if err != nil { - if verbose { - log.Warn().Msgf("invalid URI parsed: %s", host) - } - continue - } - - // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) - if uri.Scheme == "" { - if scheme != "" { - uri.Scheme = scheme - } else { - // hardcoded assumption - uri.Scheme = "https" - } - } - - // tidy up slashes and update arg with new value - uri.Path = strings.TrimSuffix(uri.Path, "/") - uri.Path = strings.ReplaceAll(uri.Path, "//", "/") - - // for hosts with unspecified ports, add ports to scan from flag - if uri.Port() == "" { - var tmp []string - for _, port := range ports { - uri.Host += fmt.Sprintf(":%d", port) - tmp = append(tmp, uri.String()) - } - formattedHosts = append(formattedHosts, tmp) - } else { - formattedHosts = append(formattedHosts, []string{uri.String()}) - } - - } - return formattedHosts -} - -// FormatIPUrls() takes a list of IP addresses and ports and builds full URLs in the -// form of scheme://host:port. If no scheme is provided, it will use "https" by -// default. -// -// Returns a 2D string slice where each slice contains URL host strings for each -// port. The intention is to have all of the URLs for a single host combined into -// a single slice to initiate one goroutine per host, but making request to multiple -// ports. -func FormatIPUrls(ips []string, ports []int, scheme string, verbose bool) [][]string { - // format each positional arg as a complete URL - var formattedHosts [][]string - for _, ip := range ips { - if scheme == "" { - scheme = "https" - } - // make an entirely new object since we're expecting just IPs - uri := &url.URL{ - Scheme: scheme, - Host: ip, - } - - // tidy up slashes and update arg with new value - uri.Path = strings.ReplaceAll(uri.Path, "//", "/") - uri.Path = strings.TrimSuffix(uri.Path, "/") - - // for hosts with unspecified ports, add ports to scan from flag - if uri.Port() == "" { - if len(ports) == 0 { - ports = append(ports, 443) - } - var tmp []string - for _, port := range ports { - uri.Host += fmt.Sprintf(":%d", port) - tmp = append(tmp, uri.String()) - } - formattedHosts = append(formattedHosts, tmp) - } else { - formattedHosts = append(formattedHosts, []string{uri.String()}) - } - - } - return formattedHosts -}