Fixed issue with host string and added internal url package

cmd/collect.go

@@ -3,10 +3,10 @@ package cmd
 import (
 	"fmt"
 	"os/user"
-	"strings"
 
 	magellan "github.com/OpenCHAMI/magellan/internal"
 	"github.com/OpenCHAMI/magellan/internal/cache/sqlite"
+	urlx "github.com/OpenCHAMI/magellan/internal/url"
 	"github.com/OpenCHAMI/magellan/pkg/auth"
 	"github.com/cznic/mathutil"
 	"github.com/rs/zerolog/log"
@@ -33,8 +33,10 @@ var collectCmd = &cobra.Command{
 		}
 
 		// URL sanitanization for host argument
-		host = strings.TrimSuffix(host, "/")
-		host = strings.ReplaceAll(host, "//", "/")
+		host, err = urlx.Sanitize(host)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to sanitize host")
+		}
 
 		// try to load access token either from env var, file, or config if var not set
 		if accessToken == "" {

cmd/crawl.go

@@ -4,9 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
-	"net/url"
-	"strings"
 
+	urlx "github.com/OpenCHAMI/magellan/internal/url"
 	"github.com/OpenCHAMI/magellan/pkg/crawler"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -25,19 +24,14 @@ var crawlCmd = &cobra.Command{
 		"  magellan crawl https://bmc.example.com -i -u username -p password",
 	Args: func(cmd *cobra.Command, args []string) error {
 		// Validate that the only argument is a valid URI
+		var err error
 		if err := cobra.ExactArgs(1)(cmd, args); err != nil {
 			return err
 		}
-		parsedURI, err := url.ParseRequestURI(args[0])
+		args[0], err = urlx.Sanitize(args[0])
 		if err != nil {
-			return fmt.Errorf("invalid URI specified: %s", args[0])
+			return fmt.Errorf("failed to sanitize URI: %w", err)
 		}
-		// Remove any trailing slashes
-		parsedURI.Path = strings.TrimSuffix(parsedURI.Path, "/")
-		// Collapse any doubled slashes
-		parsedURI.Path = strings.ReplaceAll(parsedURI.Path, "//", "/")
-		// Update the URI in the args slice
-		args[0] = parsedURI.String()
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {

cmd/scan.go

@@ -9,9 +9,9 @@ import (
 
 	magellan "github.com/OpenCHAMI/magellan/internal"
 	"github.com/OpenCHAMI/magellan/internal/cache/sqlite"
-	"github.com/OpenCHAMI/magellan/pkg/client"
 	"github.com/rs/zerolog/log"
 
+	urlx "github.com/OpenCHAMI/magellan/internal/url"
 	"github.com/cznic/mathutil"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -72,8 +72,8 @@ var scanCmd = &cobra.Command{
 		}
 
 		// format and combine flag and positional args
-		targetHosts = append(targetHosts, client.FormatHostUrls(args, ports, scheme, verbose)...)
-		targetHosts = append(targetHosts, client.FormatHostUrls(hosts, ports, scheme, verbose)...)
+		targetHosts = append(targetHosts, urlx.FormatHosts(args, ports, scheme, verbose)...)
+		targetHosts = append(targetHosts, urlx.FormatHosts(hosts, ports, scheme, verbose)...)
 
 		// add more hosts specified with `--subnet` flag
 		if debug {

internal/cache/cache.go

@@ -4,6 +4,7 @@ import (
 	"database/sql/driver"
 )
 
+// TODO: implement extendable storage drivers using cache interface (sqlite, duckdb, etc.)
 type Cache[T any] interface {
 	CreateIfNotExists(path string) (driver.Connector, error)
 	Insert(path string, data ...T) error

internal/scan.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	urlx "github.com/OpenCHAMI/magellan/internal/url"
 	"github.com/OpenCHAMI/magellan/pkg/client"
 	"github.com/rs/zerolog/log"
 )
@@ -164,7 +165,7 @@ func GenerateHostsWithSubnet(subnet string, subnetMask *net.IPMask, additionalPo
 
 	// generate new IPs from subnet and format to full URL
 	subnetIps := generateIPsWithSubnet(&subnetIp, subnetMask)
-	return client.FormatIPUrls(subnetIps, additionalPorts, defaultScheme, false)
+	return urlx.FormatIPs(subnetIps, additionalPorts, defaultScheme, false)
 }
 
 // GetDefaultPorts() returns a list of default ports. The only reason to have

internal/url/url.go

@@ -0,0 +1,116 @@
+package url
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+)
+
+func Sanitize(uri string) (string, error) {
+	// URL sanitanization for host argument
+	parsedURI, err := url.ParseRequestURI(uri)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse URI: %w", err)
+	}
+	// Remove any trailing slashes
+	parsedURI.Path = strings.TrimSuffix(uri, "/")
+	// Collapse any doubled slashes
+	parsedURI.Path = strings.ReplaceAll(uri, "//", "/")
+	return parsedURI.String(), nil
+}
+
+// FormatHosts() takes a list of hosts and ports and builds full URLs in the
+// form of scheme://host:port. If no scheme is provided, it will use "https" by
+// default.
+//
+// Returns a 2D string slice where each slice contains URL host strings for each
+// port. The intention is to have all of the URLs for a single host combined into
+// a single slice to initiate one goroutine per host, but making request to multiple
+// ports.
+func FormatHosts(hosts []string, ports []int, scheme string, verbose bool) [][]string {
+	// format each positional arg as a complete URL
+	var formattedHosts [][]string
+	for _, host := range hosts {
+		uri, err := url.ParseRequestURI(host)
+		if err != nil {
+			if verbose {
+				log.Warn().Msgf("invalid URI parsed: %s", host)
+			}
+			continue
+		}
+
+		// check if scheme is set, if not set it with flag or default value ('https' if flag is not set)
+		if uri.Scheme == "" {
+			if scheme != "" {
+				uri.Scheme = scheme
+			} else {
+				// hardcoded assumption
+				uri.Scheme = "https"
+			}
+		}
+
+		// tidy up slashes and update arg with new value
+		uri.Path = strings.TrimSuffix(uri.Path, "/")
+		uri.Path = strings.ReplaceAll(uri.Path, "//", "/")
+
+		// for hosts with unspecified ports, add ports to scan from flag
+		if uri.Port() == "" {
+			var tmp []string
+			for _, port := range ports {
+				uri.Host += fmt.Sprintf(":%d", port)
+				tmp = append(tmp, uri.String())
+			}
+			formattedHosts = append(formattedHosts, tmp)
+		} else {
+			formattedHosts = append(formattedHosts, []string{uri.String()})
+		}
+
+	}
+	return formattedHosts
+}
+
+// FormatIPs() takes a list of IP addresses and ports and builds full URLs in the
+// form of scheme://host:port. If no scheme is provided, it will use "https" by
+// default.
+//
+// Returns a 2D string slice where each slice contains URL host strings for each
+// port. The intention is to have all of the URLs for a single host combined into
+// a single slice to initiate one goroutine per host, but making request to multiple
+// ports.
+func FormatIPs(ips []string, ports []int, scheme string, verbose bool) [][]string {
+	// format each positional arg as a complete URL
+	var formattedHosts [][]string
+	for _, ip := range ips {
+		if scheme == "" {
+			scheme = "https"
+		}
+		// make an entirely new object since we're expecting just IPs
+		uri := &url.URL{
+			Scheme: scheme,
+			Host:   ip,
+		}
+
+		// tidy up slashes and update arg with new value
+		uri.Path = strings.ReplaceAll(uri.Path, "//", "/")
+		uri.Path = strings.TrimSuffix(uri.Path, "/")
+
+		// for hosts with unspecified ports, add ports to scan from flag
+		if uri.Port() == "" {
+			if len(ports) == 0 {
+				ports = append(ports, 443)
+			}
+			var tmp []string
+			for _, port := range ports {
+				uri.Host += fmt.Sprintf(":%d", port)
+				tmp = append(tmp, uri.String())
+			}
+			formattedHosts = append(formattedHosts, tmp)
+		} else {
+			formattedHosts = append(formattedHosts, []string{uri.String()})
+		}
+
+	}
+	return formattedHosts
+}

pkg/client/net.go

@@ -7,10 +7,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"net/url"
-	"strings"
-
-	"github.com/rs/zerolog/log"
 )
 
 // HTTP aliases for readibility
@@ -82,97 +78,3 @@ func MakeRequest(client *http.Client, url string, httpMethod string, body HTTPBo
 	}
 	return res, b, err
 }
-
-// FormatHostUrls() takes a list of hosts and ports and builds full URLs in the
-// form of scheme://host:port. If no scheme is provided, it will use "https" by
-// default.
-//
-// Returns a 2D string slice where each slice contains URL host strings for each
-// port. The intention is to have all of the URLs for a single host combined into
-// a single slice to initiate one goroutine per host, but making request to multiple
-// ports.
-func FormatHostUrls(hosts []string, ports []int, scheme string, verbose bool) [][]string {
-	// format each positional arg as a complete URL
-	var formattedHosts [][]string
-	for _, host := range hosts {
-		uri, err := url.ParseRequestURI(host)
-		if err != nil {
-			if verbose {
-				log.Warn().Msgf("invalid URI parsed: %s", host)
-			}
-			continue
-		}
-
-		// check if scheme is set, if not set it with flag or default value ('https' if flag is not set)
-		if uri.Scheme == "" {
-			if scheme != "" {
-				uri.Scheme = scheme
-			} else {
-				// hardcoded assumption
-				uri.Scheme = "https"
-			}
-		}
-
-		// tidy up slashes and update arg with new value
-		uri.Path = strings.TrimSuffix(uri.Path, "/")
-		uri.Path = strings.ReplaceAll(uri.Path, "//", "/")
-
-		// for hosts with unspecified ports, add ports to scan from flag
-		if uri.Port() == "" {
-			var tmp []string
-			for _, port := range ports {
-				uri.Host += fmt.Sprintf(":%d", port)
-				tmp = append(tmp, uri.String())
-			}
-			formattedHosts = append(formattedHosts, tmp)
-		} else {
-			formattedHosts = append(formattedHosts, []string{uri.String()})
-		}
-
-	}
-	return formattedHosts
-}
-
-// FormatIPUrls() takes a list of IP addresses and ports and builds full URLs in the
-// form of scheme://host:port. If no scheme is provided, it will use "https" by
-// default.
-//
-// Returns a 2D string slice where each slice contains URL host strings for each
-// port. The intention is to have all of the URLs for a single host combined into
-// a single slice to initiate one goroutine per host, but making request to multiple
-// ports.
-func FormatIPUrls(ips []string, ports []int, scheme string, verbose bool) [][]string {
-	// format each positional arg as a complete URL
-	var formattedHosts [][]string
-	for _, ip := range ips {
-		if scheme == "" {
-			scheme = "https"
-		}
-		// make an entirely new object since we're expecting just IPs
-		uri := &url.URL{
-			Scheme: scheme,
-			Host:   ip,
-		}
-
-		// tidy up slashes and update arg with new value
-		uri.Path = strings.ReplaceAll(uri.Path, "//", "/")
-		uri.Path = strings.TrimSuffix(uri.Path, "/")
-
-		// for hosts with unspecified ports, add ports to scan from flag
-		if uri.Port() == "" {
-			if len(ports) == 0 {
-				ports = append(ports, 443)
-			}
-			var tmp []string
-			for _, port := range ports {
-				uri.Host += fmt.Sprintf(":%d", port)
-				tmp = append(tmp, uri.String())
-			}
-			formattedHosts = append(formattedHosts, tmp)
-		} else {
-			formattedHosts = append(formattedHosts, []string{uri.String()})
-		}
-
-	}
-	return formattedHosts
-}