diff --git a/cmd/crawl.go b/cmd/crawl.go index 87f093b..84fb24e 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "net/url" + "strings" "github.com/OpenCHAMI/magellan/pkg/crawler" "github.com/spf13/cobra" @@ -18,10 +19,16 @@ var crawlCmd = &cobra.Command{ if err := cobra.ExactArgs(1)(cmd, args); err != nil { return err } - _, err := url.ParseRequestURI(args[0]) + parsedURI, err := url.ParseRequestURI(args[0]) if err != nil { return fmt.Errorf("invalid URI specified: %s", args[0]) } + // Remove any trailing slashes + parsedURI.Path = strings.TrimSuffix(parsedURI.Path, "/") + // Collapse any doubled slashes + parsedURI.Path = strings.ReplaceAll(parsedURI.Path, "//", "/") + // Update the URI in the args slice + args[0] = parsedURI.String() return nil }, Run: func(cmd *cobra.Command, args []string) {