towk/magellan
1
0
Fork 0
mirror of https://github.com/davidallendj/magellan.git synced 2025-12-20 11:37:01 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: Clean up URI handling in crawl command

Browse source
This commit is contained in:
Alex Lovell-Troy 2024-07-02 16:37:39 -04:00
parent eccd9adb41
commit afe84f6129
No known key found for this signature in database
1 changed files with 8 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

9
cmd/crawl.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"log"
"net/url"
"strings"
"github.com/OpenCHAMI/magellan/pkg/crawler"
"github.com/spf13/cobra"
@ -18,10 +19,16 @@ var crawlCmd = &cobra.Command{
if err := cobra.ExactArgs(1)(cmd, args); err != nil {
return err
}
_, err := url.ParseRequestURI(args[0])
parsedURI, err := url.ParseRequestURI(args[0])
if err != nil {
return fmt.Errorf("invalid URI specified: %s", args[0])
}
// Remove any trailing slashes
parsedURI.Path = strings.TrimSuffix(parsedURI.Path, "/")
// Collapse any doubled slashes
parsedURI.Path = strings.ReplaceAll(parsedURI.Path, "//", "/")
// Update the URI in the args slice
args[0] = parsedURI.String()
return nil
},
Run: func(cmd *cobra.Command, args []string) {