Merge branch 'major-rewrite'

.gitignore

@@ -1,4 +1,5 @@
 **configurator
+**makeshift
 **.yaml
 **.yml
 **.so
@@ -6,3 +7,7 @@
 **.ignore
 **.tar.gz
 dist/
+tests/data
+tests/downloads
+tests/profiles
+tests/plugins

.goreleaser.yaml

@@ -5,7 +5,7 @@ before:
     - go mod download
     - make plugins
 builds:
-  - id: "configurator"
+  - id: "makeshift"
     goos:
         - linux
     goarch:
@@ -30,10 +30,10 @@ archives:
 dockers:
   - 
     image_templates:
-      - ghcr.io/openchami/{{.ProjectName}}:latest
+      - git.towk2.me/towk/{{.ProjectName}}:latest
-      - ghcr.io/openchami/{{.ProjectName}}:{{ .Tag }}
+      - git.towk2.me/towk/{{.ProjectName}}:{{ .Tag }}
-      - ghcr.io/openchami/{{.ProjectName}}:v{{ .Major }}
+      - git.towk2.me/towk/{{.ProjectName}}:v{{ .Major }}
-      - ghcr.io/openchami/{{.ProjectName}}:v{{ .Major }}.{{ .Minor }}
+      - git.towk2.me/towk/{{.ProjectName}}:v{{ .Major }}.{{ .Minor }}
     build_flag_templates:
       - "--pull"
       - "--label=org.opencontainers.image.created={{.Date}}"

CHANGELOG.md

@@ -5,10 +5,7 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-[0.2.0]
-
-- Major rewrite of entire code base
 
 [0.1.0]
 
-- Initial prerelease of configurator
+- Initial prerelease of makeshift

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright © 2024 Triad National Security, LLC.  This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration.
+Copyright © 2025 David J. Allen
 
 Permission is hereby granted, free of charge, to any person obtaining a
 copy of this software and associated documentation files (the "Software"),

Makefile

@@ -1,7 +1,7 @@
 # Unless set otherwise, the container runtime is Docker
 DOCKER ?= docker
 
-prog ?= configurator
+prog ?= makeshift
 git_tag := $(shell git describe --abbrev=0 --tags --always)
 sources := main.go $(wildcard cmd/*.go)
 plugin_source_prefix := pkg/generator/plugins
@@ -36,18 +36,18 @@ container-testing: binaries
 plugins: $(plugin_binaries)
 
 # how to make each plugin
-lib/%.so: pkg/generator/plugins/%/*.go
+lib/%.so: pkg/plugins/%/*.go
 	mkdir -p lib
 	go build -buildmode=plugin -o $@ $<
 
 docs:
-	go doc github.com/OpenCHAMI/cmd
+	go doc git.towk2.me/towk/makeshift/cmd
-	go doc github.com/OpenCHAMI/pkg/configurator
+	go doc git.towk2.me/towk/makeshift/pkg/${prog}
 
 # remove executable and all built plugins
 .PHONY: clean
 clean:
-	rm -f configurator
+	rm -f ${prog}
 	rm -f lib/*
 
 # run all of the unit tests

README.md

@@ -1,219 +1,241 @@
-# OpenCHAMI Configurator
+# << Makeshift >>
 
-The `configurator` is an extensible tool that is capable of dynamically generating files on the fly. The tool includes a built-in generator that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate files based on Jinja 2 template files. The tool and generator plugins are written in Go and plugins can be written by following the ["Creating Generator Plugins"](#creating-generator-plugins) section of this README.
+The `makeshift` tool is a service that serves files and CLI that downloads them with a couple of handy features baked-in. Although the CLI and server component function more like a glorified FTP, the power of this tool comes from the plugin system. For example, the file cobbler is built to run external plugins for more advanced processing files before serving them (e.g. fetching from a data source, rendering Jinja 2 templates, etc.).
 
-## Building and Usage
 
-The `configurator` is built using standard `go` build tools. The project separates the client, server, and generator components using build tags. To get started, clone the project, download the dependencies, and build the project:
+## Building and Go!
+
+
+The `makeshift` tool is built using standard `go` build tools. To get started, clone the project, download the dependencies, and build the project:
 
 ```bash
-git clone https://github.com/OpenCHAMI/configurator.git
+git clone https://git.towk2.me/towk/makeshift.git
 go mod tidy
-go build --tags all # equivalent to `go build --tags client,server``
+go build
 ```
 
 This will build the main driver program with the default generators that are found in the `pkg/generators` directory.
 
-> [!WARNING]
+> [!NOTE]
-> Not all of the plugins have completed generation implementations and are a WIP.
+> The project does not current separate the client, server, and plugin components using build tags, but will eventually. This will allow users to only compile and distribute specific parts of the tool with limited functionality.
 
-### Running Configurator with CLI
 
-After you build the program, run the following command to use the tool:
+## Basic Examples
+
+Here are some of the common commands you may want to try right off the bat (aside from `makeshift help` of course). The commands below that do not include the `--host`, `--path`, or `--root` flags are set using the environment variables.
 
 ```bash
-export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs...
+export MAKESHIFT_HOST=localhost
-./configurator generate --config config.yaml --target coredhcp -o coredhcp.conf --cacert ochami.pem
+export MAKESHIFT_PATH=/test
+export MAKESHIFT_ROOT=./test
+export MAKESHIFT_LOG_FILE=logs/makeshift.log
+export MAKESHIFT_LOG_LEVEL=debug
 ```
 
-This will generate a new `coredhcp` config file based on the Jinja 2 template specified in the config file for "coredhcp". The files will be written to `coredhcp.conf` as specified with the `-o/--output` flag. The `--target` flag specifies the type of config file to generate by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details).
+Start the server. The `--init` flag with create the default files and directory to get started at the `--root` path.
 
-In other words, there should be an entry in the config file that looks like this:
+```bash
+makeshift serve --root $HOME/apps/makeshift/server --init
+```
 
-```yaml
+From here, you might want to see what files are available by default.
-...
-targets:
-  coredhcp:
-    plugin: "lib/coredhcp.so"  # optional, if we want to use an external plugin instead
-    templates:
-      - templates/coredhcp.j2
-...
 
+```bash
+# list the files in the root directory
+makeshift list
+
+# list files store in the template directory with a specified host
+makeshift list --host http://localhost:5050 --path templates
+
+# list all available plugins
+makeshift list plugins
+
+# list specific plugin (same as 'makeshift plugins info jinja2')
+makeshift list plugins jinja2
+
+# list all available profiles
+makeshift list profiles
+
+# list specific profile information
+makeshift list profiles default
+```
+
+Then, we can start downloading some files or directories (as archives).
+
+```bash
+# download all data (notice --host and --port are not set here)
+makeshift download
+
+# download the 'help.txt' file without processing (i.e. using plugins)
+makeshift download --host http://localhost:5050 --path help.txt
+
+# download files with rendering using Jinja 2 plugin and default profile 
+makeshift download -p help.txt --plugins jinja2 --profile default
+
+# download directory with rendering using plugins to fetch data and render
+# using a custom 'compute' profile
+makeshift download -p templates --plugins smd,jinja2 --profile compute
+
+# do everything in the above example but extract and remove archive
+makeshift download -p templates --plugins smd,jinja2 --profile compute -xr
+
+# download a raw plugin
+makeshift download plugin jinja2
+
+# download a profile
+makeshift download profile default
 ```
 
 > [!NOTE]
-> The `configurator` tool requires a valid access token when making requests to an instance of SMD that has protected routes.
+> Plugins are ran in order specified with the `--plugins` flag, which means if you're creating a plugin to write to a data store and then read in a subsequent plugin, the order specified with the CLI matters!
 
-### Running Configurator as a Service
+(WIP) Files, directories, profiles, and plugins will eventually be able to be uploaded to the server.
-
-The tool can also run as a service to generate files for clients:
 
 ```bash
-export CONFIGURATOR_JWKS_URL="http://my.openchami.cluster:8443/key"
+# upload a single file in root directory
-./configurator serve --config config.yaml
+  makeshift upload -d @compute-base.yaml
+  
+  # upload a directory (not working yet...)
+  makeshift upload -d @setup/
+
+  # upload an archive (extracted and saved on server - not working yet...)
+  makeshift upload -d @setup.tar.gz -t archive
+
+   # upload a new profile
+  makeshift upload profile -d @compute.json kubernetes.json
+
+  # upload a new profile with a specific path
+  makeshift upload profile -d @kubernetes.json
+  makeshift upload profile -d '{"id": "custom", "data": {}}' kubernetes.json
+
+  # upload a new plugin
+  makeshift upload plugin -d @slurm.so
+  makeshift upload plugin slurm.so
 ```
 
-Once the server is up and listening for HTTP requests, you can try making a request to it with `curl` or `configurator fetch`. Both commands below are essentially equivalent:
+> [!NOTE]
+> Although every command has a `curl` equivalent, it is better to use the CLI since it has other features such as extracting and remove archives after downloading and saving archives as files automatically.
+
+## Server Root Structure
+
+The `makeshift` server serves files at the specified `--root` path (also set with `MAKESHIFT_ROOT` environment variable). The directory structure looks like the following by default with initializing with `makeshift init $MAKESHIFT_ROOT`.
 
 ```bash
-export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs...
+├── data
-curl http://127.0.0.1:3334/generate?target=coredhcp -X GET -H "Authorization: Bearer $ACCESS_TOKEN" --cacert ochami.pem
+├── plugins
-# ...or...
+└── profiles
-./configurator fetch --target coredhcp --host http://127.0.0.1:3334 --cacert ochami.pem
 ```
 
-This will do the same thing as the `generate` subcommand, but through a GET request where the file contents is returned in the response. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set when starting the server with `serve`. The `ACCESS_TOKEN` environment variable is passed to `curl` using the `Authorization` header and expects a token as a JWT.
+Each directory holds specific files for different purposes:
 
-### Docker
+- `data` - Stores any and all miscellaenous files and directories.
+- `plugins` - Stores plugins defined in the ["Creating Plugins"](#creating-plugins) section.
+- `profiles` - Stores profiles in JSON format as defined in the ["Creating Profiles"](#creating-profiles) section.
 
-New images can be built and tested using the `Dockerfile` provided in the project. However, the binary executable and the generator plugins must first be built before building the image since the Docker build copies the binary over. Therefore, build all of the binaries first by following the first section of ["Building and Usage"](#building-and-usage). Running `make docker` from the Makefile will automate this process. Otherwise, run the `docker build` command after building the executable and libraries.
 
-```bash
+## Creating Plugins
-docker build -t configurator:testing path/to/configurator/Dockerfile
-```
 
-If you want to easily include your own external generator plugins, you can build it and copy the `lib.so` file to `lib/`. Make sure that the `Generator` interface is implemented correctly as described in the ["Creating Generator Plugins"](#creating-generator-plugins) or the plugin will not load (you should get an error that specifically says this). Additionally, the name string returned from the `GetName()` method is used for looking up the plugin with the `--target` flag by the main driver program.
+The `makeshift` tool defines a plugin as an interface that can be implemented and compiled.
-
-Alternatively, pull the latest existing image/container from the GitHub container repository.
-
-```bash
-docker pull ghcr.io/openchami/configurator:latest
-```
-
-Then, run the Docker container similarly to running the binary.
-
-```bash
-export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs...
-docker run ghcr.io/openchami/configurator:latest configurator generate --config config.yaml --target coredhcp -o coredhcp.conf --cacert configurator.pem
-```
-
-### Creating Generator Plugins
-
-The `configurator` uses built-in and user-defined generators that implement the `Generator` interface to describe how config files should be generated. The interface is defined like so:
 
 ```go
-// maps the file path to its contents
+type Plugin interface {
-type FileMap = map[string][]byte
+	Name() string
+	Version() string
+	Description() string
+	Metadata() Metadata
 
-// interface for generator plugins
+	Init() error
-type Generator interface {
+	Run(data storage.KVStore, args []string) error
-  GetName() string
+	Cleanup() error
-  GetVersion() string
-  GetDescription() string
-  Generate(config *configurator.Config, opts ...util.Option) (FileMap, error)
 }
 ```
 
-A new plugin can be created by implementing the methods from interface and exporting a symbol with `Generator` as the name and the plugin struct as the type. The `GetName()` function returns the name that is used for looking up the corresponding target set in your config file. It can also be included in the templated files with the default plugins using the `{{ plugin_name }}` in your template. The `GetVersion()` and `GetDescription()` functions returns the version and description of the plugin which can be included in the templated files using `{{ plugin_version }}` and `{{ plugin_description }}` respectively with the default plugins. The `Generate` function is where the magic happens to build the config file from a template.
+Plugins can *literally* contain whatever you want and is written in Go. Here is a simple example implementation to demonstrate how that is done which we will save at `src/example.go`.
 
 ```go
-package main
+type Example struct{}
 
-type MyGenerator struct {
+func (p *Example) Name() string        { return "example" }
-  PluginInfo map[string]any
+func (p *Example) Version() string     { return "v0.0.1-alpha" }
-}
+func (p *Example) Description() string { return "An example plugin" }
-
+func (p *Example) Metadata() map[string]string {
-var pluginInfo map[string]any
+	return makeshift.Metadata{
-
+		"author": map[string]any{
-// this function is not a part of the `Generator` interface
+			"name":  "John Smith",
-func (g *MyGenerator) LoadFromFile() map[string]any{ /*...*/ }
+			"email": "john.smith@example",
-
+			"links": []string{
-func (g *MyGenerator) GetName() string {
+				"https://example.com",
-  // just an example...this can be done however you want
+			},
-  g.PluginInfo := LoadFromFile("path/to/plugin/info.json")
+		},
-  return g.PluginInfo["name"]
-}
-
-func (g *MyGenerator) GetVersion() string {
-  return g.PluginInfo["version"] // "v1.0.0"
-}
-
-func (g *MyGenerator) GetDescription() string {
-  return g.PluginInfo["description"] // "This is an example plugin."
-}
-
-func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) {
-  // do config generation stuff here...
-  var (
-    params = generator.GetParams(opts...)
-    client = generator.GetClient(params)
-    output = ""
-  )
-  if client {
-    eths, err := client.FetchEthernetInterfaces(opts...)
-    // ... blah, blah, blah, check error, format output, and so on...
-  
-
-    // apply the substitutions to Jinja template and return output as FileMap (i.e. path and it's contents)
-    return generator.ApplyTemplate(path, generator.Mappings{
-      "plugin_name":        g.GetName(),
-      "plugin_version":     g.GetVersion(),
-      "plugin_description": g.GetDescription(),
-      "dhcp_hosts": output,
-    })
 	}
 }
 
-// this MUST be named "Generator" for symbol lookup in main driver
+func (p *Example) Init() error {
-var Generator MyGenerator
+	// Initialize the plugin if necessary.
+	return nil
+}
+
+func (p *Example) Run(data storage.KVStore, args []string) error {
+  // Plugins can read and write to a data stores passed in.
+  // See the 'jinja2' plugin for reading and 'smd' plugin for writing.
+	return nil
+}
+
+func (p *Example) Clean() error {
+  // Clean up resources if necessary.
+	return nil
+}
+
+// This MUST be included to find the symbol in the main driver executable.
+var Makeshift Example
 ```
 
-> [!NOTE]
+Then, we can use the built-in `makeshift plugins compile` command to compile it.
-> The keys in `generator.ApplyTemplate` must not contain illegal characters such as a `-` or else the templates will not apply correctly.
-
-Finally, build the plugin and put it somewhere specified by `plugins` in your config. Make sure that the package is `main` before building.
 
 ```bash
-go build -buildmode=plugin -o lib/mygenerator.so path/to/mygenerator.go
+makeshift plugins compile src/example.go -o $MAKESHIFT_ROOT/plugins/example.so
 ```
 
-Now your plugin should be available to use with the `configurator` main driver program. If you get an error about not loading the correct symbol type, make sure that your generator function definitions match the `Generator` interface entirely and that you don't have a partially implemented interface.
-
 > [!TIP]
-> See the `examples/test.go` file for a plugin and template example.
+> Make sure you move all of your plugins to `$MAKESHIFT_ROOT/plugins` to use them and should have an `*.so` name for lookup. For example, to use a custom plugin with `makeshift download -p templates/hosts.j2 --plugins my-plugin`, there has to a plugin `$MAKESHIFT_ROOT/plugins/my-plugin.so`.
 
-## Configuration
+## Creating Profiles
 
-Here is an example config file to start using configurator:
+On the other hand, profiles are simply objects that contain data used to populate data stores. The `makeshift` tool does not currently use all fields of a profile which will likely be removed in the near future.
 
-```yaml
+```go
-server:         # Server-related parameters when using as service
+type Profile struct {
-  host: 127.0.0.1
+	ID          string         `json:"id"`                    // profile ID
-  port: 3334
+	Description string         `json:"description,omitempty"` // profile description
-  jwks:         # Set the JWKS uri for protected routes
+	Tags        []string       `json:"tags,omitempty"`        // tags used for filtering (not implemented yet)
-    uri: ""
+	Data        map[string]any `json:"data,omitempty"`        // include render data
-    retries: 5
+}
-smd:            # SMD-related parameters
-  host: http://127.0.0.1:27779
-plugins:        # path to plugin directories
-  - "lib/"
-targets:        # targets to call with --target flag
-  coredhcp:
-    templates:
-      - templates/coredhcp.j2
-    files:      # files to be copied without templating
-      - extra/nodes.conf
-    targets:    # additional targets to run (does not run recursively)
-      - dnsmasq
 ```
 
-The `server` section sets the properties for running the `configurator` tool as a service and is not required if you're only using the CLI. Also note that the `jwks.uri` parameter is only needed for protecting endpoints. If it is not set, then all API routes are entirely public. The `smd` section tells the `configurator` tool where to find the SMD service to pull state management data used internally by the client's generator. The `templates` section is where the paths are mapped to each generator by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `plugins` is a list of paths to search for and load external generator plugins.
+Profiles can be created using JSON and only require an `id` with optional `data`. See the example in `$MAKESHIFT_ROOT/profiles/default.json`.
 
-## Running the Tests
+```json
-
+{
-The `configurator` project includes a collection of tests focused on verifying plugin behavior and generating files. The tests do not include fetching information from any remote sources, can be ran with the following command:
+  "id": "default",
-
+  "description": "Makeshift default profile",
-```bash
+  "data": {
-go test ./tests/generate_test.go --tags=all
+        "host": "localhost",
+        "path": "/test",
+        "server_root": "./test"
+  }
+}
 ```
 
-## Known Issues
+> [!TIP]
+> Make sure that you store your custom profiles in `$MAKESHIFT_ROOT/profiles` and that you set the name you want to use for lookup with a `*.json` extension (e.g. `compute.json`).
 
-- Adds a new `OAuthClient` with every token request
+## TODO: Missing Features
-- Plugins are being loaded each time a file is generated
 
-## TODO
+There are some features still missing that will be added later.
 
-- Add group functionality to create by files by groups
+1. Running `makeshift` locally with profiles and plugins
-- Extend SMD client functionality (or make extensible?)
+2. Plugin to add user data for one-time use without creating a profile
-- Handle authentication with `OAuthClient`'s correctly
+3. Optionally build plugins directly into the main driver
+4. Protected routes that require authentication
+5. Configuration file for persistent runs
+6. `Dockerfile` and `docker-compose.yml` files to build containers

bin/compile-plugins.sh

@@ -0,0 +1,26 @@
+#!/usr/bin bash
+
+
+function compile_default_plugins() {
+    makeshift_exe=./makeshift
+    go_exe=go
+
+    # make sure go build tools are installed
+    if command -v $go_exe >/dev/null 2>&1; then
+        # make sure that MAKESHIFT_ROOT is set
+        if [[ ! -v MAKESHIFT_ROOT ]]; then
+            # Compile the default external plugins
+            go build
+            $makeshift_exe compile pkg/plugins/jinja2/jinja2.go -o $MAKESHIFT_ROOT/plugins/jinja2.go
+            $makeshift_exe compile pkg/plugins/smd/smd.go -o $MAKESHIFT_ROOT/plugins/smd.so
+            $makeshift_exe compile pkg/plugins/userdata/userdata.go -o $MAKESHIFT_ROOT/plugins/userdata.go
+        else 
+            echo "requires MAKESHIFT_ROOT to be set"
+        fi
+    else 
+        echo "Go build tools must be installed"
+    fi
+}
+
+
+compile_default_plugins

build/archlinux/PKGBUILD

@@ -1,15 +1,15 @@
-# Maintainer: David J. Allen <allend@lanl.gov>
+# Maintainer: David J. Allen <davidallendj@gmail.com>
-pkgname=configurator
+pkgname=makeshift
 pkgver=v0.1.0alpha
 pkgrel=1
-pkgdesc="An extensible tool to dynamically generate config files from SMD with Jinja 2 templating support."
+pkgdesc="Extensible file cobbler"
 arch=("x86_64")
-url="https://github.com/OpenCHAMI/configurator"
+url="https://git.towk2.me/towk/makeshift"
 license=('MIT')
-groups=("openchami")
+# groups=("towk")
-provides=('configurator')
+provides=('makeshift')
-conflicts=('configurator')
+conflicts=('makeshift')
-https://git.towk2.me/towk/configurator/releases/download/v0.1.0-alpha/configurator
+# https://git.towk2.me/towk/makeshift/releases/download/v0.1.0-alpha/makeshift
 source_x86_64=(
 	"${url}/releases/download/v0.1.0-alpha/${pkgname}.tar.gz"
 )
@@ -27,7 +27,7 @@ package() {
 	# install the binary to /usr/bin
 	mkdir -p "${pkgdir}/usr/bin"
 	mkdir -p "${pkgdir}/usr/lib/${pkgname}"
-	install -m755 configurator "${pkgdir}/usr/bin/configurator"
+	install -m755 makeshift "${pkgdir}/usr/bin/makeshift"
 
 	# install plugins to /usr/lib
 	install -m755 *.so "${pkgdir}/usr/lib/${pkgname}"

cmd/delete.go

@@ -0,0 +1,155 @@
+package cmd
+
+import (
+	"fmt"
+	"net/http"
+
+	"git.towk2.me/towk/makeshift/pkg/client"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var deleteCmd = &cobra.Command{
+	Use: "delete",
+	Example: `
+  # set up environment
+  export MAKESHIFT_HOST=http://localhost:5050
+  export MAKESHIFT_PATH=test
+
+  # delete a file or directory (cannot delete root)
+  makeshift delete -p help.txt 
+  makeshift delete --host http://localhost:5555 --path templates
+`,
+	Short: "Delete files and directories",
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		setenv(cmd, "host", "MAKESHIFT_HOST")
+		setenv(cmd, "path", "MAKESHIFT_PATH")
+		setenv(cmd, "cacert", "MAKESHIFT_CACERT")
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			paths, _      = cmd.Flags().GetStringSlice("path")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+
+			c     = client.New(host)
+			res   *http.Response
+			query string
+			err   error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("cacert", cacertPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		for _, path := range paths {
+			if path == "" {
+				log.Warn().Msg("skipping empty path")
+				continue
+			}
+
+			query = fmt.Sprintf("/delete/%s?", path)
+			res, _, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   query,
+				Method: http.MethodDelete,
+			})
+			handleResponseError(res, host, query, err)
+		}
+	},
+}
+
+var deleteProfilesCmd = &cobra.Command{
+	Use: "profiles",
+	Example: `
+  # delete profile(s) by its ID
+  makeshift delete profiles kubernetes slurm compute
+`,
+	Args:  cobra.MinimumNArgs(1),
+	Short: "Delete profile(s)",
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+
+			c     = client.New(host)
+			res   *http.Response
+			query string
+			err   error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("cacert", cacertPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+		for _, profileID := range args {
+			if profileID == "default" {
+				log.Warn().Msg("cannot delete the default profile")
+				continue
+			}
+
+			query = fmt.Sprintf("/profiles/%s", profileID)
+			res, _, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   query,
+				Method: http.MethodDelete,
+			})
+			handleResponseError(res, host, query, err)
+		}
+	},
+}
+
+var deletePluginsCmd = &cobra.Command{
+	Use: "plugins",
+	Example: `
+  # delete plugin(s) by name
+  makeshift delete plugins weather slurm user
+`,
+	Args:  cobra.MinimumNArgs(1),
+	Short: "Delete plugin(s)",
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+
+			c     = client.New(host)
+			res   *http.Response
+			query string
+			err   error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("cacert", cacertPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		for _, pluginName := range args {
+			query = fmt.Sprintf("/plugins/%s", pluginName)
+			res, _, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   query,
+				Method: http.MethodDelete,
+			})
+			handleResponseError(res, host, query, err)
+		}
+	},
+}
+
+func init() {
+	deleteCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift server host (can be set with MAKESHIFT_HOST)")
+	deleteCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load")
+	deleteCmd.Flags().StringSliceP("path", "p", []string{}, "Set the paths to delete files and directories")
+	deleteCmd.AddCommand(deleteProfilesCmd, deletePluginsCmd)
+
+	rootCmd.AddCommand(deleteCmd)
+}

cmd/download.go

@@ -1,7 +1,305 @@
 package cmd
 
-import "github.com/spf13/cobra"
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"git.towk2.me/towk/makeshift/internal/archive"
+	"git.towk2.me/towk/makeshift/pkg/client"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
 
 var downloadCmd = cobra.Command{
 	Use: "download",
+	Example: `
+  # set up environment
+  export MAKESHIFT_HOST=http://localhost:5050
+  export MAKESHIFT_PATH=test
+  
+  # download a file or directory (as archive)
+  makeshift download 
+  makeshift download --host http://localhost:5050.com --path test
+
+  # download a file or directory and run plugins with profile data
+  makeshift download --plugins smd,jinja2 --profile compute
+  curl $MAKESHIFT_HOST/download/test?plugins=smd,jinja2&profile=test
+
+  # download directory and extract it's contents automatically
+  # then, remove the downloaded archive
+  makeshift download -xr
+`,
+	Short: "Download and modify files with plugins",
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		setenv(cmd, "host", "MAKESHIFT_HOST")
+		setenv(cmd, "path", "MAKESHIFT_PATH")
+		setenv(cmd, "cacert", "MAKESHIFT_CACERT")
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _          = cmd.Flags().GetString("host")
+			path, _          = cmd.Flags().GetString("path")
+			outputPath, _    = cmd.Flags().GetString("output")
+			cacertPath, _    = cmd.Flags().GetString("cacert")
+			pluginNames, _   = cmd.Flags().GetStringSlice("plugins")
+			profileIDs, _    = cmd.Flags().GetStringSlice("profiles")
+			extract, _       = cmd.Flags().GetBool("extract")
+			removeArchive, _ = cmd.Flags().GetBool("remove-archive")
+
+			c     = client.New(host)
+			res   *http.Response
+			query string
+			body  []byte
+			err   error
+		)
+
+		query = fmt.Sprintf("/download/%s?", path)
+		if len(pluginNames) > 0 {
+			query += "plugins=" + url.QueryEscape(strings.Join(pluginNames, ","))
+		}
+		if len(profileIDs) > 0 {
+			query += "&profiles=" + url.QueryEscape(strings.Join(profileIDs, ","))
+		}
+
+		log.Debug().
+			Str("host", host).
+			Str("path", path).
+			Str("query", query).
+			Str("output", outputPath).
+			Strs("profiles", profileIDs).
+			Strs("plugins", pluginNames).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		res, body, err = c.MakeRequest(client.HTTPEnvelope{
+			Path:   query,
+			Method: http.MethodGet,
+		})
+		if err != nil {
+			log.Error().Err(err).
+				Str("host", host).
+				Str("path", path).
+				Str("output", outputPath).
+				Msg("failed to make request")
+			os.Exit(1)
+		}
+		if res.StatusCode != http.StatusOK {
+			log.Error().
+				Any("status", map[string]any{
+					"code":    res.StatusCode,
+					"message": res.Status,
+				}).
+				Str("host", host).
+				Str("path", path).
+				Str("output", outputPath).
+				Msg("response returned bad status")
+			os.Exit(1)
+		}
+
+		// determine if output path is an archive or file
+		switch res.Header.Get("FILETYPE") {
+		case "archive":
+			// write archive to disk with or without '-o' specified
+			if outputPath == "" {
+				outputPath = fmt.Sprintf("%s.tar.gz", path)
+				writeFiles(outputPath, body)
+				log.Debug().Str("path", outputPath).Msg("wrote archive to pre-determined path")
+			} else {
+				writeFiles(outputPath, body)
+				log.Debug().Str("path", outputPath).Msg("wrote archive to specified path")
+			}
+
+			// extract files if '-x' flag is passed
+			if extract {
+				var (
+					dir  = filepath.Dir(outputPath)
+					base = strings.TrimSuffix(filepath.Base(outputPath), ".tar.gz")
+				)
+				err = archive.Expand(outputPath, fmt.Sprintf("%s/%s", dir, base))
+				if err != nil {
+					log.Error().Err(err).
+						Str("path", outputPath).
+						Msg("failed to expand archive")
+					os.Exit(1)
+				}
+			}
+
+			// optionally, remove archive if '-r' flag is passed
+			// NOTE: this can only be used if `-x` flag is set
+			if removeArchive {
+				if !extract {
+					log.Warn().Msg("requires '-x/--extract' flag to be set to 'true'")
+				} else {
+					err = os.Remove(outputPath)
+					if err != nil {
+						log.Error().Err(err).
+							Str("path", outputPath).
+							Msg("failed to remove archive")
+					}
+				}
+			}
+		case "file":
+			// write to file if '-o' specified otherwise stdout
+			if outputPath != "" {
+				writeFiles(outputPath, body)
+				log.Debug().Str("path", outputPath).Msg("wrote file to specified path")
+			} else {
+				fmt.Println(string(body))
+			}
+		}
+	},
+}
+
+var downloadProfileCmd = &cobra.Command{
+	Use: "profile",
+	Example: `
+  // download a profile
+  makeshift download profile default
+`,
+	Args:  cobra.ExactArgs(1),
+	Short: "Download a profile",
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			outputPath, _ = cmd.Flags().GetString("output")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+
+			c     = client.New(host)
+			res   *http.Response
+			body  []byte
+			query string
+			err   error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("output", outputPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		for _, profileID := range args {
+			query = fmt.Sprintf("/profiles/%s", profileID)
+			res, body, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   query,
+				Method: http.MethodGet,
+			})
+			if err != nil {
+				log.Error().Err(err).
+					Str("host", host).
+					Msg("failed to make request")
+				os.Exit(1)
+			}
+			if res.StatusCode != http.StatusOK {
+				log.Error().
+					Any("status", map[string]any{
+						"code":    res.StatusCode,
+						"message": res.Status,
+					}).
+					Str("host", host).
+					Msg("response returned bad status")
+				os.Exit(1)
+			}
+			if outputPath != "" {
+				writeFiles(outputPath, body)
+			} else {
+				fmt.Println(string(body))
+			}
+		}
+	},
+}
+
+var downloadPluginCmd = &cobra.Command{
+	Use: "plugin",
+	Example: `
+  // download a plugin
+  makeshift download plugin smd jinja2
+`,
+	Args:  cobra.ExactArgs(1),
+	Short: "Download a raw plugin",
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			outputPath, _ = cmd.Flags().GetString("output")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+
+			c     = client.New(host)
+			res   *http.Response
+			query string
+			body  []byte
+			err   error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("output", outputPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		for _, pluginName := range args {
+			query = fmt.Sprintf("/plugins/%s/raw", pluginName)
+			res, body, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   query,
+				Method: http.MethodGet,
+			})
+			if err != nil {
+				log.Error().Err(err).
+					Str("host", host).
+					Str("query", query).
+					Msg("failed to make request")
+				os.Exit(1)
+			}
+			if res.StatusCode != http.StatusOK {
+				log.Error().
+					Any("status", map[string]any{
+						"code":    res.StatusCode,
+						"message": res.Status,
+					}).
+					Str("host", host).
+					Msg("response returned bad status")
+				os.Exit(1)
+			}
+			if outputPath != "" {
+				writeFiles(outputPath, body)
+			} else {
+				writeFiles(fmt.Sprintf("%s.so", pluginName), body)
+			}
+		}
+	},
+}
+
+func init() {
+	downloadCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)")
+	downloadCmd.PersistentFlags().StringP("output", "o", "", "Set the output path to write files")
+	downloadCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load")
+	downloadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)")
+	downloadCmd.Flags().StringSlice("profiles", []string{}, "Set the profile(s) to use to populate data store")
+	downloadCmd.Flags().StringSlice("plugins", []string{}, "Set the plugin(s) to run before downloading files")
+	downloadCmd.Flags().BoolP("extract", "x", false, "Set whether to extract archive locally after downloading")
+	downloadCmd.Flags().BoolP("remove-archive", "r", false, "Set whether to remove the archive after extracting (used with '--extract' flag)")
+
+	downloadCmd.AddCommand(downloadProfileCmd, downloadPluginCmd)
+
+	rootCmd.AddCommand(&downloadCmd)
+}
+
+// helper to write downloaded files
+func writeFiles(path string, body []byte) {
+	var err = os.WriteFile(path, body, 0o755)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to write file(s) from download")
+		os.Exit(1)
+	}
 }

cmd/init.go

@@ -0,0 +1,42 @@
+package cmd
+
+import (
+	"git.towk2.me/towk/makeshift/pkg/service"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var initCmd = &cobra.Command{
+	Use: "init",
+	Example: `
+  # create default files and directories at specified root path
+  # (must be set with positional argument)
+  makeshift init $MAKESHIFT_ROOT
+`,
+	Args:  cobra.ExactArgs(1),
+	Short: "Initialize directory with default files",
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			server *service.Service
+			err    error
+		)
+
+		// create the server root files and directories
+		server = service.New()
+		server.RootPath = args[0]
+		err = server.Init()
+		if err != nil {
+			log.Error().Err(err).
+				Str("root", server.RootPath).
+				Msg("failed to initialize server root")
+			return
+		}
+		log.Debug().
+			Str("root", server.RootPath).
+			Msg("initialize makeshift files at root path")
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(initCmd)
+}

cmd/list.go

@@ -0,0 +1,216 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+
+	makeshift "git.towk2.me/towk/makeshift/pkg"
+	"git.towk2.me/towk/makeshift/pkg/client"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var listCmd = &cobra.Command{
+	Use: "list",
+	Example: `
+  # list files in a remote data directory
+  configurator list --path test
+  configurator list --host http://localhost:5050 --path test
+
+  # list files using 'curl'
+  curl http://localhost:5050/list/test
+`,
+	Args:  cobra.NoArgs,
+	Short: "List all files in a remote data directory",
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		setenv(cmd, "host", "MAKESHIFT_HOST")
+		setenv(cmd, "path", "MAKESHIFT_PATH")
+		setenv(cmd, "cacert", "MAKESHIFT_CACERT")
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			path, _       = cmd.Flags().GetString("path")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+
+			c      = client.New(host)
+			body   []byte
+			output []string
+			err    error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("path", path).
+			Str("cacert", cacertPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		// make request to /list endpoint
+		_, body, err = c.MakeRequest(client.HTTPEnvelope{
+			Path:   fmt.Sprintf("/list/%s", path),
+			Method: http.MethodGet,
+		})
+		if err != nil {
+			log.Error().Err(err).
+				Str("host", host).
+				Str("path", path).
+				Msg("failed to make request")
+			os.Exit(1)
+		}
+
+		err = json.Unmarshal(body, &output)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to unmarshal response body")
+			os.Exit(1)
+		}
+
+		// show the list of files and directories
+		log.Info().Strs("output", output).Send()
+	},
+}
+
+var listPluginsCmd = &cobra.Command{
+	Use: "plugins",
+	Example: `
+  # show all plugins
+  makeshift list plugins
+
+  # show details for specific plugins
+  makeshift list plugins smd jinja2
+`,
+	Short: "Show plugins information",
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+
+			c       = client.New(host)
+			res     *http.Response
+			query   string
+			plugins []string
+			body    []byte
+			err     error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("cacert", cacertPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		if len(args) == 0 {
+			// make request to /list endpoint
+			res, body, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   "/plugins",
+				Method: http.MethodGet,
+			})
+			handleResponseError(res, host, "/plugins", err)
+			err = json.Unmarshal(body, &plugins)
+			if err != nil {
+				log.Error().Err(err).
+					Msg("failed to unmarshal plugins")
+				return
+			}
+		} else {
+			for _, pluginName := range args {
+				// make request to /list endpoint
+				query = fmt.Sprintf("/plugins/%s/info", pluginName)
+				res, body, err = c.MakeRequest(client.HTTPEnvelope{
+					Path:   query,
+					Method: http.MethodGet,
+				})
+				handleResponseError(res, host, query, err)
+
+				plugins = append(plugins, string(body))
+			}
+		}
+		log.Info().Strs("plugins", plugins).Send()
+	},
+}
+
+var listProfilesCmd = &cobra.Command{
+	Use: "profiles",
+	Example: `
+  # list all profiles
+  makeshift list profiles
+
+  # live individual profiles
+  makeshift list profiles default custom
+`,
+	Short: "Show all available profiles",
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+
+			c        = client.New(host)
+			res      *http.Response
+			profiles []makeshift.Profile
+			body     []byte
+			query    string
+			err      error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("cacert", cacertPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		if len(args) == 0 {
+			// make request to /list endpoint
+			res, body, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   "/profiles",
+				Method: http.MethodGet,
+			})
+			handleResponseError(res, host, "/profiles", err)
+
+			err = json.Unmarshal(body, &profiles)
+			if err != nil {
+				log.Error().Err(err).
+					Msg("failed to unmarshal plugins")
+				return
+			}
+		} else {
+			for _, profileID := range args {
+				// make request to /list endpoint
+				query = fmt.Sprintf("/profiles/%s", profileID)
+				res, body, err = c.MakeRequest(client.HTTPEnvelope{
+					Path:   query,
+					Method: http.MethodGet,
+				})
+				handleResponseError(res, host, query, err)
+				var profile makeshift.Profile
+				err = json.Unmarshal(body, &profile)
+				if err != nil {
+					log.Error().Err(err).
+						Msg("failed to unmarshal plugin")
+					continue
+				}
+				profiles = append(profiles, profile)
+			}
+		}
+		log.Info().Any("plugins", profiles).Send()
+	},
+}
+
+func init() {
+	listCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the configurator remote host (can be set with MAKESHIFT_HOST)")
+	listCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load")
+	listCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)")
+
+	listCmd.AddCommand(listPluginsCmd, listProfilesCmd)
+	rootCmd.AddCommand(listCmd)
+}

cmd/plugins.go

@@ -0,0 +1,246 @@
+package cmd
+
+import (
+	"fmt"
+	"io/fs"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	makeshift "git.towk2.me/towk/makeshift/pkg"
+	"git.towk2.me/towk/makeshift/pkg/client"
+	"git.towk2.me/towk/makeshift/pkg/service"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var pluginsCmd = &cobra.Command{
+	Use:   "plugins",
+	Short: "Manage, inspect, and compile plugins (requires Go build tools)",
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		setenv(cmd, "host", "MAKESHIFT_HOST")
+	},
+}
+
+var pluginsCompileCmd = &cobra.Command{
+	Use: "compile",
+	Example: `
+  # compile plugin using Go build tools
+  go build -buildmode=plugin -o lib/myplugin.so src/plugins/myplugin.go
+
+  # try to compile all plugins in current directory
+  cd src/plugins
+  makeshift plugin compile
+
+  # try to compile all plugins in specified directory
+  makeshift plugin compile src/plugins
+  
+  # compile 'src/plugins/myplugin.go' and save to 'lib/myplugin.so'
+  makeshift plugin compile src/plugins/myplugin.go -o lib/myplugin.so
+`,
+	Args: cobra.MaximumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			outputPath, _ = cmd.Flags().GetString("output")
+			output        []byte
+			fileInfo      os.FileInfo
+			err           error
+		)
+
+		// make the directory
+		err = os.MkdirAll(filepath.Dir(outputPath), 0o777)
+		if err != nil {
+			log.Fatal().Err(err).Msg("failed to make output directory")
+		}
+
+		// one arg passed, so determine if it is file or directory
+		if len(args) > 0 {
+			if fileInfo, err = os.Stat(args[0]); err == nil {
+				if fileInfo.IsDir() {
+					err = compilePluginsDir(args[0], outputPath)
+					if err != nil {
+						log.Fatal().Err(err).
+							Bytes("output", output).
+							Msg("failed to compile plugin")
+					}
+				} else {
+					// not a directory so check if Go file so try and compile it
+					if filepath.Ext(args[0]) == ".go" {
+						output, err = compilePlugin(outputPath, args[0])
+						if err != nil {
+							log.Fatal().Err(err).
+								Bytes("output", output).
+								Msg("failed to compile plugin")
+						}
+					} else {
+						log.Fatal().Msg("argument is not a valid plugin (must be Go file)")
+					}
+				}
+			} else if err != nil {
+				log.Fatal().Err(err).Msgf("failed to stat provided plugin path")
+			}
+		} else {
+			// no args passed, so use current directory
+			err = compilePluginsDir(".", outputPath)
+			if err != nil {
+				log.Fatal().Err(err).
+					Bytes("output", output).
+					Msg("failed to compile plugin")
+			}
+		}
+	},
+}
+
+var pluginsInspectCmd = &cobra.Command{
+	Use:  "inspect",
+	Args: cobra.MinimumNArgs(1),
+	Example: `
+  # inspect a plugin and print its information
+  makeshift plugin inspect lib/jinja2.so
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		for _, path := range args {
+			var (
+				plugin makeshift.Plugin
+				err    error
+			)
+			plugin, err = service.LoadPluginFromFile(path)
+			if err != nil {
+				log.Error().Err(err).
+					Str("path", path).
+					Msg("failed to load plugin from file")
+				continue
+			}
+
+			log.Info().Any("plugin", map[string]any{
+				"name":        plugin.Name(),
+				"version":     plugin.Version(),
+				"description": plugin.Description(),
+				"metadata":    plugin.Metadata(),
+			}).Send()
+		}
+	},
+}
+
+var pluginsInfoCmd = &cobra.Command{
+	Use: "info",
+	Example: `
+  # show information of a remote plugin
+  makeshift plugins info jinja2 smd
+
+  # show information of a local plugin (same as 'makeshift inspect')
+  makeshift plugins info --local $MAKESHIFT_ROOT/plugins/jinja2.so
+`,
+	Short: "Show plugin information",
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			outputPath, _ = cmd.Flags().GetString("output")
+			local, _      = cmd.Flags().GetBool("local")
+
+			c     = client.New(host)
+			res   *http.Response
+			query string
+			body  []byte
+			err   error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("output", outputPath).
+			Send()
+
+		if local {
+			var (
+				plugins []map[string]any
+				plugin  makeshift.Plugin
+				err     error
+			)
+			for _, path := range args {
+				plugin, err = service.LoadPluginFromFile(path)
+				if err != nil {
+					log.Error().Err(err).
+						Str("path", path).
+						Msg("failed to load plugin from path")
+					continue
+				}
+				plugins = append(plugins, makeshift.PluginToMap(plugin))
+			}
+			log.Info().Any("plugins", plugins).Send()
+		} else {
+			for _, pluginName := range args {
+				query = fmt.Sprintf("/plugins/%s/info", pluginName)
+				res, body, err = c.MakeRequest(client.HTTPEnvelope{
+					Path:   query,
+					Method: http.MethodGet,
+				})
+				if err != nil {
+					log.Error().Err(err).
+						Str("host", host).
+						Str("query", query).
+						Msg("failed to make request")
+					os.Exit(1)
+				}
+				if res.StatusCode != http.StatusOK {
+					log.Error().
+						Any("status", map[string]any{
+							"code":    res.StatusCode,
+							"message": res.Status,
+							"body":    string(body),
+						}).
+						Str("host", host).
+						Msg("response returned bad status")
+					os.Exit(1)
+				}
+				if outputPath != "" {
+					writeFiles(outputPath, body)
+				} else {
+					fmt.Println(string(body))
+				}
+			}
+		}
+	},
+}
+
+func init() {
+	pluginsCompileCmd.Flags().StringP("output", "o", "", "Set the path to save compiled plugin (matches source type, i.e. uses files or directory)")
+	pluginsInfoCmd.Flags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)")
+	pluginsInfoCmd.Flags().Bool("local", false, "Set whether to display information of a local plugin")
+
+	pluginsCmd.AddCommand(pluginsCompileCmd, pluginsInspectCmd, pluginsInfoCmd)
+	rootCmd.AddCommand(pluginsCmd)
+}
+
+func compilePlugin(outputPath string, srcPath string) ([]byte, error) {
+	var (
+		commandArgs string
+		command     *exec.Cmd
+	)
+	// execute command to build the plugin
+	commandArgs = fmt.Sprintf("go build -buildmode=plugin -o=%s %s", outputPath, srcPath)
+	command = exec.Command("bash", "-c", commandArgs)
+	return command.CombinedOutput()
+}
+
+func compilePluginsDir(dirpath string, outputPath string) error {
+	err := filepath.WalkDir(dirpath, func(path string, d fs.DirEntry, err error) error {
+		// not a directory and is Go file, so try and compile it
+		if !d.IsDir() && filepath.Ext(path) == ".go" {
+			var (
+				localOutputPath string = outputPath + "/" + path
+			)
+			output, err := compilePlugin(localOutputPath, path)
+			if err != nil {
+				log.Fatal().Err(err).
+					Bytes("output", output).
+					Str("path", localOutputPath).
+					Msg("failed to compile plugin")
+				return err
+			}
+		}
+		return nil
+	})
+	return err
+}

cmd/root.go

@@ -2,22 +2,58 @@ package cmd
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 
+	logger "git.towk2.me/towk/makeshift/pkg/log"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 )
 
-var rootCmd = cobra.Command{
+var (
-	Use: "configurator",
+	loglevel logger.LogLevel = logger.INFO
-	Run: func(cmd *cobra.Command, args []string) {
+)
 
+var rootCmd = cobra.Command{
+	Use:   "makeshift",
+	Short: "Extensible file cobbler",
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		var (
+			logFile string
+			err     error
+		)
+
+		// initialize the logger
+		logFile, _ = cmd.Flags().GetString("log-file")
+		err = logger.InitWithLogLevel(loglevel, logFile)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to initialize logger")
+			os.Exit(1)
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		// try and set flags using env vars
+		setenv(cmd, "log-file", "MAKESHIFT_LOG_FILE")
+		setenv(cmd, "log-level", "MAKESHIFT_LOG_LEVEL")
+		if len(args) == 0 {
+			err := cmd.Help()
+			if err != nil {
+				log.Error().Err(err).Msg("failed to print help")
+			}
+			os.Exit(0)
+		}
+	},
+	PostRun: func(cmd *cobra.Command, args []string) {
+		log.Debug().Msg("closing log file")
+		err := logger.LogFile.Close()
+		if err != nil {
+			log.Error().Err(err).Msg("failed to close log file")
+		}
 	},
 }
 
 func Execute() {
-	// run initialization code first
+	// run the main program
-	initEnv()
-
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)
@@ -25,13 +61,60 @@ func Execute() {
 }
 
 func init() {
+	cobra.OnInitialize(
+		initLogger,
+	)
 	// initialize the config a single time
+	rootCmd.PersistentFlags().VarP(&loglevel, "log-level", "l", "Set the log level output")
+	rootCmd.PersistentFlags().String("log-file", "", "Set the log file path (can be set with MAKESHIFT_LOG_FILE)")
 }
 
-func initConfigFromFile(path string) {
+func setenv(cmd *cobra.Command, varname string, envvar string) {
-
+	if cmd.Flags().Changed(varname) {
+		return
+	}
+	val := os.Getenv(envvar)
+	if val != "" {
+		cmd.Flags().Set(varname, val)
+	}
 }
 
-func initEnv() {
+func setenvp(cmd *cobra.Command, varname string, envvar string) {
-
+	if cmd.Flags().Changed(varname) {
+		return
+	}
+	val := os.Getenv(envvar)
+	if val != "" {
+		cmd.PersistentFlags().Set(varname, val)
+	}
+}
+
+func initLogger() {
+	// initialize the logger
+	logfile, _ := rootCmd.PersistentFlags().GetString("log-file")
+	err := logger.InitWithLogLevel(loglevel, logfile)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to initialize logger")
+		os.Exit(1)
+	}
+}
+
+func handleResponseError(res *http.Response, host, query string, err error) {
+	if err != nil {
+		log.Error().Err(err).
+			Str("host", host).
+			Str("query", query).
+			Msg("failed to make request")
+		os.Exit(1)
+	}
+	if res.StatusCode != http.StatusOK {
+		log.Error().
+			Any("status", map[string]any{
+				"code":    res.StatusCode,
+				"message": res.Status,
+			}).
+			Str("host", host).
+			Msg("response returned bad status")
+		os.Exit(1)
+	}
 }

cmd/run.go

@@ -0,0 +1,28 @@
+package cmd
+
+import "github.com/spf13/cobra"
+
+var runCmd = &cobra.Command{
+	Use: "run",
+	Example: `
+  NOTE: This command is not implemented yet!
+
+  # set up environment
+  export MAKESHIFT_HOST=http://localhost:5050
+  export MAKESHIFT_PATH=help.txt
+  export MAKESHIFT_ROOT=/opt/makeshift
+
+  # run locally similar to 'download'
+  makeshift run --plugins jinja2 --profiles default
+  makeshift run --root $HOME/apps/makeshift -p help.txt --plugins jinja2 --profiles default
+`,
+	Args:  cobra.NoArgs,
+	Short: "Run locally with plugins and profiles",
+	Run: func(cmd *cobra.Command, args []string) {
+
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(runCmd)
+}

cmd/serve.go

@@ -0,0 +1,101 @@
+package cmd
+
+import (
+	"net/url"
+	"time"
+
+	"git.towk2.me/towk/makeshift/pkg/service"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var serveCmd = &cobra.Command{
+	Use: "serve",
+	Example: `
+  # start the service in current directory
+  makeshift serve
+  
+  # start the service with root path and initialize
+  makeshift serve --root ./test --init -l debug
+`,
+	Args: cobra.NoArgs,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		setenv(cmd, "host", "MAKESHIFT_HOST")
+		setenv(cmd, "root", "MAKESHIFT_ROOT")
+		setenv(cmd, "timeout", "MAKESHIFT_TIMEOUT")
+		setenv(cmd, "cacert", "MAKESHIFT_CACERT")
+		setenv(cmd, "keyfile", "MAKESHIFT_KEYFILE")
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			rootPath, _   = cmd.Flags().GetString("root")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+			keyfile, _    = cmd.Flags().GetString("keyfile")
+			timeout, _    = cmd.Flags().GetInt("timeout")
+
+			parsed *url.URL
+			server *service.Service
+			err    error
+		)
+
+		// parse the host to remove scheme if needed
+		parsed, err = url.Parse(host)
+		if err != nil {
+			log.Warn().Err(err).
+				Str("host", host).
+				Msg("could not parse host")
+		}
+
+		// set the server values
+		server = service.New()
+		server.Addr = parsed.Host
+		server.RootPath = rootPath
+		server.CACertFile = cacertPath
+		server.CACertKeyfile = keyfile
+		server.Timeout = time.Duration(timeout) * time.Second
+
+		// show some debugging information
+		log.Debug().
+			Str("host", parsed.Host).
+			Any("paths", map[string]string{
+				"root":     rootPath,
+				"cacert":   cacertPath,
+				"keyfile":  keyfile,
+				"data":     server.PathForData(),
+				"profiles": server.PathForProfiles(),
+				"plugins":  server.PathForPlugins(),
+				"metadata": server.PathForMetadata(),
+			}).
+			Send()
+
+		// make the default directories and files if flag is passed
+		if cmd.Flags().Changed("init") {
+			err = server.Init()
+			if err != nil {
+				log.Error().Err(err).
+					Str("host", parsed.Host).
+					Str("root", rootPath).
+					Msg("failed to initialize server root")
+				return
+			}
+		}
+
+		// serve and log why the server closed
+		err = server.Serve()
+		log.Error().Err(err).Msg("server closed")
+	},
+}
+
+func init() {
+	serveCmd.Flags().Bool("init", false, "Initializes default files at specified with the '--root' flag")
+	serveCmd.Flags().String("host", "localhost:5050", "Set the configurator server host (can be set with MAKESHIFT_HOST)")
+	serveCmd.Flags().String("root", "./", "Set the root path to serve files (can be set with MAKESHIFT_ROOT)")
+	serveCmd.Flags().IntP("timeout", "t", 60, "Set the timeout in seconds for requests (can be set with MAKESHIFT_TIMEOUT)")
+	serveCmd.Flags().String("cacert", "", "Set the CA certificate path to load (can be set with MAKESHIFT_CACERT)")
+	serveCmd.Flags().String("keyfile", "", "Set the CA key file to use (can be set with MAKESHIFT_KEYFILE)")
+
+	serveCmd.MarkFlagsRequiredTogether("cacert", "keyfile")
+
+	rootCmd.AddCommand(serveCmd)
+}

cmd/upload.go

@@ -1,14 +1,462 @@
 package cmd
 
-import "github.com/spf13/cobra"
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"maps"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"git.towk2.me/towk/makeshift/internal/format"
+	makeshift "git.towk2.me/towk/makeshift/pkg"
+	"git.towk2.me/towk/makeshift/pkg/client"
+	"git.towk2.me/towk/makeshift/pkg/service"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var (
+	inputFormat format.DataFormat = format.JSON
+)
 
 var uploadCmd = &cobra.Command{
 	Use: "upload",
-	Run: func(cmd *cobra.Command, args []string) {
+	Example: `
+  # upload a single file in root directory
+  makeshift upload -d @compute-base.yaml
   
+  # upload a directory
+  makeshift upload -d @setup/
+
+  # upload an archive (extracted and saved on server)
+  makeshift upload -d @setup.tar.gz -t archive
+
+  # upload multiple files with a specific path (used to set remote location)
+  makeshift upload -d @kubernetes.json -p nodes/kubernetes.json
+  makeshift upload -d @slurm.json -d @compute.json -p nodes
+`,
+	Short: "Upload files and directories",
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		setenv(cmd, "host", "MAKESHIFT_HOST")
+		setenv(cmd, "path", "MAKESHIFT_PATH")
+		setenv(cmd, "cacert", "MAKESHIFT_CACERT")
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			path, _       = cmd.Flags().GetString("path")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+			dataArgs, _   = cmd.Flags().GetStringArray("data")
+
+			inputData        = processFiles(dataArgs)
+			useDirectoryPath = len(inputData) > 1
+			c                = client.New(host)
+			res              *http.Response
+			query            string
+			err              error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("path", path).
+			Str("query", query).
+			Str("cacert", cacertPath).
+			Any("input", inputData).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		for inputPath, contents := range inputData {
+			log.Debug().Str("path", path).Int("size", len(contents)).Send()
+			if useDirectoryPath {
+				query = path + "/" + filepath.Clean(inputPath)
+			} else {
+				// use flag value if supplied
+				if cmd.Flags().Changed("path") {
+					query = path
+				} else {
+					query = inputPath
+				}
+			}
+
+			query = fmt.Sprintf("/upload/%s", query)
+			res, _, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   query,
+				Method: http.MethodPost,
+				Body:   contents,
+			})
+			handleResponseError(res, host, query, err)
+		}
+	},
+}
+
+var uploadProfilesCmd = &cobra.Command{
+	Use: "profile [profile_id]",
+	Example: `
+  # upload a new profile
+  makeshift upload profile -d @compute.json kubernetes.json
+
+  # upload a new profile with a specific path
+  makeshift upload profile -d @kubernetes.json
+  makeshift upload profile -d '{"id": "custom", "data": {}}' kubernetes.json
+`,
+	Short: "Upload a new profile",
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			dataArgs, _   = cmd.Flags().GetStringArray("data")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+			profiles      = processProfiles(dataArgs)
+
+			c     = client.New(host)
+			res   *http.Response
+			query string
+			body  []byte
+			err   error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("query", query).
+			Str("cacert", cacertPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		// load files from args
+		for i, path := range args {
+			body, err = os.ReadFile(path)
+			if err != nil {
+				log.Error().Err(err).
+					Int("index", i).
+					Str("path", path).
+					Msg("failed to read profile file")
+				continue
+			}
+			var profile *makeshift.Profile
+			err = json.Unmarshal(body, &profile)
+			if err != nil {
+				log.Error().Err(err).
+					Int("index", i).
+					Str("path", path).
+					Msg("failed to unmarshal profile")
+			}
+			profiles = append(profiles, profile)
+		}
+
+		// send each loaded profile to server
+		for _, profile := range profiles {
+			if profile == nil {
+				continue
+			}
+
+			body, err = json.Marshal(profile)
+			if err != nil {
+				log.Error().Err(err).Msg("failed to marshal profile")
+				continue
+			}
+
+			query = fmt.Sprintf("/profiles/%s", profile.ID)
+			res, body, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   query,
+				Method: http.MethodPost,
+				Body:   body,
+			})
+			handleResponseError(res, host, query, err)
+		}
+	},
+}
+
+var uploadPluginsCmd = &cobra.Command{
+	Use: "plugin [plugin_name]",
+	Example: `
+  # upload a new plugin
+  makeshift upload plugin -d @slurm.so
+  makeshift upload plugin slurm.so
+`,
+	Args:  cobra.ExactArgs(1),
+	Short: "Upload a new plugin",
+	Run: func(cmd *cobra.Command, args []string) {
+		// make one request be host positional argument (restricted to 1 for now)
+		// temp := append(handleArgs(args), processDataArgs(dataArgs)...)
+		var (
+			host, _       = cmd.Flags().GetString("host")
+			dataArgs, _   = cmd.Flags().GetStringArray("data")
+			cacertPath, _ = cmd.Flags().GetString("cacert")
+
+			plugins = processFiles(dataArgs)
+			c       = client.New(host)
+			res     *http.Response
+			query   string
+			body    []byte
+			plugin  makeshift.Plugin
+			err     error
+		)
+
+		log.Debug().
+			Str("host", host).
+			Str("query", query).
+			Str("cacert", cacertPath).
+			Send()
+
+		if cacertPath != "" {
+			c.LoadCertificateFromPath(cacertPath)
+		}
+
+		// load files from args
+		for i, path := range args {
+			body, err = os.ReadFile(path)
+			if err != nil {
+				log.Error().Err(err).
+					Int("index", i).
+					Str("path", path).
+					Msg("failed to read plugin file")
+				continue
+			}
+
+			plugins[path] = body
+		}
+
+		for path, contents := range plugins {
+			plugin, err = service.LoadPluginFromFile(path)
+			if err != nil {
+				log.Error().Err(err).
+					Str("path", path).
+					Msg("failed to load plugin from file")
+			}
+
+			query = fmt.Sprintf("/plugins/%s", plugin.Name())
+			res, _, err = c.MakeRequest(client.HTTPEnvelope{
+				Path:   query,
+				Method: http.MethodPost,
+				Body:   contents,
+			})
+			handleResponseError(res, host, query, err)
+		}
 	},
 }
 
 func init() {
+	uploadCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)")
+	uploadCmd.PersistentFlags().StringArrayP("data", "d", []string{}, "Set the data to send to specified host (prepend @ for files)")
+	uploadCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load")
+
+	uploadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)")
+
+	uploadProfilesCmd.Flags().VarP(&inputFormat, "format", "F", "Set the input format for profile")
+
+	uploadCmd.AddCommand(uploadProfilesCmd, uploadPluginsCmd)
 	rootCmd.AddCommand(uploadCmd)
 }
+
+func processFiles(args []string) map[string][]byte {
+	// load data either from file or directly from args
+	var collection = make(map[string][]byte, len(args))
+	for _, arg := range args {
+		// skip empty string args
+		if len(arg) > 0 {
+			// determine if we're reading from file to load contents
+			if strings.HasPrefix(arg, "@") {
+				var path string = strings.TrimLeft(arg, "@")
+
+				// process sub-directories recursively
+				newCollection, err := processDir(path)
+				if err != nil {
+					log.Warn().
+						Err(err).
+						Str("path", path).
+						Msg("failed to process directory at path")
+				}
+				log.Trace().
+					Str("path", path).
+					Msg("new collection added at path")
+				maps.Copy(collection, newCollection)
+
+			} else {
+				log.Warn().Msg("only files can be uploaded (add @ before the path with '--data' flag)")
+
+				continue
+			}
+		}
+	}
+	return collection
+}
+
+// processProfiles takes a slice of strings that check for the @ symbol and loads
+// the contents from the file specified in place (which replaces the path).
+//
+// NOTE: The purpose is to make the input arguments uniform for our request. This
+// function is meant to handle data passed with the `-d/--data` flag and positional
+// args from the CLI.
+func processProfiles(args []string) []*makeshift.Profile {
+	// load data either from file or directly from args
+	var collection = make([]*makeshift.Profile, len(args))
+	for i, arg := range args {
+		// if arg is empty string, then skip and continue
+		if len(arg) > 0 {
+			// determine if we're reading from file to load contents
+			if strings.HasPrefix(arg, "@") {
+				var (
+					path     string = strings.TrimLeft(arg, "@")
+					contents []byte
+					data     *makeshift.Profile
+					err      error
+				)
+				contents, err = os.ReadFile(path)
+				if err != nil {
+					log.Error().
+						Err(err).
+						Str("path", path).
+						Msg("failed to read file")
+					continue
+				}
+
+				// skip empty files
+				if len(contents) == 0 {
+					log.Warn().
+						Str("path", path).
+						Msg("file is empty")
+					continue
+				}
+
+				// convert/validate input data
+				data, err = parseProfile(contents, format.DataFormatFromFileExt(path, inputFormat))
+				if err != nil {
+					log.Error().
+						Err(err).
+						Str("path", path).
+						Msg("failed to validate input from file")
+				}
+
+				// add loaded data to collection of all data
+				collection = append(collection, data)
+			} else {
+				// input should be a valid JSON
+				var (
+					data  *makeshift.Profile
+					input = []byte(arg)
+					err   error
+				)
+				if !json.Valid(input) {
+					log.Error().Msgf("argument %d not a valid JSON", i)
+					continue
+				}
+				err = json.Unmarshal(input, &data)
+				if err != nil {
+					log.Error().
+						Err(err).
+						Msgf("failed to unmarshal input for argument %d", i)
+				}
+				return []*makeshift.Profile{data}
+			}
+		}
+	}
+	return collection
+}
+
+func processDir(path string) (map[string][]byte, error) {
+	var (
+		collection = map[string][]byte{}
+		fileInfo   os.FileInfo
+		contents   []byte
+		err        error
+	)
+	// determine if path is directory
+	if fileInfo, err = os.Stat(path); err == nil {
+		if fileInfo.IsDir() {
+			filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+				if !d.IsDir() {
+					contents, err = os.ReadFile(path)
+					if err != nil {
+						log.Error().Err(err).Str("path", path).Msg("failed to read file")
+						return nil
+					}
+
+					// skip empty files
+					if len(contents) == 0 {
+						log.Warn().Str("path", path).Msg("file is empty")
+						return nil
+					}
+
+					log.Debug().
+						Str("path", path).
+						Msg("file added to collection")
+
+					// add loaded data to collection of all data
+					collection[path] = contents
+				} else {
+					// process sub-directories recursively
+					newCollection, err := processDir(path)
+					if err != nil {
+						return fmt.Errorf("failed to process directory at path '%s': %v", path, err)
+					}
+					log.Trace().
+						Str("path", path).
+						Msg("new collection added from nested directory")
+					maps.Copy(collection, newCollection)
+				}
+				return nil
+			})
+		} else {
+			contents, err = os.ReadFile(path)
+			if err != nil {
+				return collection, fmt.Errorf("failed to read file at path '%s': %v", path, err)
+			}
+
+			// skip empty files
+			if len(contents) == 0 {
+				return collection, fmt.Errorf("file is empty")
+			}
+
+			log.Debug().
+				Str("path", path).
+				Msg("file added to collection")
+
+			// add loaded data to collection of all data
+			collection[path] = contents
+		}
+	} else {
+		return nil, fmt.Errorf("failed to stat file: %v", err)
+	}
+	return collection, nil
+}
+
+func parseProfile(contents []byte, dataFormat format.DataFormat) (*makeshift.Profile, error) {
+	var (
+		data *makeshift.Profile
+		err  error
+	)
+
+	// convert/validate JSON input format
+	err = format.Unmarshal(contents, &data, dataFormat)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal profile: %v", err)
+	}
+	return data, nil
+}
+
+// ReadStdin reads all of standard input and returns the bytes. If an error
+// occurs during scanning, it is returned.
+func ReadStdin() ([]byte, error) {
+	var b []byte
+	input := bufio.NewScanner(os.Stdin)
+	for input.Scan() {
+		b = append(b, input.Bytes()...)
+		b = append(b, byte('\n'))
+		if len(b) == 0 {
+			break
+		}
+	}
+	if err := input.Err(); err != nil {
+		return b, fmt.Errorf("failed to read stdin: %w", err)
+	}
+	return b, nil
+}

examples/plugin/test.go

@@ -1,23 +0,0 @@
-package main
-
-import (
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/generator"
-)
-
-type TestGenerator struct{}
-
-func (g *TestGenerator) GetName() string    { return "test" }
-func (g *TestGenerator) GetVersion() string { return "v1.0.0" }
-func (g *TestGenerator) GetDescription() string {
-	return "This is a plugin creating for running tests."
-}
-func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) {
-	return generator.ApplyTemplates(generator.Mappings{
-		"plugin_name":        g.GetName(),
-		"plugin_version":     g.GetVersion(),
-		"plugin_description": g.GetDescription(),
-	}, params.Templates)
-}
-
-var Generator TestGenerator

examples/templates/conman.jinja

@@ -1,23 +0,0 @@
-#
-# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin:
-# Name:        {{ plugin_name }} 
-# Version:     {{ plugin_version }}
-# Description: {{ plugin_description }}
-# 
-# Source code:      https://github.com/OpenCHAMI/configurator
-# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins
-#
-SERVER keepalive=ON
-SERVER logdir="/var/log/conman"
-SERVER logfile="/var/log/conman.log"
-SERVER loopback=ON
-SERVER pidfile="/var/run/conman.pid"
-SERVER resetcmd="/usr/bin/powerman -0 \%N; sleep 5; /usr/bin/powerman -1 \%N"
-SERVER tcpwrappers=ON
-#SERVER timestamp=1h
-
-GLOBAL seropts="115200,8n1"
-GLOBAL log="/var/log/conman/console.\%N"
-GLOBAL logopts="sanitize,timestamp"
-
-{{ consoles }}

examples/templates/dhcpd.jinja

@@ -1,51 +0,0 @@
-#
-# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin:
-# Name:        {{ plugin_name }} 
-# Version:     {{ plugin_version }}
-# Description: {{ plugin_description }}
-# 
-# Source code:      https://github.com/OpenCHAMI/configurator
-# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins
-#
-allow booting;
-allow bootp;
-ddns-update-style interim;
-authoritative;
-
-option space ipxe;
-
-# Tell iPXE to not wait for ProxyDHCP requests to speed up boot.
-option ipxe.no-pxedhcp code 176 = unsigned integer 8;
-option ipxe.no-pxedhcp 1;
-
-option architecture-type   code 93  = unsigned integer 16;
-
-if exists user-class and option user-class = "iPXE" {
-    filename "http://%{IPADDR}/WW/ipxe/cfg/${mac}";
-} else {
-    if option architecture-type = 00:0B {
-        filename "/warewulf/ipxe/bin-arm64-efi/snp.efi";
-    } elsif option architecture-type = 00:0A {
-        filename "/warewulf/ipxe/bin-arm32-efi/placeholder.efi";
-    } elsif option architecture-type = 00:09 {
-        filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi";
-    } elsif option architecture-type = 00:07 {
-        filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi";
-    } elsif option architecture-type = 00:06 {
-        filename "/warewulf/ipxe/bin-i386-efi/snp.efi";
-    } elsif option architecture-type = 00:00 {
-        filename "/warewulf/ipxe/bin-i386-pcbios/undionly.kpxe";
-    }
-}
-
-subnet %{NETWORK} netmask %{NETMASK} {
-   not authoritative;
-   # option interface-mtu 9000;
-   option subnet-mask %{NETMASK};
-}
-
-# Compute Nodes (WIP - see the dhcpd generator plugin)
-{{ compute_nodes }}
-
-# Node entries will follow below
-{{ node_entries }}

examples/templates/dnsmasq.jinja

@@ -1,10 +0,0 @@
-#
-# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin:
-# Name:        {{ plugin_name }} 
-# Version:     {{ plugin_version }}
-# Description: {{ plugin_description }}
-# 
-# Source code:      https://github.com/OpenCHAMI/configurator
-# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins
-#
-{{ dhcp_hosts }}

examples/templates/powerman.jinja

@@ -1,18 +0,0 @@
-#
-# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin:
-# Name:        {{ plugin_name }} 
-# Version:     {{ plugin_version }}
-# Description: {{ plugin_description }}
-# 
-# Source code:      https://github.com/OpenCHAMI/configurator
-# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins
-#
-include "/etc/powerman/ipmipower.dev"
-include "/etc/powerman/ipmi.dev"
-
-
-# list of devices
-{{ devices }}
-
-# create nodes based on found nodes in hostfile
-{{ nodes }}

examples/templates/test.j2

@@ -1,16 +0,0 @@
-#
-# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin:
-# Name:        {{ plugin_name }} 
-# Version:     {{ plugin_version }}
-# Description: {{ plugin_description }}
-# 
-# Source code:      https://github.com/OpenCHAMI/configurator
-# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins
-#
-
-# TODO: test variables
-
-# TODO: test if/else statements
-
-# TODO: test for loops
-

go.mod

@@ -1,19 +1,21 @@
-module git.towk2.me/towk/configurator
+module git.towk2.me/towk/makeshift
 
-go 1.21.5
+go 1.24.4
+
+toolchain go1.24.6
 
 require (
 	github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18
+	github.com/cavaliergopher/grab/v3 v3.0.1
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/lestrrat-go/jwx/v2 v2.1.1
-	github.com/nikolalohinski/gonja/v2 v2.2.0
+	github.com/nikolalohinski/gonja/v2 v2.3.5
-	github.com/openchami/chi-middleware/auth v0.0.0-20240812224658-b16b83c70700
+	github.com/rs/zerolog v1.34.0
-	github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700
-	github.com/rodaine/table v1.2.0
-	github.com/rs/zerolog v1.33.0
 	github.com/spf13/cobra v1.8.0
-	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
+	github.com/tidwall/sjson v1.2.5
-	gopkg.in/yaml.v2 v2.4.0
+	go.yaml.in/yaml/v3 v3.0.4
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -22,7 +24,7 @@ require (
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kr/text v0.1.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/httprc v1.0.6 // indirect
@@ -36,7 +38,10 @@ require (
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tidwall/gjson v1.14.2 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	golang.org/x/crypto v0.25.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/sys v0.32.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
 )

go.sum

@@ -2,8 +2,11 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk=
 github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8=
+github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4=
+github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -13,27 +16,26 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
 github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
 github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
 github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
@@ -51,34 +53,27 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/nikolalohinski/gonja/v2 v2.2.0 h1:tAs3BDHNjvPj48F2BL5t7iVhN32HhgeldAl3EmdsLh8=
+github.com/nikolalohinski/gonja/v2 v2.3.5 h1:7ukCnsokmOIGXOjgW/WrM+xqgwjsQcU0ejFrrz4HQXk=
-github.com/nikolalohinski/gonja/v2 v2.2.0/go.mod h1:l9DuWJvT/BddBr2SsmEimESD6msSqRw7u5HzI2Um+sc=
+github.com/nikolalohinski/gonja/v2 v2.3.5/go.mod h1:UIzXPVuOsr5h7dZ5DUbqk3/Z7oFA/NLGQGMjqT4L2aU=
-github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
+github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
-github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
+github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
-github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
+github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
-github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
+github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
-github.com/openchami/chi-middleware/auth v0.0.0-20240812224658-b16b83c70700 h1:XADGipD2FZ9swuFUqeL7h63j3voiq9qA7P0aKsqgZKg=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
-github.com/openchami/chi-middleware/auth v0.0.0-20240812224658-b16b83c70700/go.mod h1:kswb9kU5cZAFRAvf1dAUJRWbQyjDEb0qkxW4ncDdEXg=
-github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700 h1:Gzt5f6RK39CHvY3SJudzBb/RK4tVh/S3CpJ0eQlbNdg=
-github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700/go.mod h1:UuXvr2loD4MtvZeKr57W0WpBs+gm0KM1kdtcXrE8M6s=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
-github.com/rodaine/table v1.2.0 h1:38HEnwK4mKSHQJIkavVj+bst1TEY7j9zhLMWu4QJrMA=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
-github.com/rodaine/table v1.2.0/go.mod h1:wejb/q/Yd4T/SVmBSRMr7GCq3KlcZp3gyNYdLSBhkaE=
+github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
-github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
-github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
-github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
@@ -89,38 +84,43 @@ github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyh
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
+go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
 golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
-golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
-golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
-golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
+golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
-golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
+golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/archive/archive.go

@@ -0,0 +1,220 @@
+package archive
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	makeshift "git.towk2.me/towk/makeshift/pkg"
+)
+
+func Create(filenames []string, buf io.Writer, hooks []makeshift.Hook) error {
+	// Create new Writers for gzip and tar
+	// These writers are chained. Writing to the tar writer will
+	// write to the gzip writer which in turn will write to
+	// the "buf" writer
+	gw := gzip.NewWriter(buf)
+	defer gw.Close()
+	tw := tar.NewWriter(gw)
+	defer tw.Close()
+
+	// Iterate over files and add them to the tar archive
+	for _, filename := range filenames {
+		err := addToArchive(tw, filename, hooks)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func Expand(tarname, xpath string) error {
+	tarfile, err := os.Open(tarname)
+	if err != nil {
+		return err
+	}
+	defer tarfile.Close()
+	// absPath, err := filepath.Abs(xpath)
+	// if err != nil {
+	// 	return err
+	// }
+	tr := tar.NewReader(tarfile)
+	if strings.HasSuffix(tarname, ".gz") {
+		gz, err := gzip.NewReader(tarfile)
+		if err != nil {
+			return fmt.Errorf("failed to create new gzip reader: %v", err)
+		}
+		defer gz.Close()
+		tr = tar.NewReader(gz)
+	}
+
+	// untar each segment
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("failed to get next tar header: %v", err)
+		}
+
+		// determine proper file path info
+		var (
+			fileinfo = header.FileInfo()
+			filename = header.Name
+			file     *os.File
+			abspath  string
+			dirpath  string
+		)
+
+		// absFileName := filepath.Join(absPath, filename) // if a dir, create it, then go to next segment
+		if fileinfo.Mode().IsDir() {
+			if err := os.MkdirAll(filename, 0o755); err != nil {
+				return fmt.Errorf("failed to make directory '%s': %v", filename, err)
+			}
+			continue
+		}
+
+		dirpath = filepath.Dir(filename)
+		if err = os.MkdirAll(dirpath, 0o777); err != nil {
+			return fmt.Errorf("failed to make directory '%s': %v", err)
+		}
+
+		// create new file with original file mode
+		abspath, err = filepath.Abs(filename)
+		if err != nil {
+			return fmt.Errorf("failed to get absolute path: %v", err)
+		}
+		file, err = os.OpenFile(
+			abspath,
+			os.O_RDWR|os.O_CREATE|os.O_TRUNC,
+			fileinfo.Mode().Perm(),
+		)
+		if err != nil {
+			return fmt.Errorf("failed to open file: %v", err)
+		}
+		// fmt.Printf("x %s\n", filename)
+
+		// copy the contents to the new file
+		n, err := io.Copy(file, tr)
+		if err != nil {
+			return fmt.Errorf("failed to copy file: %v", err)
+		}
+		if err = file.Close(); err != nil {
+			return fmt.Errorf("failed to close file: %v", err)
+		}
+		if n != fileinfo.Size() {
+			return fmt.Errorf("wrote %d, want %d", n, fileinfo.Size())
+		}
+	}
+	return nil
+}
+
+func addToArchive(tw *tar.Writer, filename string, hooks []makeshift.Hook) error {
+	var (
+		tempfile = fmt.Sprintf("%s.tmp", filename)
+		file     *os.File
+		contents []byte
+		data     any
+		err      error
+	)
+	// run pre-hooks to modify the contents of the file
+	// before archiving using plugins
+	for _, hook := range hooks {
+		// set the file in the data store before running hook
+		contents, err = os.ReadFile(filename)
+		if err != nil {
+			return fmt.Errorf("failed to read '%s' to download: %v", filename, err)
+		}
+		hook.Data.Set("file", contents)
+
+		err = hook.Init()
+		if err != nil {
+			return err
+		}
+		err = hook.Run()
+		if err != nil {
+			return err
+		}
+		err = hook.Cleanup()
+		if err != nil {
+			return err
+		}
+
+		// create temporary file to use to add to archive
+		hook = hooks[len(hooks)-1]
+		data, err = hook.Data.Get("out")
+		if err != nil {
+			return fmt.Errorf("failed to get output data from '%s' plugin: %v", hook.Plugin.Name(), err)
+		}
+
+		err = os.WriteFile(tempfile, data.([]byte), 0o777)
+		if err != nil {
+			return fmt.Errorf("failed to write temporary file: %v", err)
+		}
+	}
+
+	// use original file if no hooks to write archive
+	if len(hooks) == 0 {
+		file, err = os.Open(filename)
+	} else {
+		file, err = os.Open(tempfile)
+	}
+	if err != nil {
+		return fmt.Errorf("failed to open archive file: %v", err)
+	}
+	defer file.Close()
+
+	// get FileInfo for file size, mode, etc.
+	info, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	// skip file if it's a directory
+	if info.IsDir() {
+		return nil
+	}
+
+	// create a tar Header from the FileInfo data
+	header, err := tar.FileInfoHeader(info, info.Name())
+	if err != nil {
+		return fmt.Errorf("failed to create FileInfoHeader: %v", err)
+	}
+
+	// use full path as name (FileInfoHeader only takes the basename) to
+	// preserve directory structure
+	// see for more info: https://golang.org/src/archive/tar/common.go?#L626
+	header.Name = filename
+
+	// Write file header to the tar archive
+	err = tw.WriteHeader(header)
+	if err != nil {
+		return err
+	}
+
+	// copy file content to tar archive
+	if len(hooks) == 0 {
+		_, err = io.Copy(tw, file)
+	} else {
+		_, err = io.Copy(tw, strings.NewReader(string(data.([]byte))))
+	}
+	if err != nil {
+		return err
+	}
+
+	// delete the temporary file since we're done with it
+	if len(hooks) != 0 {
+		err = os.Remove(tempfile)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/format/format.go

@@ -0,0 +1,104 @@
+package format
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+
+	"gopkg.in/yaml.v3"
+)
+
+type DataFormat string
+
+const (
+	List DataFormat = "list"
+	JSON DataFormat = "json"
+	YAML DataFormat = "yaml"
+)
+
+func (df DataFormat) String() string {
+	return string(df)
+}
+
+func (df *DataFormat) Set(v string) error {
+	switch DataFormat(v) {
+	case List, JSON, YAML:
+		*df = DataFormat(v)
+		return nil
+	default:
+		return fmt.Errorf("must be one of %v", []DataFormat{
+			List, JSON, YAML,
+		})
+	}
+}
+
+func (df DataFormat) Type() string {
+	return "DataFormat"
+}
+
+// MarshalData marshals arbitrary data into a byte slice formatted as outFormat.
+// If a marshalling error occurs or outFormat is unknown, an error is returned.
+//
+// Supported values are: json, list, yaml
+func Marshal(data interface{}, outFormat DataFormat) ([]byte, error) {
+	switch outFormat {
+	case JSON:
+		if bytes, err := json.MarshalIndent(data, "", "  "); err != nil {
+			return nil, fmt.Errorf("failed to marshal data into JSON: %w", err)
+		} else {
+			return bytes, nil
+		}
+	case YAML:
+		if bytes, err := yaml.Marshal(data); err != nil {
+			return nil, fmt.Errorf("failed to marshal data into YAML: %w", err)
+		} else {
+			return bytes, nil
+		}
+	case List:
+		return nil, fmt.Errorf("this data format cannot be marshaled")
+	default:
+		return nil, fmt.Errorf("unknown data format: %s", outFormat)
+	}
+}
+
+// UnmarshalData unmarshals a byte slice formatted as inFormat into an interface
+// v. If an unmarshalling error occurs or inFormat is unknown, an error is
+// returned.
+//
+// Supported values are: json, list, yaml
+func Unmarshal(data []byte, v interface{}, inFormat DataFormat) error {
+	switch inFormat {
+	case JSON:
+		if err := json.Unmarshal(data, v); err != nil {
+			return fmt.Errorf("failed to unmarshal data into JSON: %w", err)
+		}
+	case YAML:
+		if err := yaml.Unmarshal(data, v); err != nil {
+			return fmt.Errorf("failed to unmarshal data into YAML: %w", err)
+		}
+	case List:
+		return fmt.Errorf("this data format cannot be unmarshaled")
+	default:
+		return fmt.Errorf("unknown data format: %s", inFormat)
+	}
+
+	return nil
+}
+
+// DataFormatFromFileExt determines the type of the contents
+// (JSON or YAML) based on the filname extension. The default
+// format is passed in, so if it doesn't match one of the cases,
+// that's what we will use. The defaultFmt value takes into account
+// both the standard default format (JSON) and any command line
+// change to that provided by options.
+func DataFormatFromFileExt(path string, defaultFmt DataFormat) DataFormat {
+	switch filepath.Ext(path) {
+	case ".json", ".JSON":
+		// The file is a JSON file
+		return JSON
+	case ".yaml", ".yml", ".YAML", ".YML":
+		// The file is a YAML file
+		return YAML
+	}
+	return defaultFmt
+}

main.go

@@ -1,6 +1,6 @@
 package main
 
-import "git.towk2.me/towk/configurator/cmd"
+import "git.towk2.me/towk/makeshift/cmd"
 
 func main() {
 	cmd.Execute()

pkg/client/client.go

@@ -0,0 +1,157 @@
+package client
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"git.towk2.me/towk/makeshift/pkg/util"
+	"github.com/cavaliergopher/grab/v3"
+)
+
+type HTTPBody []byte
+type HTTPHeader map[string]string
+type HTTPEnvelope struct {
+	Path   string
+	Method string
+	Header HTTPHeader
+	Body   HTTPBody
+	CACert string
+}
+
+type Client struct {
+	http.Client
+	BaseURI     string
+	AccessToken string
+}
+
+func New(uri string) Client {
+	return Client{
+		BaseURI: strings.TrimSuffix(uri, "/"),
+	}
+}
+
+func NewHTTPEnvelope() HTTPEnvelope {
+	return HTTPEnvelope{
+		Path:   "",
+		Method: http.MethodGet,
+		Header: nil,
+		Body:   nil,
+		CACert: "",
+	}
+}
+
+func (c *Client) MakeRequest(env HTTPEnvelope) (*http.Response, []byte, error) {
+	return util.MakeRequest(c.BaseURI+env.Path, env.Method, env.Body, env.Header)
+}
+
+func (c *Client) Download(out string, env HTTPEnvelope) (*grab.Response, error) {
+	if out == "" {
+		return grab.Get(out, c.BaseURI+env.Path)
+	}
+	return grab.Get(out, c.BaseURI+env.Path)
+}
+
+func (c *Client) UploadMultipartFile(uri, key, path string) (*http.Response, error) {
+	body, writer := io.Pipe()
+
+	req, err := http.NewRequest(http.MethodPost, uri, body)
+	if err != nil {
+		return nil, err
+	}
+
+	mwriter := multipart.NewWriter(writer)
+	req.Header.Add("Content-Type", mwriter.FormDataContentType())
+
+	errchan := make(chan error)
+
+	go func() {
+		defer close(errchan)
+		defer writer.Close()
+		defer mwriter.Close()
+
+		w, err := mwriter.CreateFormFile(key, path)
+		if err != nil {
+			errchan <- err
+			return
+		}
+
+		in, err := os.Open(path)
+		if err != nil {
+			errchan <- err
+			return
+		}
+		defer in.Close()
+
+		if written, err := io.Copy(w, in); err != nil {
+			errchan <- fmt.Errorf("error copying %s (%d bytes written): %v", path, written, err)
+			return
+		}
+
+		if err := mwriter.Close(); err != nil {
+			errchan <- err
+			return
+		}
+	}()
+
+	resp, err := c.Do(req)
+	merr := <-errchan
+
+	if err != nil || merr != nil {
+		return resp, fmt.Errorf("http error: %v, multipart error: %v", err, merr)
+	}
+
+	return resp, nil
+}
+
+func (c *Client) LoadCertificateFromPath(path string) error {
+	cacert, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read certificate at path: %s", path)
+	}
+	certPool := x509.NewCertPool()
+	certPool.AppendCertsFromPEM(cacert)
+	err = c.LoadCertificateFromPool(certPool)
+	if err != nil {
+		return fmt.Errorf("could not initialize certificate from pool: %v", err)
+	}
+	return nil
+}
+
+func (c *Client) LoadCertificateFromPool(certPool *x509.CertPool) error {
+	// make sure we have a valid cert pool
+	if certPool == nil {
+		return fmt.Errorf("invalid cert pool")
+	}
+
+	// make sure that we can access the internal client
+	c.Transport = &http.Transport{
+		TLSClientConfig: &tls.Config{
+			RootCAs:            certPool,
+			InsecureSkipVerify: false,
+		},
+		DisableKeepAlives: true,
+		Dial: (&net.Dialer{
+			Timeout:   120 * time.Second,
+			KeepAlive: 120 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout:   120 * time.Second,
+		ResponseHeaderTimeout: 120 * time.Second,
+	}
+	return nil
+}
+
+func mustOpen(f string) *os.File {
+	r, err := os.Open(f)
+	if err != nil {
+		panic(err)
+	}
+	return r
+}

pkg/generator/conman.go

@@ -1,56 +0,0 @@
-package generator
-
-import (
-	"fmt"
-
-	configurator "git.towk2.me/towk/configurator/pkg"
-	"git.towk2.me/towk/configurator/pkg/client"
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-)
-
-type Conman struct{}
-
-func (g *Conman) GetName() string {
-	return "conman"
-}
-
-func (g *Conman) GetVersion() string {
-	return util.GitCommit()
-}
-
-func (g *Conman) GetDescription() string {
-	return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
-}
-
-func (g *Conman) Generate(config *config.Config, params Params) (FileMap, error) {
-	var (
-		smdClient       = client.NewSmdClient(params.ClientOpts...)
-		eps             = []configurator.RedfishEndpoint{}
-		err       error = nil
-		consoles        = ""
-	)
-
-	// fetch required data from SMD to create config
-	eps, err = smdClient.FetchRedfishEndpoints(params.Verbose)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch redfish endpoints with client: %v", err)
-	}
-
-	// format output to write to config file
-	consoles = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n"
-	for _, ep := range eps {
-		consoles += fmt.Sprintf("CONSOLE name=%s dev=ipmi:%s-bmc ipmiopts=U:%s,P:%s,W:solpayloadsize\n", ep.Name, ep.Name, ep.User, ep.Password)
-	}
-	consoles += "# ====================================================================="
-
-	// apply template substitutions and return output as byte array
-	return ApplyTemplates(Mappings{
-		"plugin_name":        g.GetName(),
-		"plugin_version":     g.GetVersion(),
-		"plugin_description": g.GetDescription(),
-		"server_opts":        "",
-		"global_opts":        "",
-		"consoles":           consoles,
-	}, params.Templates)
-}

pkg/generator/coredhcp.go

@@ -1,26 +0,0 @@
-package generator
-
-import (
-	"fmt"
-
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-)
-
-type CoreDhcp struct{}
-
-func (g *CoreDhcp) GetName() string {
-	return "coredhcp"
-}
-
-func (g *CoreDhcp) GetVersion() string {
-	return util.GitCommit()
-}
-
-func (g *CoreDhcp) GetDescription() string {
-	return fmt.Sprintf("Configurator generator plugin for '%s' to generate config files. (WIP)", g.GetName())
-}
-
-func (g *CoreDhcp) Generate(config *config.Config, params Params) (FileMap, error) {
-	return nil, fmt.Errorf("plugin does not implement generation function")
-}

pkg/generator/dhcpd.go

@@ -1,64 +0,0 @@
-package generator
-
-import (
-	"fmt"
-
-	configurator "git.towk2.me/towk/configurator/pkg"
-	"git.towk2.me/towk/configurator/pkg/client"
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-)
-
-type DHCPd struct{}
-
-func (g *DHCPd) GetName() string {
-	return "dhcpd"
-}
-
-func (g *DHCPd) GetVersion() string {
-	return util.GitCommit()
-}
-
-func (g *DHCPd) GetDescription() string {
-	return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
-}
-
-func (g *DHCPd) Generate(config *config.Config, params Params) (FileMap, error) {
-	var (
-		smdClient          = client.NewSmdClient(params.ClientOpts...)
-		eths               = []configurator.EthernetInterface{}
-		computeNodes       = ""
-		err          error = nil
-	)
-
-	//
-	eths, err = smdClient.FetchEthernetInterfaces(params.Verbose)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %w", err)
-	}
-
-	// check if we have the required params first
-	if eths == nil {
-		return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)")
-	}
-	if len(eths) <= 0 {
-		return nil, fmt.Errorf("no ethernet interfaces found")
-	}
-
-	// format output to write to config file
-	computeNodes = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n"
-	for _, eth := range eths {
-		if len(eth.IpAddresses) == 0 {
-			continue
-		}
-		computeNodes += fmt.Sprintf("host %s { hardware ethernet %s; fixed-address %s} ", eth.ComponentId, eth.MacAddress, eth.IpAddresses[0])
-	}
-	computeNodes += "# ====================================================================="
-	return ApplyTemplates(Mappings{
-		"plugin_name":        g.GetName(),
-		"plugin_version":     g.GetVersion(),
-		"plugin_description": g.GetDescription(),
-		"compute_nodes":      computeNodes,
-		"node_entries":       "",
-	}, params.Templates)
-}

pkg/generator/dnsmasq.go

@@ -1,71 +0,0 @@
-package generator
-
-import (
-	"fmt"
-
-	configurator "git.towk2.me/towk/configurator/pkg"
-	"git.towk2.me/towk/configurator/pkg/client"
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-)
-
-type DNSMasq struct{}
-
-func (g *DNSMasq) GetName() string {
-	return "dnsmasq"
-}
-
-func (g *DNSMasq) GetVersion() string {
-	return util.GitCommit()
-}
-
-func (g *DNSMasq) GetDescription() string {
-	return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
-}
-
-func (g *DNSMasq) Generate(config *config.Config, params Params) (FileMap, error) {
-	// make sure we have a valid config first
-	if config == nil {
-		return nil, fmt.Errorf("invalid config (config is nil)")
-	}
-
-	// set all the defaults for variables
-	var (
-		smdClient       = client.NewSmdClient(params.ClientOpts...)
-		eths            = []configurator.EthernetInterface{}
-		err       error = nil
-	)
-
-	// if we have a client, try making the request for the ethernet interfaces
-	eths, err = smdClient.FetchEthernetInterfaces(params.Verbose)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err)
-	}
-
-	// check if we have the required params first
-	if eths == nil {
-		return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)")
-	}
-	if len(eths) <= 0 {
-		return nil, fmt.Errorf("no ethernet interfaces found")
-	}
-
-	// format output to write to config file
-	output := "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n"
-	for _, eth := range eths {
-		if eth.Type == "NodeBMC" {
-			output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n"
-		} else {
-			output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n"
-		}
-	}
-	output += "# ====================================================================="
-
-	// apply template substitutions and return output as byte array
-	return ApplyTemplates(Mappings{
-		"plugin_name":        g.GetName(),
-		"plugin_version":     g.GetVersion(),
-		"plugin_description": g.GetDescription(),
-		"dhcp_hosts":         output,
-	}, params.Templates)
-}

pkg/generator/example.go

@@ -1,31 +0,0 @@
-package generator
-
-import (
-	"fmt"
-
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-)
-
-type Example struct {
-	Message string
-}
-
-func (g *Example) GetName() string {
-	return "example"
-}
-
-func (g *Example) GetVersion() string {
-	return util.GitCommit()
-}
-
-func (g *Example) GetDescription() string {
-	return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
-}
-
-func (g *Example) Generate(config *config.Config, params Params) (FileMap, error) {
-	g.Message = `
-	This is an example generator plugin. See the file in 'internal/generator/plugins/example/example.go' on
-	information about constructing plugins and plugin requirements.`
-	return FileMap{"example": []byte(g.Message)}, nil
-}

pkg/generator/generator.go

@@ -1,264 +0,0 @@
-package generator
-
-import (
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"plugin"
-
-	configurator "git.towk2.me/towk/configurator/pkg"
-	"git.towk2.me/towk/configurator/pkg/client"
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-	"github.com/rs/zerolog/log"
-)
-
-type (
-	Mappings map[string]any
-	FileMap  map[string][]byte
-	FileList [][]byte
-
-	// Generator interface used to define how files are created. Plugins can
-	// be created entirely independent of the main driver program.
-	Generator interface {
-		GetName() string
-		GetVersion() string
-		GetDescription() string
-		Generate(config *config.Config, params Params) (FileMap, error)
-	}
-)
-
-var DefaultGenerators = createDefaultGenerators()
-
-func createDefaultGenerators() map[string]Generator {
-	var (
-		generatorMap = map[string]Generator{}
-		generators   = []Generator{
-			&Conman{}, &DHCPd{}, &DNSMasq{}, &Warewulf{}, &Example{}, &CoreDhcp{},
-		}
-	)
-	for _, g := range generators {
-		generatorMap[g.GetName()] = g
-	}
-	return generatorMap
-}
-
-// Converts the file outputs from map[string][]byte to map[string]string.
-func ConvertContentsToString(f FileMap) map[string]string {
-	n := make(map[string]string, len(f))
-	for k, v := range f {
-		n[k] = string(v)
-	}
-	return n
-}
-
-// Loads files without applying any Jinja 2 templating.
-func LoadFiles(paths ...string) (FileMap, error) {
-	var outputs = FileMap{}
-	for _, path := range paths {
-		expandedPaths, err := filepath.Glob(path)
-		if err != nil {
-			return nil, fmt.Errorf("failed to glob path: %w", err)
-		}
-		for _, expandedPath := range expandedPaths {
-			info, err := os.Stat(expandedPath)
-			if err != nil {
-				fmt.Println(err)
-				return nil, fmt.Errorf("failed to stat file or directory: %w", err)
-			}
-			// skip any directories found
-			if info.IsDir() {
-				continue
-			}
-			b, err := os.ReadFile(expandedPath)
-			if err != nil {
-				return nil, fmt.Errorf("failed to read file: %w", err)
-			}
-
-			outputs[expandedPath] = b
-		}
-	}
-
-	return outputs, nil
-}
-
-// Loads a single generator plugin given a single file path.
-func LoadPlugin(path string) (Generator, error) {
-	// skip loading plugin if path is a directory with no error
-	if isDir, err := util.IsDirectory(path); err == nil && isDir {
-		return nil, nil
-	} else if err != nil {
-		return nil, fmt.Errorf("failed to test if plugin path is directory: %w", err)
-	}
-
-	// try and open the plugin
-	p, err := plugin.Open(path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open plugin: %w", err)
-	}
-
-	// load the "Generator" symbol from plugin
-	symbol, err := p.Lookup("Generator")
-	if err != nil {
-		return nil, fmt.Errorf("failed to look up symbol at path '%s': %w", path, err)
-	}
-
-	// assert that the plugin loaded has a valid generator
-	gen, ok := symbol.(Generator)
-	if !ok {
-		return nil, fmt.Errorf("failed to load the correct symbol type at path '%s'", path)
-	}
-	return gen, nil
-}
-
-// Loads all generator plugins in a given directory.
-//
-// Returns a map of generators. Each generator can be accessed by the name
-// returned by the generator.GetName() implemented.
-func LoadPlugins(dirpath string, opts ...Option) (map[string]Generator, error) {
-	// check if verbose option is supplied
-	var (
-		generators = make(map[string]Generator)
-		params     = ToParams(opts...)
-	)
-
-	//
-	err := filepath.Walk(dirpath, func(path string, info fs.FileInfo, err error) error {
-		// skip trying to load generator plugin if directory or error
-		if info.IsDir() || err != nil {
-			return nil
-		}
-
-		// only try loading if file has .so extension
-		if filepath.Ext(path) != ".so" {
-			return nil
-		}
-
-		// load the generator plugin from current path
-		gen, err := LoadPlugin(path)
-		if err != nil {
-			return fmt.Errorf("failed to load generator in directory '%s': %w", path, err)
-		}
-
-		// show the plugins found if verbose flag is set
-		if params.Verbose {
-			log.Info().Str("plugin_name", gen.GetName()).Msg("found plugin")
-		}
-
-		// map each generator plugin by name for lookup
-		generators[gen.GetName()] = gen
-		return nil
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("failed to walk directory: %w", err)
-	}
-
-	return generators, nil
-}
-
-// Generate() is the main function to generate a collection of files and returns them as a map.
-// This function only expects a path to a plugin and paths to a collection of templates to
-// be used. This function will only load the plugin on-demand and fetch resources as needed.
-//
-// This function requires that a target and plugin path be set at minimum.
-func Generate(config *config.Config, plugin string, params Params) (FileMap, error) {
-	var (
-		gen Generator
-		ok  bool
-		err error
-	)
-
-	// check if generator is built-in first before loading external plugin
-	log.Debug().Any("generators", DefaultGenerators).Msg("available generators")
-	gen, ok = DefaultGenerators[plugin]
-	if !ok {
-		// only load the plugin needed for this target if we don't find default
-		log.Error().Str("plugin", plugin).Msg("could not find target in default generators")
-		gen, err = LoadPlugin(plugin)
-		if err != nil {
-			return nil, fmt.Errorf("failed to load plugin from file: %v", err)
-		}
-	}
-
-	return gen.Generate(config, params)
-}
-
-// Main function to generate a collection of files as a map with the path as the key and
-// the contents of the file as the value. This function currently expects a list of plugin
-// paths to load all plugins within a directory. Then, each plugin's generator.GenerateWithTarget()
-// function is called for each target specified.
-//
-// This function is the corresponding implementation for the "generate" CLI subcommand.
-// It is also call when running the configurator as a service with the "/generate" route.
-//
-// TODO: Separate loading plugins so we can load them once when running as a service.
-func GenerateWithTarget(config *config.Config, target string) (FileMap, error) {
-	// load generator plugins to generate configs or to print
-	var (
-		opts       []client.Option
-		targetInfo configurator.Target
-		generator  Generator
-		params     Params
-		err        error
-		ok         bool
-	)
-
-	// check if a target is supplied
-	if target == "" {
-		return nil, fmt.Errorf("must specify a target")
-	}
-
-	// load target information from config
-	targetInfo, ok = config.Targets[target]
-	if !ok {
-		log.Warn().Str("target", target).Msg("target not found in config")
-	}
-
-	// if no plugin supplied in config target, then using the target supplied
-	if targetInfo.Plugin == "" {
-		targetInfo.Plugin = target
-	}
-
-	// check if generator is built-in first before loading
-	generator, ok = DefaultGenerators[target]
-	if !ok {
-		// only load the plugin needed for this target if we don't find default
-		log.Warn().Str("target", target).Msg("could not find target in default generators")
-		generator, err = LoadPlugin(targetInfo.Plugin)
-		if err != nil {
-			return nil, fmt.Errorf("failed to load plugin: %v", err)
-		}
-	}
-
-	// check if there's at least one template available
-	if len(targetInfo.TemplatePaths) <= 0 {
-		return nil, fmt.Errorf("expects at least one template to be available")
-	}
-
-	// prepare params to pass into generator
-	params.Templates = map[string]Template{}
-	for _, templatePath := range targetInfo.TemplatePaths {
-		template := Template{}
-		template.LoadFromFile(templatePath)
-		params.Templates[templatePath] = template
-	}
-
-	// set the client options
-	if config.AccessToken != "" {
-		params.ClientOpts = append(opts, client.WithAccessToken(config.AccessToken))
-	}
-	if config.CertPath != "" {
-		params.ClientOpts = append(opts, client.WithCertPoolFile(config.CertPath))
-	}
-
-	// load files that are not to be copied
-	params.Files, err = LoadFiles(targetInfo.FilePaths...)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load files to copy: %v", err)
-	}
-
-	// run the generator plugin from target passed
-	return generator.Generate(config, params)
-}

pkg/generator/hostfile.go

@@ -1,26 +0,0 @@
-package generator
-
-import (
-	"fmt"
-
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-)
-
-type Hostfile struct{}
-
-func (g *Hostfile) GetName() string {
-	return "hostfile"
-}
-
-func (g *Hostfile) GetVersion() string {
-	return util.GitCommit()
-}
-
-func (g *Hostfile) GetDescription() string {
-	return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
-}
-
-func (g *Hostfile) Generate(config *config.Config, opts ...Option) (FileMap, error) {
-	return nil, fmt.Errorf("plugin does not implement generation function")
-}

pkg/generator/params.go

@@ -1,43 +0,0 @@
-package generator
-
-import (
-	configurator "git.towk2.me/towk/configurator/pkg"
-	"git.towk2.me/towk/configurator/pkg/client"
-	"git.towk2.me/towk/configurator/pkg/config"
-)
-
-type (
-	// Params used by the generator
-	Params struct {
-		Templates  map[string]Template
-		Files      map[string][]byte
-		ClientOpts []client.Option
-		Verbose    bool
-	}
-	Option func(Params)
-)
-
-func ToParams(opts ...Option) Params {
-	params := Params{}
-	for _, opt := range opts {
-		opt(params)
-	}
-	return params
-}
-
-func WithClientOpts(opts ...client.Option) Option {
-	return func(p Params) {
-		p.ClientOpts = opts
-	}
-}
-
-func WithTemplates(templates map[string]Template) Option {
-	return func(p Params) {
-		p.Templates = templates
-	}
-}
-
-// Helper function to get the target in generator.Generate() plugin implementations.
-func GetTarget(config *config.Config, key string) configurator.Target {
-	return config.Targets[key]
-}

pkg/generator/powerman.go

@@ -1,26 +0,0 @@
-package generator
-
-import (
-	"fmt"
-
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-)
-
-type Powerman struct{}
-
-func (g *Powerman) GetName() string {
-	return "powerman"
-}
-
-func (g *Powerman) GetVersion() string {
-	return util.GitCommit()
-}
-
-func (g *Powerman) GetDescription() string {
-	return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
-}
-
-func (g *Powerman) Generate(config *config.Config, opts ...Option) (FileMap, error) {
-	return nil, fmt.Errorf("plugin does not implement generation function")
-}

pkg/generator/syslog.go

@@ -1,26 +0,0 @@
-package generator
-
-import (
-	"fmt"
-
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-)
-
-type Syslog struct{}
-
-func (g *Syslog) GetName() string {
-	return "syslog"
-}
-
-func (g *Syslog) GetVersion() string {
-	return util.GitCommit()
-}
-
-func (g *Syslog) GetDescription() string {
-	return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName())
-}
-
-func (g *Syslog) Generate(config *config.Config, opts ...Option) (FileMap, error) {
-	return nil, fmt.Errorf("plugin does not implement generation function")
-}

pkg/generator/templates.go

@@ -1,98 +0,0 @@
-package generator
-
-import (
-	"bytes"
-	"fmt"
-	"os"
-
-	"git.towk2.me/towk/configurator/pkg/util"
-	"github.com/nikolalohinski/gonja/v2"
-	"github.com/nikolalohinski/gonja/v2/exec"
-	"github.com/rs/zerolog/log"
-)
-
-type Template struct {
-	Contents []byte `json:"contents"`
-}
-
-func (t *Template) LoadFromFile(path string) error {
-	// skip loading template if path is a directory with no error
-	if isDir, err := util.IsDirectory(path); err == nil && isDir {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("failed to test if template path is directory: %w", err)
-	}
-
-	// try and read the contents of the file
-	// NOTE: we don't care if this is actually a Jinja template
-	// or not...at least for now.
-	contents, err := os.ReadFile(path)
-	if err != nil {
-		return fmt.Errorf("failed to read file: %v", err)
-	}
-	t.Contents = contents
-	return nil
-}
-
-func (t *Template) IsEmpty() bool {
-	return len(t.Contents) <= 0
-}
-
-// Wrapper function to slightly abstract away some of the nuances with using gonja
-// into a single function call. This function is *mostly* for convenience and
-// simplication. If no paths are supplied, then no templates will be applied and
-// there will be no output.
-//
-// The "FileList" returns a slice of byte arrays in the same order as the argument
-// list supplied, but with the Jinja templating applied.
-func ApplyTemplates(mappings Mappings, templates map[string]Template) (FileMap, error) {
-	var (
-		data    = exec.NewContext(mappings)
-		outputs = FileMap{}
-	)
-
-	for path, template := range templates {
-		// load jinja template from file
-		t, err := gonja.FromBytes(template.Contents)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read template from file: %w", err)
-		}
-
-		// execute/render jinja template
-		b := bytes.Buffer{}
-		if err = t.Execute(&b, data); err != nil {
-			return nil, fmt.Errorf("failed to execute: %w", err)
-		}
-		outputs[path] = b.Bytes()
-	}
-
-	log.Debug().Any("templates", templates).Any("outputs", outputs).Any("mappings", mappings).Msg("apply templates")
-
-	return outputs, nil
-}
-
-// Wrapper function similiar to "ApplyTemplates" but takes file paths as arguments.
-// This function will load templates from a file instead of using file contents.
-func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) {
-	var (
-		data    = exec.NewContext(mappings)
-		outputs = FileMap{}
-	)
-
-	for _, path := range paths {
-		// load jinja template from file
-		t, err := gonja.FromFile(path)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read template from file: %w", err)
-		}
-
-		// execute/render jinja template
-		b := bytes.Buffer{}
-		if err = t.Execute(&b, data); err != nil {
-			return nil, fmt.Errorf("failed to execute: %w", err)
-		}
-		outputs[path] = b.Bytes()
-	}
-
-	return outputs, nil
-}

pkg/generator/warewulf.go

@@ -1,78 +0,0 @@
-package generator
-
-import (
-	"fmt"
-	"maps"
-	"strings"
-
-	"git.towk2.me/towk/configurator/pkg/client"
-	"git.towk2.me/towk/configurator/pkg/config"
-	"git.towk2.me/towk/configurator/pkg/util"
-	"github.com/rs/zerolog/log"
-)
-
-type Warewulf struct{}
-
-func (g *Warewulf) GetName() string {
-	return "warewulf"
-}
-
-func (g *Warewulf) GetVersion() string {
-	return util.GitCommit()
-}
-
-func (g *Warewulf) GetDescription() string {
-	return "Configurator generator plugin for 'warewulf' config files."
-}
-
-func (g *Warewulf) Generate(config *config.Config, params Params) (FileMap, error) {
-	var (
-		smdClient   = client.NewSmdClient(params.ClientOpts...)
-		outputs     = make(FileMap, len(params.Templates))
-		nodeEntries = ""
-		paths       = []string{}
-	)
-
-	// if we have a client, try making the request for the ethernet interfaces
-	eths, err := smdClient.FetchEthernetInterfaces(params.Verbose)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err)
-	}
-
-	// check if we have the required params first
-	if eths == nil {
-		return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)")
-	}
-	if len(eths) <= 0 {
-		return nil, fmt.Errorf("no ethernet interfaces found")
-	}
-
-	// fetch redfish endpoints and handle errors
-	eps, err := smdClient.FetchRedfishEndpoints(params.Verbose)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch redfish endpoints: %v", err)
-	}
-	if len(eps) <= 0 {
-		return nil, fmt.Errorf("no redfish endpoints found")
-	}
-
-	templates, err := ApplyTemplates(Mappings{
-		"node_entries": nodeEntries,
-	}, params.Templates)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load templates: %v", err)
-	}
-
-	maps.Copy(outputs, params.Files)
-	maps.Copy(outputs, templates)
-
-	// print message if verbose param is found
-	if params.Verbose {
-		for path, _ := range outputs {
-			paths = append(paths, path)
-		}
-	}
-	log.Info().Str("paths", strings.Join(paths, ":")).Msg("templates and files loaded: \n")
-
-	return outputs, err
-}

pkg/log/log.go

@@ -0,0 +1,119 @@
+package log
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"strings"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+)
+
+// string representation that directly corresponds to zerolog.Level
+type LogLevel string
+type LogLevelList []LogLevel
+type LogFilter string
+
+const (
+	DEBUG    LogLevel = "debug"
+	INFO     LogLevel = "info"
+	WARN     LogLevel = "warn"
+	ERROR    LogLevel = "error"
+	DISABLED LogLevel = "disabled"
+	TRACE    LogLevel = "trace"
+)
+
+var Levels = [6]LogLevel{DEBUG, INFO, WARN, ERROR, DISABLED, TRACE}
+var LogFile *os.File
+
+func (ll LogLevel) String() string {
+	return string(ll)
+}
+
+func (ll *LogLevel) Set(v string) error {
+	switch LogLevel(v) {
+	case DEBUG, INFO, WARN, ERROR, DISABLED, TRACE:
+		*ll = LogLevel(v)
+		return nil
+	default:
+		return fmt.Errorf("must be one of %v", []LogLevel{
+			DEBUG,
+			INFO,
+			WARN,
+			ERROR,
+			DISABLED,
+			TRACE,
+		})
+	}
+}
+
+func (df LogLevel) Type() string {
+	return "LogLevel"
+}
+
+func InitWithLogLevel(logLevel LogLevel, logPath string) error {
+	var (
+		logger  zerolog.Logger
+		level   zerolog.Level
+		writer  zerolog.LevelWriter
+		writers []io.Writer
+		err     error
+	)
+
+	// set the logging level
+	level, err = strToLogLevel(logLevel)
+	if err != nil {
+		return fmt.Errorf("failed to convert log level: %v", err)
+	}
+
+	// add the default stderr writer
+	writers = append(writers, &zerolog.FilteredLevelWriter{
+		Writer: &zerolog.LevelWriterAdapter{os.Stderr},
+		Level:  level,
+	})
+
+	// add another writer to write to a log file
+	if logPath != "" {
+		LogFile, err = os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664)
+		if err != nil {
+			return fmt.Errorf("failed to open log file: %v", err)
+		}
+
+		// add another write to write to the specified log file
+		writers = append(writers, &zerolog.FilteredLevelWriter{
+			Writer: zerolog.LevelWriterAdapter{LogFile},
+			Level:  level,
+		})
+	}
+	writer = zerolog.MultiLevelWriter(writers...)
+	logger = zerolog.New(writer).Level(level).With().Timestamp().Caller().Logger()
+	zerolog.SetGlobalLevel(level)
+	log.Logger = logger
+	return nil
+}
+
+func strToLogLevel(ll LogLevel) (zerolog.Level, error) {
+	var tostr = func(lls []LogLevel) []string {
+		s := []string{}
+		for _, l := range lls {
+			s = append(s, string(l))
+		}
+		return s
+	}
+
+	if index := slices.Index(Levels[:], ll); index >= 0 {
+		// handle special cases to map index to DISABLED and TRACE
+		switch index {
+		case 4:
+			return zerolog.Disabled, nil
+		case 5:
+			return zerolog.TraceLevel, nil
+		}
+		return zerolog.Level(index), nil
+	}
+	return -100, fmt.Errorf(
+		"invalid log level (options: %s)", strings.Join(tostr(Levels[:]), ", "),
+	) // use 'info' by default
+}

pkg/models.go

@@ -0,0 +1,52 @@
+package makeshift
+
+import (
+	"git.towk2.me/towk/makeshift/pkg/storage"
+)
+
+type ProfileMap map[string]*Profile
+type Profile struct {
+	ID          string         `json:"id"`                    // profile ID
+	Description string         `json:"description,omitempty"` // profile description
+	Tags        []string       `json:"tags,omitempty"`        // tags used for ...
+	Data        map[string]any `json:"data,omitempty"`        // include render data
+}
+
+type Plugin interface {
+	Name() string
+	Version() string
+	Description() string
+	Metadata() Metadata
+
+	Init() error
+	Run(data storage.KVStore, args []string) error
+	Cleanup() error
+}
+type Metadata map[string]any
+type Hook struct {
+	Data   storage.KVStore
+	Args   []string
+	Plugin Plugin
+}
+
+func (h *Hook) Init() error {
+	return h.Plugin.Init()
+}
+
+func (h *Hook) Run() error {
+	return h.Plugin.Run(h.Data, h.Args)
+}
+
+func (h *Hook) Cleanup() error {
+	return h.Plugin.Cleanup()
+}
+
+func PluginToMap(p Plugin) map[string]any {
+	return map[string]any{
+		"name":        p.Name(),
+		"version":     p.Version(),
+		"description": p.Description(),
+		"metadata":    p.Metadata(),
+	}
+
+}

pkg/plugin.go

@@ -1,14 +0,0 @@
-package configurator
-
-type Plugin interface {
-	// plugin data
-	Name() string
-	Version() string
-	Description() string
-	Metadata() map[string]string
-
-	// run the plugin
-	Init() error
-	Run() error
-	Cleanup() error
-}

pkg/plugins/jinja2/jinja2.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	makeshift "git.towk2.me/towk/makeshift/pkg"
+	"git.towk2.me/towk/makeshift/pkg/storage"
+	"github.com/nikolalohinski/gonja/v2"
+	"github.com/nikolalohinski/gonja/v2/exec"
+	"github.com/rs/zerolog/log"
+)
+
+type Jinja2 struct{}
+
+func (p *Jinja2) Name() string        { return "jinja2" }
+func (p *Jinja2) Version() string     { return "v0.0.1-alpha" }
+func (p *Jinja2) Description() string { return "Renders Jinja 2 templates" }
+func (p *Jinja2) Metadata() makeshift.Metadata {
+	return makeshift.Metadata{
+		"author": map[string]any{
+			"name":  "David J. Allen",
+			"email": "davidallendj@gmail.com",
+			"links": []string{
+				"https://github.com/davidallendj",
+				"https://git.towk2.me/towk",
+			},
+		},
+	}
+}
+
+func (p *Jinja2) Init() error {
+	// nothing to initialize
+	log.Debug().Str("plugin", p.Name()).Msg("jinja2.Init()")
+	return nil
+}
+
+func (p *Jinja2) Run(store storage.KVStore, args []string) error {
+	// render the files using Jinja 2 from args
+	var (
+		mappings struct {
+			Data map[string]any `json:"data"`
+		}
+		context  *exec.Context
+		template *exec.Template
+		profiles any // makeshift.ProfileMap
+		input    any // []byte
+		output   bytes.Buffer
+		err      error
+	)
+	log.Debug().
+		Str("plugin", p.Name()).
+		Any("store", store).
+		Strs("args", args).
+		Int("arg_count", len(args)).
+		Msg("(jinja2) Run()")
+
+	profiles, err = store.Get("profiles")
+	if err != nil {
+		return fmt.Errorf("(jinja2) failed to get profiles: %v", err)
+	}
+
+	input, err = store.Get("file")
+	if err != nil {
+		return fmt.Errorf("(jinja2) failed to get input data: %v", err)
+	}
+
+	// get the templates provided as args to the plugin
+	template, err = gonja.FromBytes(input.([]byte))
+	if err != nil {
+		return fmt.Errorf("(jinja2) failed to get template from args: %v", err)
+	}
+
+	// get mappings from shared data (optional)
+	shared, err := store.Get("shared")
+	if err != nil {
+		log.Warn().Err(err).Msg("(jinja2) could not retrieve shared data")
+	} else {
+		err = json.Unmarshal(shared.([]byte), &mappings)
+		if err != nil {
+			return fmt.Errorf("(jinja2) failed to unmarshal mappings from shared data: %v", err)
+		}
+	}
+
+	var ps = make(map[string]any)
+	for profileID, profile := range profiles.(makeshift.ProfileMap) {
+		ps[profileID] = map[string]any{
+			"id":          profile.ID,
+			"description": profile.Description,
+			"data":        profile.Data,
+		}
+	}
+
+	// inject profiles and plugin-specific mapping
+	mappings.Data = map[string]any{
+		"makeshift": map[string]any{
+			"profiles": ps,
+			"plugin": map[string]any{
+				"name":        p.Name(),
+				"version":     p.Version(),
+				"description": p.Description(),
+				"metadata":    p.Metadata(),
+			},
+		},
+	}
+
+	log.Debug().Any("mappings", mappings).Send()
+
+	// use the provided data in the store to render templates
+	// NOTE: this may be changed to specifically use "shared" data instead
+	context = exec.NewContext(mappings.Data)
+	if err = template.Execute(&output, context); err != nil { // Prints: Hello Bob!
+		return fmt.Errorf("(jinja2) failed to render template: %v", err)
+	}
+
+	// write render templates to data store output
+	store.Set("out", output.Bytes())
+	return nil
+}
+
+func (p *Jinja2) Cleanup() error {
+	// nothing to clean up
+	log.Debug().Str("plugin", p.Name()).Msg("(jinja2) Cleanup()")
+	return nil
+}
+
+var Makeshift Jinja2

pkg/plugins/mapper/mapper.go

@@ -0,0 +1,30 @@
+package main
+
+import "git.towk2.me/towk/makeshift/pkg/storage"
+
+type Mapper struct{}
+
+func (p *Mapper) Name() string        { return "jinja2" }
+func (p *Mapper) Version() string     { return "test" }
+func (p *Mapper) Description() string { return "Renders Jinja 2 templates" }
+func (p *Mapper) Metadata() map[string]string {
+	return map[string]string{
+		"author.name":  "David J. Allen",
+		"author.email": "davidallendj@gmail.com",
+	}
+}
+
+func (p *Mapper) Init() error {
+	// nothing to initialize
+	return nil
+}
+
+func (p *Mapper) Run(data storage.KVStore, args []string) error {
+	return nil
+}
+
+func (p *Mapper) Clean() error {
+	return nil
+}
+
+var Makeshift Mapper

pkg/plugins/smd/smd.go

@@ -0,0 +1,279 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	makeshift "git.towk2.me/towk/makeshift/pkg"
+	"git.towk2.me/towk/makeshift/pkg/storage"
+	"github.com/rs/zerolog/log"
+)
+
+// An struct that's meant to extend functionality of the base HTTP client by
+// adding commonly made requests to SMD. The implemented functions are can be
+// used in generator plugins to fetch data when it is needed to substitute
+// values for the Jinja templates used.
+type SmdClient struct {
+	http.Client `json:"-" yaml:"-"`
+	Host        string `yaml:"host"`
+	Port        int    `yaml:"port"`
+	AccessToken string `yaml:"access-token"`
+
+	RedfishEndpoints   []RedfishEndpoint   `json:"redfish_endpoints"`
+	EthernetInterfaces []EthernetInterface `json:"ethernet_interfaces"`
+	Components         []Component         `json:"components"`
+}
+
+type IPAddr struct {
+	IpAddress string `json:"IPAddress"`
+	Network   string `json:"Network"`
+}
+
+type EthernetInterface struct {
+	ID          string   `json:"ID"`
+	Description string   `json:"Description"`
+	MacAddress  string   `json:"MACAddr"`
+	LastUpdate  string   `json:"LastUpdate"`
+	ComponentID string   `json:"ComponentID"`
+	Type        string   `json:"Type"`
+	IpAddresses []IPAddr `json:"IPAddresses"`
+}
+
+type Component struct {
+	ID                  string      `json:"ID"`
+	Type                string      `json:"Type"`
+	State               string      `json:"State,omitempty"`
+	Flag                string      `json:"Flag,omitempty"`
+	Enabled             *bool       `json:"Enabled,omitempty"`
+	SwStatus            string      `json:"SoftwareStatus,omitempty"`
+	Role                string      `json:"Role,omitempty"`
+	SubRole             string      `json:"SubRole,omitempty"`
+	NID                 json.Number `json:"NID,omitempty"`
+	Subtype             string      `json:"Subtype,omitempty"`
+	NetType             string      `json:"NetType,omitempty"`
+	Arch                string      `json:"Arch,omitempty"`
+	Class               string      `json:"Class,omitempty"`
+	ReservationDisabled bool        `json:"ReservationDisabled,omitempty"`
+	Locked              bool        `json:"Locked,omitempty"`
+}
+
+type RedfishEndpoint struct {
+	ID          string `json:"ID"`
+	Type        string `json:"Type"`
+	Name        string `json:"Name,omitempty"` // user supplied descriptive name
+	Hostname    string `json:"Hostname"`
+	Domain      string `json:"Domain"`
+	FQDN        string `json:"FQDN"`
+	Enabled     bool   `json:"Enabled"`
+	UUID        string `json:"UUID,omitempty"`
+	User        string `json:"User"`
+	Password    string `json:"Password"` // Temporary until more secure method
+	UseSSDP     bool   `json:"UseSSDP,omitempty"`
+	MACRequired bool   `json:"MACRequired,omitempty"`
+	MACAddr     string `json:"MACAddr,omitempty"`
+	IPAddr      string `json:"IPAddress,omitempty"`
+}
+
+func (p *SmdClient) Name() string        { return "smd" }
+func (p *SmdClient) Version() string     { return "v0.0.1-alpha" }
+func (p *SmdClient) Description() string { return "Fetchs data from SMD and writes to store" }
+func (p *SmdClient) Metadata() makeshift.Metadata {
+	return makeshift.Metadata{
+		"author": map[string]any{
+			"name":  "David J. Allen",
+			"email": "davidallendj@gmail.com",
+			"links": []string{
+				"https://github.com/davidallendj",
+				"https://git.towk2.me/towk",
+			},
+		},
+	}
+}
+
+func (p *SmdClient) Init() error {
+	log.Debug().Str("plugin", p.Name()).Msg("(smd) Init()")
+	return nil
+}
+
+func (p *SmdClient) Run(store storage.KVStore, args []string) error {
+	// set all the defaults for variables
+	var (
+		client SmdClient
+		bytes  []byte
+		err    error
+	)
+
+	log.Debug().
+		Str("plugin", p.Name()).
+		Strs("args", args).
+		Int("arg_count", len(args)).
+		Any("store", store).
+		Msg("(smd) Run()")
+
+	// if we have a client, try making the request for the ethernet interfaces
+	err = client.FetchEthernetInterfaces()
+	if err != nil {
+		return fmt.Errorf("(smd) failed to fetch ethernet interfaces with client: %v", err)
+	}
+
+	err = client.FetchRedfishEndpoints()
+	if err != nil {
+		return fmt.Errorf("(smd) failed to fetch redfish endpoints with client: %v", err)
+	}
+	err = client.FetchComponents()
+	if err != nil {
+		return fmt.Errorf("(smd) failed to fetch components with client: %v", err)
+	}
+
+	// write data back to shared data store to be used by other plugins
+	bytes, err = json.Marshal(client)
+	if err != nil {
+		return fmt.Errorf("(smd) failed to marshal SMD client: %v", err)
+	}
+	store.Set("shared", bytes)
+
+	// apply template substitutions and return output as byte array
+	return nil
+}
+
+func (p *SmdClient) Cleanup() error {
+	log.Debug().Str("plugin", p.Name()).Msg("(smd) Init()")
+	return nil
+}
+
+// Fetch the ethernet interfaces from SMD service using its API. An access token may be required if the SMD
+// service SMD_JWKS_URL envirnoment variable is set.
+func (client *SmdClient) FetchEthernetInterfaces() error {
+	var (
+		bytes []byte
+		err   error
+	)
+	// make request to SMD endpoint
+	bytes, err = client.makeRequest("/Inventory/EthernetInterfaces")
+	if err != nil {
+		return fmt.Errorf("failed to read HTTP response: %v", err)
+	}
+
+	// unmarshal response body JSON and extract in object
+	err = json.Unmarshal(bytes, &client.EthernetInterfaces)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal response: %v", err)
+	}
+
+	// print what we got if verbose is set
+	log.Debug().Str("ethernet_interfaces", string(bytes)).Msg("found interfaces")
+
+	return nil
+}
+
+// Fetch the components from SMD using its API. An access token may be required if the SMD
+// service SMD_JWKS_URL envirnoment variable is set.
+func (client *SmdClient) FetchComponents() error {
+	var (
+		bytes []byte
+		err   error
+	)
+	// make request to SMD endpoint
+	bytes, err = client.makeRequest("/State/Components")
+	if err != nil {
+		return fmt.Errorf("failed to make HTTP request: %v", err)
+	}
+
+	// make sure our response is actually JSON first
+	if !json.Valid(bytes) {
+		return fmt.Errorf("expected valid JSON response: %v", string(bytes))
+	}
+
+	// unmarshal response body JSON and extract in object
+	var tmp map[string]any
+	err = json.Unmarshal(bytes, &tmp)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal response: %v", err)
+	}
+	bytes, err = json.Marshal(tmp["RedfishEndpoints"].([]any))
+	if err != nil {
+		return fmt.Errorf("failed to marshal JSON: %v", err)
+	}
+	err = json.Unmarshal(bytes, &client.Components)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal response: %v", err)
+	}
+
+	// print what we got if verbose is set
+	log.Debug().Str("components", string(bytes)).Msg("found components")
+
+	return nil
+}
+
+// TODO: improve implementation of this function
+func (client *SmdClient) FetchRedfishEndpoints() error {
+	var (
+		store map[string]any
+		rfeps []RedfishEndpoint
+		body  []byte
+		err   error
+	)
+
+	// make initial request to get JSON with 'RedfishEndpoints' as property
+	body, err = client.makeRequest("/Inventory/RedfishEndpoints")
+	if err != nil {
+		return fmt.Errorf("failed to make HTTP resquest: %v", err)
+	}
+	// make sure response is in JSON
+	if !json.Valid(body) {
+		return fmt.Errorf("expected valid JSON response: %s", string(body))
+	}
+	err = json.Unmarshal(body, &store)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal response: %v", err)
+	}
+
+	// marshal RedfishEndpoint JSON back to makeshift.RedfishEndpoint
+	body, err = json.Marshal(store["RedfishEndpoints"].([]any))
+	if err != nil {
+		return fmt.Errorf("failed to marshal JSON: %v", err)
+	}
+	err = json.Unmarshal(body, &rfeps)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal response: %v", err)
+	}
+
+	// show the final result
+	log.Debug().Bytes("redfish_endpoints", body).Msg("found redfish endpoints")
+
+	client.RedfishEndpoints = rfeps
+	return nil
+}
+
+func (client *SmdClient) makeRequest(endpoint string) ([]byte, error) {
+	if client == nil {
+		return nil, fmt.Errorf("client is nil")
+	}
+
+	// fetch DHCP related information from SMD's endpoint:
+	url := fmt.Sprintf("%s/hsm/v2%s", client.Host, endpoint)
+	req, err := http.NewRequest(http.MethodGet, url, bytes.NewBuffer([]byte{}))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create new HTTP request: %v", err)
+	}
+
+	// include access token in authorzation header if found
+	// NOTE: This shouldn't be needed for this endpoint since it's public
+	if client.AccessToken != "" {
+		req.Header.Add("Authorization", "Bearer "+client.AccessToken)
+	}
+
+	// make the request to SMD
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to make request: %v", err)
+	}
+
+	// read the contents of the response body
+	return io.ReadAll(res.Body)
+}
+
+var Makeshift SmdClient

pkg/service/constants.go

@@ -0,0 +1,77 @@
+package service
+
+const (
+	RELPATH_PLUGINS  = "/plugins"
+	RELPATH_PROFILES = "/profiles"
+	RELPATH_DATA     = "/data"
+	RELPATH_METADATA = "/.makeshift"
+	RELPATH_HELP     = RELPATH_DATA + "/index.html"
+	RELPATH_PROFILE  = RELPATH_PROFILES + "/default.json"
+
+	PATH_CONFIG = "$HOME/.config/makeshift/config.yaml"
+
+	DEFAULT_TIMEOUT_IN_SECS    = 60
+	DEFAULT_PLUGINS_MAX_COUNT  = 32
+	DEFAULT_PROFILES_MAX_COUNT = 256
+
+	FILE_METADATA  = ``
+	FILE_HOME_PAGE = `
+<!DOCTYPE html>
+<html>
+	<body>
+	<p>
+	Plugin Information:
+		Name: 			{{ makeshift.plugin.name }}
+		Version: 		{{ makeshift.plugin.version }}
+		Description: 	{{ makeshift.plugin.description }}
+		Author: 		{{ makeshift.plugin.metadata.name }} ({{ makeshift.plugin.metadata.email }})
+	
+	Profile Information:
+		ID:				{{ makeshift.profiles.default.id }}
+		Description:	{{ makeshift.profiles.default.description }}
+
+	# setup environment variables</br>
+	export MAKESHIFT_HOST={{ makeshift.profiles.default.data.host }}</br>
+	export MAKESHIFT_PATH={{ makeshift.profiles.default.data.path }}</br>
+	export MAKESHIFT_ROOT={{ makeshift.profiles.default.data.server_root }}</br></br>
+	
+	# start the service</br>
+	makeshift serve --root ./tests --init -l debug</br></br>
+	
+	# download a file or directory (as archive)</br>
+	makeshift download</br>
+	makeshift download --host http://localhost:5050 --path help.txt</br></br>
+	
+	# download files with rendering using plugins</br>
+	makeshift download --plugins smd,jinja2 --profile compute</br>
+	makeshift download -p templates --plugins jinja --profile io</br>
+	curl $MAKESHIFT_HOST/download/help.txt?plugins=smd,jinja2</br></br>
+	
+	# upload a file or directory (recursively)</br>
+	makeshift upload</br>
+	makeshift upload --host http://localhost:5050 --path help.txt</br></br>
+	
+	# list the files in a directory</br>
+	makeshift list --path help.txt</br>
+	makeshift list --host http://localhost:5050 --path help.txt</br>
+	curl http://localhost:5050/list/test</br>
+	</p>
+	<body>
+</html>
+`
+	FILE_DEFAULT_PROFILE = `
+{
+  "id": "default",
+  "description": "Makeshift default profile",
+  "data": {
+	"host": "localhost",
+	"path": "/test",
+	"server_root": "./test"
+  }
+}
+`
+)
+
+// makeshift.host: https://localhost:5050
+// makeshift.path: test
+// makeshift.server.root: $HOME/apps/makeshift

pkg/service/plugins.go

@@ -0,0 +1,141 @@
+package service
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+
+	makeshift "git.towk2.me/towk/makeshift/pkg"
+	"github.com/go-chi/chi/v5"
+	"github.com/rs/zerolog/log"
+)
+
+func (s *Service) ListPlugins() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			plugins map[string]makeshift.Plugin
+			names   []string
+			body    []byte
+			err     error
+		)
+
+		plugins, err = LoadPluginsFromDir(s.PathForPlugins())
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		for name := range plugins {
+			names = append(names, name)
+		}
+
+		body, err = json.Marshal(names)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Write(body)
+	}
+}
+
+func (s *Service) GetPluginInfo() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			pluginName = chi.URLParam(r, "name")
+			path       = s.PathForPluginWithName(pluginName)
+			plugin     makeshift.Plugin
+			body       []byte
+			err        error
+		)
+
+		plugin, err = LoadPluginFromFile(path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		body, err = json.Marshal(makeshift.PluginToMap(plugin))
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Write(body)
+	}
+}
+
+func (s *Service) GetPluginRaw() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			pluginName = chi.URLParam(r, "name")
+			path       = s.PathForPluginWithName(pluginName)
+			rawPlugin  []byte
+			err        error
+		)
+
+		rawPlugin, err = os.ReadFile(path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Write(rawPlugin)
+	}
+}
+
+func (s *Service) CreatePlugin() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			pluginName = chi.URLParam(r, "name")
+			body       []byte
+			path       string
+			err        error
+		)
+
+		// helper to check for valid plugin name
+		var hasValidName = func(name string) bool {
+			return name != "" && len(name) < 64
+		}
+
+		// check for a valid plugin name
+		if !hasValidName(pluginName) {
+			http.Error(w, "invalid name for plugin", http.StatusBadRequest)
+			return
+		}
+
+		body, err = io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		path = s.PathForPluginWithName(pluginName)
+		err = os.WriteFile(path, body, 0o777)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+func (s *Service) DeletePlugin() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			pluginName = chi.URLParam(r, "name")
+			path       = s.PathForPluginWithName(pluginName)
+			err        error
+		)
+
+		log.Debug().Str("path", path).Send()
+		err = os.Remove(path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}

pkg/service/profile.go

@@ -1,184 +0,0 @@
-package service
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/fs"
-	"net/http"
-	"os"
-	"path/filepath"
-
-	"github.com/go-chi/chi/v5"
-)
-
-type Profile struct {
-	ID          string         `json:"id"`          // profile ID
-	Description string         `json:"description"` // profile description
-	Tags        []string       `json:"tags"`        // tags used for ...
-	Paths       []string       `json:"paths"`       // paths to download
-	Plugins     []string       `json:"plugins"`     // plugins to run
-	Data        map[string]any `json:"data"`        // include render data
-}
-
-func (s *Service) GetProfiles() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		var (
-			path     = s.RootPath + PROFILES_RELPATH
-			profiles []*Profile
-			contents []byte
-			err      error
-		)
-
-		// walk profiles directory to load all profiles
-		err = filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-
-			// skip directories
-			if info.IsDir() {
-				return nil
-			}
-
-			// read file contents
-			var profile *Profile
-			profile, err = LoadProfile(path)
-			if err != nil {
-				return err
-			}
-
-			profiles = append(profiles, profile)
-
-			fmt.Println(path, info.Size())
-			return nil
-		})
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-		}
-
-		// marshal and send all the profiles
-		contents, err = json.Marshal(profiles)
-		if err != nil {
-			http.Error(w, fmt.Sprintf("failed to marshal profiles: %v", err), http.StatusInternalServerError)
-		}
-
-		_, err = w.Write(contents)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-	}
-}
-
-// func (s *Service) CreateProfiles() http.HandlerFunc {
-// 	return func(w http.ResponseWriter, r *http.Request) {
-// 		var (
-// 			path = chi.URLParam(r, "path")
-// 			err error
-// 		)
-
-// 	}
-// }
-
-func (s *Service) GetProfile() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		var (
-			id       = chi.URLParam(r, "id")
-			path     = s.BuildProfilePath(id)
-			profile  *Profile
-			contents []byte
-			err      error
-		)
-
-		profile, err = LoadProfile(path)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-
-		contents, err = json.Marshal(profile)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		_, err = w.Write(contents)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-	}
-}
-
-func (s *Service) CreateProfile() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		type input struct {
-			path    string   `json:"path"`
-			profile *Profile `json:"profile"`
-		}
-		var (
-			body []byte
-			in   input
-			err  error
-		)
-
-		body, err = io.ReadAll(r.Body)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		// use the request info to build profile
-		err = json.Unmarshal(body, &in)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-
-		// create a new profile on disk
-		os.WriteFile()
-
-		w.WriteHeader(http.StatusOK)
-	}
-}
-
-func (s *Service) CreateProfileVar() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func (s *Service) DeleteProfileVar() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func (s *Service) GetProfileData() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func (s *Service) GetProfileVar() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func (s *Service) CreateProfilePath() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func (s *Service) DeleteProfilePath() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func (s *Service) GetProfilePath() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func (s *Service) GetPlugins() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func (s *Service) CreatePlugins() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func (s *Service) DeletePlugins() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}

pkg/service/profiles.go

@@ -0,0 +1,278 @@
+package service
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	makeshift "git.towk2.me/towk/makeshift/pkg"
+	"github.com/go-chi/chi/v5"
+	"github.com/tidwall/sjson"
+)
+
+func (s *Service) ListProfiles() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			path     = s.RootPath + RELPATH_PROFILES
+			profiles []*makeshift.Profile
+			contents []byte
+			err      error
+		)
+
+		// walk profiles directory to load all profiles
+		err = filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			// skip directories
+			if info.IsDir() {
+				return nil
+			}
+
+			// read file contents
+			var profile *makeshift.Profile
+			profile, err = LoadProfileFromFile(path)
+			if err != nil {
+				return err
+			}
+
+			profiles = append(profiles, profile)
+
+			fmt.Println(path, info.Size())
+			return nil
+		})
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+
+		// marshal and send all the profiles
+		contents, err = json.Marshal(profiles)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("failed to marshal profiles: %v", err), http.StatusInternalServerError)
+		}
+
+		_, err = w.Write(contents)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+func (s *Service) GetProfile() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			id       = chi.URLParam(r, "id")
+			path     = s.PathForProfileWithID(id)
+			contents []byte
+			err      error
+		)
+
+		contents, err = loadProfileContents(path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+
+		_, err = w.Write(contents)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+func (s *Service) CreateProfile() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			body, contents []byte
+			path           string
+			profile        *makeshift.Profile
+			err            error
+		)
+
+		body, err = io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// use the request info to build profile
+		err = json.Unmarshal(body, &profile)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("failed to unmarshal profile: %v", err.Error()), http.StatusBadRequest)
+			return
+		}
+
+		// serialize just the profile part
+		contents, err = json.Marshal(profile)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("failed to marshal profile: %v", err.Error()), http.StatusBadRequest)
+			return
+		}
+
+		// create a new profile on disk
+		path = s.PathForProfileWithID(profile.ID)
+		err = os.WriteFile(path, contents, os.ModePerm)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+func (s *Service) DeleteProfile() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			profileID = chi.URLParam(r, "id")
+			path      string
+			err       error
+		)
+
+		if profileID == "default" {
+			http.Error(w, "cannot delete the default profile", http.StatusBadRequest)
+			return
+		}
+
+		path = s.PathForProfileWithID(profileID)
+		err = os.Remove(path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+func (s *Service) SetProfileData() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			body, contents []byte
+			newContents    string
+			profile        *makeshift.Profile
+			path           string
+			err            error
+		)
+
+		body, err = io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		err = json.Unmarshal(body, &profile)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		// make sure the request data sets an ID
+		if profile.ID == "" {
+			http.Error(w, "ID must be set to a non-empty value", http.StatusBadRequest)
+			return
+		}
+
+		// read the contents the file with profile ID
+		path = s.PathForProfileWithID(profile.ID)
+		contents, err = os.ReadFile(path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// modify the data of the profile's contents
+		newContents, err = sjson.Set(string(contents), "data", profile.Data)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// write only the data to the file with ID
+		err = os.WriteFile(path, []byte(newContents), os.ModePerm)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+func (s *Service) DeleteProfileData() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			id      = chi.URLParam(r, "id")
+			path    = s.PathForProfileWithID(id)
+			profile *makeshift.Profile
+			err     error
+		)
+
+		// get the profile
+		profile, err = LoadProfileFromFile(path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		}
+
+		// delete the profile data
+		profile.Data = map[string]any{}
+
+		// save the profile back to the file to update
+		SaveProfileToFile(path, profile)
+
+	}
+}
+
+func (s *Service) GetProfileData() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			id      = chi.URLParam(r, "id")
+			path    = s.PathForProfileWithID(id)
+			profile *makeshift.Profile
+			body    []byte
+			err     error
+		)
+
+		profile, err = LoadProfileFromFile(path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		// only marshal the profile data and not entire profile
+		body, err = json.Marshal(profile.Data)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// write body to response
+		_, err = w.Write(body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+func loadProfileContents(path string) ([]byte, error) {
+	var (
+		contents []byte
+		profile  *makeshift.Profile
+		err      error
+	)
+	profile, err = LoadProfileFromFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load profile from file: %v", err)
+	}
+
+	contents, err = json.Marshal(profile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal profile: %v", err)
+	}
+	return contents, nil
+}

pkg/service/routes.go

@@ -3,36 +3,374 @@ package service
 import (
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/fs"
 	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"git.towk2.me/towk/makeshift/internal/archive"
+	makeshift "git.towk2.me/towk/makeshift/pkg"
+	"git.towk2.me/towk/makeshift/pkg/storage"
+	"github.com/rs/zerolog/log"
 )
 
 func (s *Service) Download() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			path        = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download")
+			pluginNames = strings.Split(r.URL.Query().Get("plugins"), ",")
+			profileIDs  = strings.Split(r.URL.Query().Get("profiles"), ",")
 
+			fileInfo os.FileInfo
+			out      *os.File
+			store    *storage.MemoryStorage = new(storage.MemoryStorage)
+			hooks    []makeshift.Hook
+			contents []byte
+			errs     []error
+			err      error
+		)
+
+		// initialize storage
+		store.Init()
+
+		log.Debug().
+			Str("path", path).
+			Str("client_host", r.Host).
+			Strs("plugins", pluginNames).
+			Strs("profiles", profileIDs).
+			Any("query", r.URL.Query()).
+			Msg("Service.Download()")
+
+		// prepare profiles
+		errs = s.loadProfiles(profileIDs, store, errs)
+		if len(errs) > 0 {
+			log.Error().Errs("errs", errs).Msg("errors occurred loading profiles")
+			errs = []error{}
+		}
+
+		// determine if path is directory, file, or exists
+		if fileInfo, err = os.Stat(path); err == nil {
+			if fileInfo.IsDir() {
+				// get the final archive path
+				archivePath := fmt.Sprintf("%s.tar.gz", path)
+
+				log.Debug().
+					Str("archive_path", archivePath).
+					Str("type", "directory").
+					Msg("Service.Download()")
+
+				out, err = os.Create(archivePath)
+				if err != nil {
+					s.writeErrorResponse(w, fmt.Sprintf("failed to create named file: %v", err), http.StatusInternalServerError)
+					return
+				}
+
+				// get a list of filenames to archive
+				filenamesToArchive := []string{}
+				filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+					if !d.IsDir() {
+						filenamesToArchive = append(filenamesToArchive, path)
+					}
+					return nil
+				})
+				log.Debug().Strs("files", filenamesToArchive).Send()
+
+				// prepare plugins
+				hooks, errs = s.loadPlugins(pluginNames, store, nil, errs)
+				if len(errs) > 0 {
+					log.Error().Errs("errs", errs).Msg("errors occurred loading plugins")
+					errs = []error{}
+				}
+
+				// create an archive of the directory, run hooks, and download
+				err = archive.Create(filenamesToArchive, out, hooks)
+				if err != nil {
+					s.writeErrorResponse(w, fmt.Sprintf("failed to create archive: %v", err.Error()), http.StatusInternalServerError)
+					return
+				}
+
+				// load the final archive
+				contents, err = os.ReadFile(archivePath)
+				if err != nil {
+					s.writeErrorResponse(w, fmt.Sprintf("failed to read archive contents: %v", err.Error()), http.StatusInternalServerError)
+					return
+				}
+
+				// send the archive back as response
+				w.Header().Add("FILETYPE", "archive")
+				w.Write(contents)
+
+				// clean up the temporary archive
+				err = os.Remove(archivePath)
+				if err != nil {
+					log.Error().Err(err).Msg("failed to remove temporary archive")
+					return
+				}
+
+			} else {
+				// download individual file
+				log.Debug().
+					Str("type", "file").
+					Msg("Service.Download()")
+
+				contents, err = os.ReadFile(path)
+				if err != nil {
+					s.writeErrorResponse(w, fmt.Sprintf("failed to read file to download: %v", err), http.StatusInternalServerError)
+					return
+				}
+
+				// prepare plugins
+
+				store.Set("file", contents)
+				hooks, errs = s.loadPlugins(pluginNames, store, nil, errs)
+				if len(errs) > 0 {
+					log.Error().Errs("errs", errs).Msg("errors occurred loading plugins")
+					errs = []error{}
+				}
+				if len(hooks) > 0 {
+					// run pre-hooks to modify the contents of the file before archiving
+					log.Debug().Int("hook_count", len(hooks)).Msg("running hooks")
+					for _, hook := range hooks {
+						log.Debug().Any("hook", map[string]any{
+							"store": hook.Data,
+							"args":  hook.Args,
+							"plugin": map[string]string{
+								"name":        hook.Plugin.Name(),
+								"description": hook.Plugin.Description(),
+								"version":     hook.Plugin.Version(),
+							},
+						}).Send()
+						err = hook.Init()
+						if err != nil {
+							log.Error().
+								Err(err).
+								Str("plugin", hook.Plugin.Name()).
+								Msg("failed to initialize plugin")
+							continue
+						}
+						err = hook.Run()
+						if err != nil {
+							log.Error().
+								Err(err).
+								Str("plugin", hook.Plugin.Name()).
+								Msg("failed to run plugin")
+							continue
+						}
+						err = hook.Cleanup()
+						if err != nil {
+							log.Error().
+								Err(err).
+								Str("plugin", hook.Plugin.Name()).
+								Msg("failed to cleanup plugin")
+							continue
+						}
+					}
+
+					// take the contents from the last hook and update files
+					var (
+						hook = hooks[len(hooks)-1]
+						data any
+					)
+					data, err = hook.Data.Get("out")
+					if err != nil {
+						s.writeErrorResponse(w, fmt.Sprintf("failed to get data from hook: %v", err), http.StatusInternalServerError)
+						return
+					}
+
+					// send processed (with plugins) file back as response
+					w.Header().Add("FILETYPE", "file")
+					w.Write(data.([]byte))
+				} else {
+					// send non-processed file back as response
+					w.Header().Add("FILETYPE", "file")
+					w.Write(contents)
+				}
+
+			}
+		} else {
+			s.writeErrorResponse(w, err.Error(), http.StatusBadRequest)
+			return
+		}
 	}
 }
 
 func (s *Service) Upload() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			path    = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/upload")
+			body    []byte
+			dirpath string
+			err     error
+		)
 
+		// show what we're uploading
+		log.Debug().
+			Str("path", path).
+			Msg("Service.Upload()")
+
+		// take the provided path and store the file contents
+		dirpath = filepath.Dir(path)
+		err = os.MkdirAll(dirpath, 0o777)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// write file to disk
+		body, err = io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		err = os.WriteFile(path, body, 0o777)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.WriteHeader(http.StatusOK)
 	}
 }
 
 func (s *Service) List() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			path    = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/list")
+			entries []string
+			body    []byte
+			err     error
+		)
 
+		// show what we're listing
+		log.Debug().Str("path", path).Msg("Service.List()")
+
+		// walk directory and show all entries "ls"
+		err = filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			entries = append(entries, d.Name())
+			return nil
+		})
+		if err != nil {
+			switch err {
+			case fs.ErrNotExist, fs.ErrInvalid:
+				http.Error(w, "No such file or directory...", http.StatusBadRequest)
+			case fs.ErrPermission:
+				http.Error(w, "Invalid permissions...", http.StatusForbidden)
+			default:
+				http.Error(w, "Something went wrong (file or directory *probably* does not exist)...", http.StatusInternalServerError)
+			}
+			return
+		}
+
+		body, err = json.Marshal(entries)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Write(body)
+	}
+}
+
+func (s *Service) Delete() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/delete")
+			err  error
+		)
+
+		err = os.RemoveAll(path)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.WriteHeader(http.StatusOK)
 	}
 }
 
 func (s *Service) GetStatus(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
-	data := map[string]any{
+	err := json.NewEncoder(w).Encode(map[string]any{
-		"code":    200,
+		"code":    http.StatusOK,
-		"message": "Configurator is healthy",
+		"message": "The makeshift server is healthy",
-	}
+	})
-	err := json.NewEncoder(w).Encode(data)
 	if err != nil {
-		fmt.Printf("failed to encode JSON: %v\n", err)
+		fmt.Printf("failed to encode JSON response body: %v\n", err)
 		return
 	}
 }
+
+func (s *Service) loadProfiles(profileIDs []string, store storage.KVStore, errs []error) []error {
+	// load data from profiles into the data store
+	var profiles = make(makeshift.ProfileMap, len(profileIDs))
+	for i, profileID := range profileIDs {
+		var (
+			profilePath = s.PathForProfileWithID(profileID)
+			profile     *makeshift.Profile
+			err         error
+		)
+		if i > s.ProfilesMaxCount {
+			log.Warn().Msg("max profiles count reached...stopping")
+			return errs
+		}
+		if profileID == "" {
+			log.Warn().Msg("profile ID is empty...skipping")
+			continue
+		}
+		log.Debug().
+			Str("id", profileID).
+			Str("path", profilePath).
+			Msg("load profile")
+		profile, err = LoadProfileFromFile(profilePath)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		profiles[profileID] = profile
+	}
+	store.Set("profiles", profiles)
+
+	return errs
+}
+
+func (s *Service) loadPlugins(pluginNames []string, store storage.KVStore, args []string, errs []error) ([]makeshift.Hook, []error) {
+	// create hooks to run from provided plugins specified
+	var hooks []makeshift.Hook
+	for i, pluginName := range pluginNames {
+		var (
+			pluginPath string = s.PathForPluginWithName(pluginName)
+			plugin     makeshift.Plugin
+			err        error
+		)
+		if i > s.PluginsMaxCount {
+			log.Warn().Msg("max plugins count reached or exceeded...stopping")
+			return hooks, errs
+		}
+		if pluginName == "" {
+			log.Warn().Msgf("no plugin name found with index %d...skipping", i)
+			continue
+		}
+		log.Debug().
+			Str("name", pluginName).
+			Str("path", pluginPath).
+			Msg("load plugin")
+
+		// load the plugin from disk
+		plugin, err = LoadPluginFromFile(pluginPath)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		hooks = append(hooks, makeshift.Hook{
+			Data:   store,
+			Args:   args,
+			Plugin: plugin,
+		})
+	}
+	return hooks, errs
+}

pkg/service/service.go

@@ -3,43 +3,83 @@ package service
 import (
 	"encoding/json"
 	"fmt"
+	"io/fs"
+	"net/http"
 	"os"
+	"path/filepath"
+	"plugin"
+	"slices"
 	"time"
 
-	configurator "git.towk2.me/towk/configurator/pkg"
+	makeshift "git.towk2.me/towk/makeshift/pkg"
-	"github.com/go-chi/chi/middleware"
+	"git.towk2.me/towk/makeshift/pkg/util"
 	"github.com/go-chi/chi/v5"
-)
+	"github.com/go-chi/chi/v5/middleware"
-
+	"github.com/rs/zerolog/log"
-const (
-	PLUGINS_RELPATH   = "/plugins"
-	TEMPLATES_RELPATH = "/templates"
-	PROFILES_RELPATH  = "/profiles"
 )
 
 type Service struct {
+	Addr          string
 	RootPath      string `yaml:"root,omitempty"`
-	Environment map[string]string
+	CACertFile    string `yaml:"cacert,omitempty"`
+	CACertKeyfile string `yaml:"keyfile,omitempty"`
 
 	// max counts
 	PluginsMaxCount  int
 	ProfilesMaxCount int
+	Timeout          time.Duration
 }
 
-// New creates the directories at specified path
+// New creates a new Service instance with default values
 func New() *Service {
 	return &Service{
-		RootPath: ".",
+		Addr:             ":5050",
-		Environment: map[string]string{
+		RootPath:         "./",
-			"CONFIGURATOR_HOST_URI": "",
+		PluginsMaxCount:  DEFAULT_PLUGINS_MAX_COUNT,
-			"ACCESS_TOKEN":          "",
+		ProfilesMaxCount: DEFAULT_PROFILES_MAX_COUNT,
-		},
+		Timeout:          DEFAULT_TIMEOUT_IN_SECS,
-		PluginsMaxCount:  64,
-		ProfilesMaxCount: 256,
 	}
 }
 
-// Serve() starts the configurator service and waits for requests.
+// Init() sets up the default files and directories for the service
+func (s *Service) Init() error {
+	// create the default directories
+	var err error
+	err = os.MkdirAll(s.RootPath, 0o777)
+	if err != nil {
+		return fmt.Errorf("failed to make service root path: %v", err)
+	}
+	err = os.MkdirAll(s.PathForPlugins(), 0o777)
+	if err != nil {
+		return fmt.Errorf("failed to make service plugin path: %v", err)
+	}
+	err = os.MkdirAll(s.PathForProfiles(), 0o777)
+	if err != nil {
+		return fmt.Errorf("failed to make service profile path: %v", err)
+	}
+	err = os.MkdirAll(s.PathForData(), 0o777)
+	if err != nil {
+		return fmt.Errorf("failed to make service data path: %v", err)
+	}
+
+	// create the default files
+	err = os.WriteFile(s.PathForMetadata(), []byte(FILE_METADATA), 0o777)
+	if err != nil {
+		return fmt.Errorf("failed to make service metadata file: %v", err)
+	}
+	err = os.WriteFile(s.PathForHome(), []byte(FILE_HOME_PAGE), 0o777)
+	if err != nil {
+		return fmt.Errorf("failed to make service home page file: %v", err)
+	}
+	err = os.WriteFile(s.PathForProfileWithID("default"), []byte(FILE_DEFAULT_PROFILE), 0o777)
+	if err != nil {
+		return fmt.Errorf("failed to make service default profile file: %v", err)
+	}
+
+	return nil
+}
+
+// Serve() starts the makeshift service and waits for requests.
 func (s *Service) Serve() error {
 	router := chi.NewRouter()
 	router.Use(middleware.RequestID)
@@ -47,38 +87,41 @@ func (s *Service) Serve() error {
 	router.Use(middleware.Logger)
 	router.Use(middleware.Recoverer)
 	router.Use(middleware.StripSlashes)
-	router.Use(middleware.Timeout(60 * time.Second))
+	router.Use(middleware.Timeout(s.Timeout * time.Second))
 
 	if s.requireAuth() {
 
 	} else {
 		// general
-		router.Get("/download", s.Download())
+		router.Get("/download/*", s.Download())
-		router.Post("/upload", s.Upload())
+		router.Post("/upload/*", s.Upload())
-		router.Get("/list", s.List())
+		router.Get("/list/*", s.List())
+		router.Delete("/delete/*", s.Delete())
 
 		// profiles
-		router.Get("/profiles", s.GetProfiles())
+		router.Get("/profiles", s.ListProfiles())
-		// router.Post("/profiles", s.CreateProfiles())
+		router.Get("/profiles/{id}", s.GetProfile())
-		router.Get("/profile/{id}", s.GetProfile())
+		router.Post("/profiles/{id}", s.CreateProfile())
-		router.Post("/profile/{id}", s.CreateProfile())
+		router.Delete("/profiles/{id}", s.DeleteProfile())
-		router.Post("/profile/{id}/data/{varname}", s.CreateProfileVar())
+		router.Get("/profiles/{id}/data", s.GetProfileData())
-		router.Delete("/profile/{id}/data/{varname}", s.DeleteProfileVar())
+		router.Post("/profiles/{id}/data", s.SetProfileData())
-		router.Get("/profile/{id}/data", s.GetProfileData())
+		router.Delete("/profiles/{id}/data", s.DeleteProfileData())
-		router.Get("/profile/{id}/data/{varname}", s.GetProfileVar())
-		router.Post("/profile/{id}/paths/{path}", s.CreateProfilePath())
-		router.Delete("/profile/{id}/paths/{path}", s.DeleteProfilePath())
-		router.Get("/profile/{id}/paths/{path}", s.GetProfilePath())
 
 		// plugins
-		router.Get("/plugins", s.GetPlugins())
+		router.Get("/plugins", s.ListPlugins())
-		router.Post("/plugins", s.CreatePlugins())
+		router.Get("/plugins/{name}/info", s.GetPluginInfo())
-		router.Delete("/plugins/{id}", s.DeletePlugins())
+		router.Get("/plugins/{name}/raw", s.GetPluginRaw())
+		router.Post("/plugins/{name}", s.CreatePlugin())
+		router.Delete("/plugins/{name}", s.DeletePlugin())
 	}
 
 	// always available public routes go here
 	router.HandleFunc("/status", s.GetStatus)
-	return nil
+	if s.CACertFile != "" && s.CACertKeyfile != "" {
+		return http.ListenAndServeTLS(s.Addr, s.CACertFile, s.CACertKeyfile, router)
+	} else {
+		return http.ListenAndServe(s.Addr, router)
+	}
 }
 
 func (s *Service) requireAuth() bool {
@@ -89,15 +132,96 @@ func (s *Service) FetchJwks(uri string) {
 
 }
 
-func LoadProfile(path string) (*Profile, error) {
+func LoadProfileFromFile(path string) (*makeshift.Profile, error) {
-	return LoadFromJSONFile[Profile](path)
+	return loadFromJSONFile[makeshift.Profile](path)
 }
 
-func LoadPlugin(path string) (*configurator.Plugin, error) {
+// LoadPluginFromFile loads a single plugin given a single file path
-	return LoadFromJSONFile[configurator.Plugin](path)
+func LoadPluginFromFile(path string) (makeshift.Plugin, error) {
+	var (
+		isDir        bool
+		err          error
+		loadedPlugin *plugin.Plugin
+	)
+	// skip loading plugin if path is a directory with no error
+	isDir, err = util.IsDirectory(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to test if plugin path is directory: %v", err)
+	} else if isDir {
+		return nil, fmt.Errorf("path is a directory")
+	}
+
+	// try and open the plugin
+	loadedPlugin, err = plugin.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open plugin at path '%s': %v", path, err)
+	}
+
+	// load the "Target" symbol from plugin
+	symbol, err := loadedPlugin.Lookup("Makeshift")
+	if err != nil {
+		return nil, fmt.Errorf("failed to look up symbol at path '%s': %v", path, err)
+	}
+
+	// assert that the plugin is a valid makeshift.Plugin
+	target, ok := symbol.(makeshift.Plugin)
+	if !ok {
+		return nil, fmt.Errorf("failed to assert the correct symbol type at path '%s'", path)
+	}
+	return target, nil
 }
 
-func LoadFromJSONFile[T any](path string) (*T, error) {
+// LoadPluginsFromDir loads all plugins in a given directory.
+//
+// Returns a map of plugins. Each plugin can be accessed by the name
+// returned by the plugin.GetName() implemented.
+func LoadPluginsFromDir(dirpath string) (map[string]makeshift.Plugin, error) {
+	// check if verbose option is supplied
+	var (
+		cps = make(map[string]makeshift.Plugin)
+		err error
+	)
+
+	// helper to check for valid extensions
+	var hasValidExt = func(path string) bool {
+		return slices.Contains([]string{".so", ".dylib", ".dll"}, filepath.Ext(path))
+	}
+
+	// walk all files in directory only loading *valid* plugins
+	err = filepath.Walk(dirpath, func(path string, info fs.FileInfo, err error) error {
+		// skip trying to load generator plugin if directory or error
+		// only try loading if file has .so extension
+		if info.IsDir() || err != nil || !hasValidExt(path) {
+			return nil
+		}
+
+		// load the plugin from current path
+		p, err := LoadPluginFromFile(path)
+		if err != nil {
+			return fmt.Errorf("failed to load plugin in directory '%s': %v", path, err)
+		}
+
+		// map each plugin by name for lookup
+		cps[p.Name()] = p
+		return nil
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to walk directory '%s': %v", dirpath, err)
+	}
+
+	return cps, nil
+}
+
+func SaveProfileToFile(path string, profile *makeshift.Profile) error {
+	return saveToJSONFile(path, profile)
+}
+
+func SavePluginToFile(path string, plugin *makeshift.Plugin) error {
+	return saveToJSONFile(path, plugin)
+}
+
+func loadFromJSONFile[T any](path string) (*T, error) {
 	var (
 		res      *T
 		contents []byte
@@ -106,17 +230,63 @@ func LoadFromJSONFile[T any](path string) (*T, error) {
 
 	contents, err = os.ReadFile(path)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read plugin file: %v", err)
+		return nil, fmt.Errorf("failed to read file: %v", err)
 	}
 
 	err = json.Unmarshal(contents, &res)
 	if err != nil {
-		return nil, fmt.Errorf("failed to unmarshal plugin: %v", err)
+		return nil, fmt.Errorf("failed to unmarshal contents from JSON: %v", err)
 	}
 
 	return res, err
 }
 
-func (s *Service) BuildProfilePath(id string) string {
+func saveToJSONFile[T any](path string, data T) error {
-	return s.RootPath + PLUGINS_RELPATH + "/" + id
+	var (
+		contents []byte
+		err      error
+	)
+	contents, err = json.Marshal(data)
+	if err != nil {
+		return fmt.Errorf("failed to marshal data to JSON: %v", err)
+	}
+	err = os.WriteFile(path, contents, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("failed to write JSON to file: %v", err)
+	}
+
+	return nil
+}
+
+func (s *Service) PathForProfileWithID(id string) string {
+	return s.RootPath + RELPATH_PROFILES + "/" + id + ".json"
+}
+
+func (s *Service) PathForPluginWithName(name string) string {
+	return s.RootPath + RELPATH_PLUGINS + "/" + name + ".so"
+}
+
+func (s *Service) PathForProfiles() string {
+	return s.RootPath + RELPATH_PROFILES + "/"
+}
+
+func (s *Service) PathForPlugins() string {
+	return s.RootPath + RELPATH_PLUGINS + "/"
+}
+
+func (s *Service) PathForData() string {
+	return s.RootPath + RELPATH_DATA
+}
+
+func (s *Service) PathForMetadata() string {
+	return s.RootPath + RELPATH_METADATA
+}
+
+func (s *Service) PathForHome() string {
+	return s.RootPath + RELPATH_HELP
+}
+
+func (s *Service) writeErrorResponse(w http.ResponseWriter, message string, code int) {
+	http.Error(w, message, code)
+	log.Error().Msg(message)
 }

pkg/storage/disk.go

@@ -0,0 +1,23 @@
+package storage
+
+type DiskStorage struct{}
+
+func (ds DiskStorage) Init() error {
+	return nil
+}
+
+func (ds DiskStorage) Cleanup() error {
+	return nil
+}
+
+func (ds DiskStorage) Get(k string) error {
+	return nil
+}
+
+func (ds DiskStorage) Set(k string, v any) error {
+	return nil
+}
+
+func (ds DiskStorage) GetData() any {
+	return nil
+}

pkg/storage/memory.go

@@ -0,0 +1,33 @@
+package storage
+
+import "fmt"
+
+type MemoryStorage struct {
+	Data map[string]any `json:"data"`
+}
+
+func (ms *MemoryStorage) Init() error {
+	ms.Data = map[string]any{}
+	return nil
+}
+
+func (ms *MemoryStorage) Cleanup() error {
+	return nil
+}
+
+func (ms *MemoryStorage) Get(k string) (any, error) {
+	v, ok := ms.Data[k]
+	if ok {
+		return v, nil
+	}
+	return nil, fmt.Errorf("value '%s' does not exist", k)
+}
+
+func (ms *MemoryStorage) Set(k string, v any) error {
+	ms.Data[k] = v
+	return nil
+}
+
+func (ms *MemoryStorage) GetData() any {
+	return ms.Data
+}

pkg/storage/storage.go

@@ -0,0 +1,19 @@
+package storage
+
+type KVStore interface {
+	Init() error
+	Cleanup() error
+
+	Get(k string) (any, error)
+	Set(k string, v any) error
+	GetData() any
+}
+
+type KVStaticStore[T any] interface {
+	Init() error
+	Cleanup() error
+
+	Get(k string) (T, error)
+	Set(k string, v T) error
+	GetData() T
+}

pkg/util/util.go

@@ -1,10 +1,8 @@
 package util
 
 import (
-	"archive/tar"
 	"bytes"
 	"cmp"
-	"compress/gzip"
 	"crypto/tls"
 	"fmt"
 	"io"
@@ -99,64 +97,3 @@ func CopyIf[T comparable](s []T, condition func(t T) bool) []T {
 	}
 	return f
 }
-
-func CreateArchive(files []string, buf io.Writer) error {
-	// Create new Writers for gzip and tar
-	// These writers are chained. Writing to the tar writer will
-	// write to the gzip writer which in turn will write to
-	// the "buf" writer
-	gw := gzip.NewWriter(buf)
-	defer gw.Close()
-	tw := tar.NewWriter(gw)
-	defer tw.Close()
-
-	// Iterate over files and add them to the tar archive
-	for _, file := range files {
-		err := addToArchive(tw, file)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func addToArchive(tw *tar.Writer, filename string) error {
-	// open file to write to archive
-	file, err := os.Open(filename)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	// get FileInfo for file size, mode, etc.
-	info, err := file.Stat()
-	if err != nil {
-		return err
-	}
-
-	// create a tar Header from the FileInfo data
-	header, err := tar.FileInfoHeader(info, info.Name())
-	if err != nil {
-		return err
-	}
-
-	// use full path as name (FileInfoHeader only takes the basename) to
-	// preserve directory structure
-	// see for more info: https://golang.org/src/archive/tar/common.go?#L626
-	header.Name = filename
-
-	// Write file header to the tar archive
-	err = tw.WriteHeader(header)
-	if err != nil {
-		return err
-	}
-
-	// copy file content to tar archive
-	_, err = io.Copy(tw, file)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

tests/01-download-localhost.hurl

@@ -0,0 +1,11 @@
+# download single file
+GET http://localhost:5050/downloads
+
+# download directory as archive
+GET
+
+# download single file using plugins
+
+# download directory as archive using plugins
+
+

tests/02-upload-localhost.hurl

@@ -0,0 +1,8 @@
+# upload a single new file
+POST http://localhost:5050/upload
+
+# upload a new directory
+
+# upload a new plugin
+
+# upload a new profile

tests/03-list-localhost.hurl

@@ -0,0 +1,2 @@
+GET http://localhost:5050/list
+GET http://localhost:5050/status

tests/04-profiles-localhost.hurl

@@ -0,0 +1,6 @@
+GET http://localhost:5050/profiles
+GET http://localhost:5050/profiles/test
+POST http://localhost:5050/profiles/test
+GET http://localhost:5050/profiles/test/data
+POST http://localhost:5050/profiles/test/data
+DELETE http://localhost:5050/profiles/test/data

tests/05-plugins-localhost.hurl

@@ -0,0 +1,6 @@
+GET http://localhost:5050/plugins
+GET http://localhost:5050/plugin/test
+POST http://localhost:5050/plugins/test
+DELETE http://localhost:5050/plugin/test
+
+