diff --git a/.gitignore b/.gitignore index a51dd2f..cc2fb4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ **configurator +**makeshift **.yaml **.yml **.so @@ -6,3 +7,7 @@ **.ignore **.tar.gz dist/ +tests/data +tests/downloads +tests/profiles +tests/plugins diff --git a/.goreleaser.yaml b/.goreleaser.yaml index d91d3e3..badde3a 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -5,7 +5,7 @@ before: - go mod download - make plugins builds: - - id: "configurator" + - id: "makeshift" goos: - linux goarch: @@ -30,10 +30,10 @@ archives: dockers: - image_templates: - - ghcr.io/openchami/{{.ProjectName}}:latest - - ghcr.io/openchami/{{.ProjectName}}:{{ .Tag }} - - ghcr.io/openchami/{{.ProjectName}}:v{{ .Major }} - - ghcr.io/openchami/{{.ProjectName}}:v{{ .Major }}.{{ .Minor }} + - git.towk2.me/towk/{{.ProjectName}}:latest + - git.towk2.me/towk/{{.ProjectName}}:{{ .Tag }} + - git.towk2.me/towk/{{.ProjectName}}:v{{ .Major }} + - git.towk2.me/towk/{{.ProjectName}}:v{{ .Major }}.{{ .Minor }} build_flag_templates: - "--pull" - "--label=org.opencontainers.image.created={{.Date}}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 51332f7..cc94e2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -[0.2.0] - -- Major rewrite of entire code base [0.1.0] -- Initial prerelease of configurator \ No newline at end of file +- Initial prerelease of makeshift \ No newline at end of file diff --git a/LICENSE b/LICENSE index aa18635..85c3d9c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright © 2024 Triad National Security, LLC. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. +Copyright © 2025 David J. Allen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile index b6784ec..5e8d1f1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Unless set otherwise, the container runtime is Docker DOCKER ?= docker -prog ?= configurator +prog ?= makeshift git_tag := $(shell git describe --abbrev=0 --tags --always) sources := main.go $(wildcard cmd/*.go) plugin_source_prefix := pkg/generator/plugins @@ -36,18 +36,18 @@ container-testing: binaries plugins: $(plugin_binaries) # how to make each plugin -lib/%.so: pkg/generator/plugins/%/*.go +lib/%.so: pkg/plugins/%/*.go mkdir -p lib go build -buildmode=plugin -o $@ $< docs: - go doc github.com/OpenCHAMI/cmd - go doc github.com/OpenCHAMI/pkg/configurator + go doc git.towk2.me/towk/makeshift/cmd + go doc git.towk2.me/towk/makeshift/pkg/${prog} # remove executable and all built plugins .PHONY: clean clean: - rm -f configurator + rm -f ${prog} rm -f lib/* # run all of the unit tests diff --git a/README.md b/README.md index ca94345..380d435 100644 --- a/README.md +++ b/README.md @@ -1,219 +1,241 @@ -# OpenCHAMI Configurator +# << Makeshift >> -The `configurator` is an extensible tool that is capable of dynamically generating files on the fly. The tool includes a built-in generator that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate files based on Jinja 2 template files. The tool and generator plugins are written in Go and plugins can be written by following the ["Creating Generator Plugins"](#creating-generator-plugins) section of this README. +The `makeshift` tool is a service that serves files and CLI that downloads them with a couple of handy features baked-in. Although the CLI and server component function more like a glorified FTP, the power of this tool comes from the plugin system. For example, the file cobbler is built to run external plugins for more advanced processing files before serving them (e.g. fetching from a data source, rendering Jinja 2 templates, etc.). -## Building and Usage -The `configurator` is built using standard `go` build tools. The project separates the client, server, and generator components using build tags. To get started, clone the project, download the dependencies, and build the project: +## Building and Go! + + +The `makeshift` tool is built using standard `go` build tools. To get started, clone the project, download the dependencies, and build the project: ```bash -git clone https://github.com/OpenCHAMI/configurator.git +git clone https://git.towk2.me/towk/makeshift.git go mod tidy -go build --tags all # equivalent to `go build --tags client,server`` +go build ``` This will build the main driver program with the default generators that are found in the `pkg/generators` directory. -> [!WARNING] -> Not all of the plugins have completed generation implementations and are a WIP. +> [!NOTE] +> The project does not current separate the client, server, and plugin components using build tags, but will eventually. This will allow users to only compile and distribute specific parts of the tool with limited functionality. -### Running Configurator with CLI -After you build the program, run the following command to use the tool: +## Basic Examples + +Here are some of the common commands you may want to try right off the bat (aside from `makeshift help` of course). The commands below that do not include the `--host`, `--path`, or `--root` flags are set using the environment variables. ```bash -export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -./configurator generate --config config.yaml --target coredhcp -o coredhcp.conf --cacert ochami.pem +export MAKESHIFT_HOST=localhost +export MAKESHIFT_PATH=/test +export MAKESHIFT_ROOT=./test +export MAKESHIFT_LOG_FILE=logs/makeshift.log +export MAKESHIFT_LOG_LEVEL=debug ``` -This will generate a new `coredhcp` config file based on the Jinja 2 template specified in the config file for "coredhcp". The files will be written to `coredhcp.conf` as specified with the `-o/--output` flag. The `--target` flag specifies the type of config file to generate by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). +Start the server. The `--init` flag with create the default files and directory to get started at the `--root` path. -In other words, there should be an entry in the config file that looks like this: +```bash +makeshift serve --root $HOME/apps/makeshift/server --init +``` -```yaml -... -targets: - coredhcp: - plugin: "lib/coredhcp.so" # optional, if we want to use an external plugin instead - templates: - - templates/coredhcp.j2 -... +From here, you might want to see what files are available by default. +```bash +# list the files in the root directory +makeshift list + +# list files store in the template directory with a specified host +makeshift list --host http://localhost:5050 --path templates + +# list all available plugins +makeshift list plugins + +# list specific plugin (same as 'makeshift plugins info jinja2') +makeshift list plugins jinja2 + +# list all available profiles +makeshift list profiles + +# list specific profile information +makeshift list profiles default +``` + +Then, we can start downloading some files or directories (as archives). + +```bash +# download all data (notice --host and --port are not set here) +makeshift download + +# download the 'help.txt' file without processing (i.e. using plugins) +makeshift download --host http://localhost:5050 --path help.txt + +# download files with rendering using Jinja 2 plugin and default profile +makeshift download -p help.txt --plugins jinja2 --profile default + +# download directory with rendering using plugins to fetch data and render +# using a custom 'compute' profile +makeshift download -p templates --plugins smd,jinja2 --profile compute + +# do everything in the above example but extract and remove archive +makeshift download -p templates --plugins smd,jinja2 --profile compute -xr + +# download a raw plugin +makeshift download plugin jinja2 + +# download a profile +makeshift download profile default ``` > [!NOTE] -> The `configurator` tool requires a valid access token when making requests to an instance of SMD that has protected routes. +> Plugins are ran in order specified with the `--plugins` flag, which means if you're creating a plugin to write to a data store and then read in a subsequent plugin, the order specified with the CLI matters! -### Running Configurator as a Service - -The tool can also run as a service to generate files for clients: +(WIP) Files, directories, profiles, and plugins will eventually be able to be uploaded to the server. ```bash -export CONFIGURATOR_JWKS_URL="http://my.openchami.cluster:8443/key" -./configurator serve --config config.yaml -``` - -Once the server is up and listening for HTTP requests, you can try making a request to it with `curl` or `configurator fetch`. Both commands below are essentially equivalent: - -```bash -export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -curl http://127.0.0.1:3334/generate?target=coredhcp -X GET -H "Authorization: Bearer $ACCESS_TOKEN" --cacert ochami.pem -# ...or... -./configurator fetch --target coredhcp --host http://127.0.0.1:3334 --cacert ochami.pem -``` - -This will do the same thing as the `generate` subcommand, but through a GET request where the file contents is returned in the response. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set when starting the server with `serve`. The `ACCESS_TOKEN` environment variable is passed to `curl` using the `Authorization` header and expects a token as a JWT. - -### Docker - -New images can be built and tested using the `Dockerfile` provided in the project. However, the binary executable and the generator plugins must first be built before building the image since the Docker build copies the binary over. Therefore, build all of the binaries first by following the first section of ["Building and Usage"](#building-and-usage). Running `make docker` from the Makefile will automate this process. Otherwise, run the `docker build` command after building the executable and libraries. - -```bash -docker build -t configurator:testing path/to/configurator/Dockerfile -``` - -If you want to easily include your own external generator plugins, you can build it and copy the `lib.so` file to `lib/`. Make sure that the `Generator` interface is implemented correctly as described in the ["Creating Generator Plugins"](#creating-generator-plugins) or the plugin will not load (you should get an error that specifically says this). Additionally, the name string returned from the `GetName()` method is used for looking up the plugin with the `--target` flag by the main driver program. - -Alternatively, pull the latest existing image/container from the GitHub container repository. - -```bash -docker pull ghcr.io/openchami/configurator:latest -``` - -Then, run the Docker container similarly to running the binary. - -```bash -export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -docker run ghcr.io/openchami/configurator:latest configurator generate --config config.yaml --target coredhcp -o coredhcp.conf --cacert configurator.pem -``` - -### Creating Generator Plugins - -The `configurator` uses built-in and user-defined generators that implement the `Generator` interface to describe how config files should be generated. The interface is defined like so: - -```go -// maps the file path to its contents -type FileMap = map[string][]byte - -// interface for generator plugins -type Generator interface { - GetName() string - GetVersion() string - GetDescription() string - Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) -} -``` - -A new plugin can be created by implementing the methods from interface and exporting a symbol with `Generator` as the name and the plugin struct as the type. The `GetName()` function returns the name that is used for looking up the corresponding target set in your config file. It can also be included in the templated files with the default plugins using the `{{ plugin_name }}` in your template. The `GetVersion()` and `GetDescription()` functions returns the version and description of the plugin which can be included in the templated files using `{{ plugin_version }}` and `{{ plugin_description }}` respectively with the default plugins. The `Generate` function is where the magic happens to build the config file from a template. - -```go -package main - -type MyGenerator struct { - PluginInfo map[string]any -} - -var pluginInfo map[string]any - -// this function is not a part of the `Generator` interface -func (g *MyGenerator) LoadFromFile() map[string]any{ /*...*/ } - -func (g *MyGenerator) GetName() string { - // just an example...this can be done however you want - g.PluginInfo := LoadFromFile("path/to/plugin/info.json") - return g.PluginInfo["name"] -} - -func (g *MyGenerator) GetVersion() string { - return g.PluginInfo["version"] // "v1.0.0" -} - -func (g *MyGenerator) GetDescription() string { - return g.PluginInfo["description"] // "This is an example plugin." -} - -func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { - // do config generation stuff here... - var ( - params = generator.GetParams(opts...) - client = generator.GetClient(params) - output = "" - ) - if client { - eths, err := client.FetchEthernetInterfaces(opts...) - // ... blah, blah, blah, check error, format output, and so on... +# upload a single file in root directory + makeshift upload -d @compute-base.yaml + # upload a directory (not working yet...) + makeshift upload -d @setup/ - // apply the substitutions to Jinja template and return output as FileMap (i.e. path and it's contents) - return generator.ApplyTemplate(path, generator.Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - "dhcp_hosts": output, - }) - } -} + # upload an archive (extracted and saved on server - not working yet...) + makeshift upload -d @setup.tar.gz -t archive -// this MUST be named "Generator" for symbol lookup in main driver -var Generator MyGenerator + # upload a new profile + makeshift upload profile -d @compute.json kubernetes.json + + # upload a new profile with a specific path + makeshift upload profile -d @kubernetes.json + makeshift upload profile -d '{"id": "custom", "data": {}}' kubernetes.json + + # upload a new plugin + makeshift upload plugin -d @slurm.so + makeshift upload plugin slurm.so ``` > [!NOTE] -> The keys in `generator.ApplyTemplate` must not contain illegal characters such as a `-` or else the templates will not apply correctly. +> Although every command has a `curl` equivalent, it is better to use the CLI since it has other features such as extracting and remove archives after downloading and saving archives as files automatically. -Finally, build the plugin and put it somewhere specified by `plugins` in your config. Make sure that the package is `main` before building. +## Server Root Structure + +The `makeshift` server serves files at the specified `--root` path (also set with `MAKESHIFT_ROOT` environment variable). The directory structure looks like the following by default with initializing with `makeshift init $MAKESHIFT_ROOT`. ```bash -go build -buildmode=plugin -o lib/mygenerator.so path/to/mygenerator.go +├── data +├── plugins +└── profiles ``` -Now your plugin should be available to use with the `configurator` main driver program. If you get an error about not loading the correct symbol type, make sure that your generator function definitions match the `Generator` interface entirely and that you don't have a partially implemented interface. +Each directory holds specific files for different purposes: + +- `data` - Stores any and all miscellaenous files and directories. +- `plugins` - Stores plugins defined in the ["Creating Plugins"](#creating-plugins) section. +- `profiles` - Stores profiles in JSON format as defined in the ["Creating Profiles"](#creating-profiles) section. + + +## Creating Plugins + +The `makeshift` tool defines a plugin as an interface that can be implemented and compiled. + +```go +type Plugin interface { + Name() string + Version() string + Description() string + Metadata() Metadata + + Init() error + Run(data storage.KVStore, args []string) error + Cleanup() error +} +``` + +Plugins can *literally* contain whatever you want and is written in Go. Here is a simple example implementation to demonstrate how that is done which we will save at `src/example.go`. + +```go +type Example struct{} + +func (p *Example) Name() string { return "example" } +func (p *Example) Version() string { return "v0.0.1-alpha" } +func (p *Example) Description() string { return "An example plugin" } +func (p *Example) Metadata() map[string]string { + return makeshift.Metadata{ + "author": map[string]any{ + "name": "John Smith", + "email": "john.smith@example", + "links": []string{ + "https://example.com", + }, + }, + } +} + +func (p *Example) Init() error { + // Initialize the plugin if necessary. + return nil +} + +func (p *Example) Run(data storage.KVStore, args []string) error { + // Plugins can read and write to a data stores passed in. + // See the 'jinja2' plugin for reading and 'smd' plugin for writing. + return nil +} + +func (p *Example) Clean() error { + // Clean up resources if necessary. + return nil +} + +// This MUST be included to find the symbol in the main driver executable. +var Makeshift Example +``` + +Then, we can use the built-in `makeshift plugins compile` command to compile it. + +```bash +makeshift plugins compile src/example.go -o $MAKESHIFT_ROOT/plugins/example.so +``` > [!TIP] -> See the `examples/test.go` file for a plugin and template example. +> Make sure you move all of your plugins to `$MAKESHIFT_ROOT/plugins` to use them and should have an `*.so` name for lookup. For example, to use a custom plugin with `makeshift download -p templates/hosts.j2 --plugins my-plugin`, there has to a plugin `$MAKESHIFT_ROOT/plugins/my-plugin.so`. -## Configuration +## Creating Profiles -Here is an example config file to start using configurator: +On the other hand, profiles are simply objects that contain data used to populate data stores. The `makeshift` tool does not currently use all fields of a profile which will likely be removed in the near future. -```yaml -server: # Server-related parameters when using as service - host: 127.0.0.1 - port: 3334 - jwks: # Set the JWKS uri for protected routes - uri: "" - retries: 5 -smd: # SMD-related parameters - host: http://127.0.0.1:27779 -plugins: # path to plugin directories - - "lib/" -targets: # targets to call with --target flag - coredhcp: - templates: - - templates/coredhcp.j2 - files: # files to be copied without templating - - extra/nodes.conf - targets: # additional targets to run (does not run recursively) - - dnsmasq +```go +type Profile struct { + ID string `json:"id"` // profile ID + Description string `json:"description,omitempty"` // profile description + Tags []string `json:"tags,omitempty"` // tags used for filtering (not implemented yet) + Data map[string]any `json:"data,omitempty"` // include render data +} ``` -The `server` section sets the properties for running the `configurator` tool as a service and is not required if you're only using the CLI. Also note that the `jwks.uri` parameter is only needed for protecting endpoints. If it is not set, then all API routes are entirely public. The `smd` section tells the `configurator` tool where to find the SMD service to pull state management data used internally by the client's generator. The `templates` section is where the paths are mapped to each generator by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `plugins` is a list of paths to search for and load external generator plugins. +Profiles can be created using JSON and only require an `id` with optional `data`. See the example in `$MAKESHIFT_ROOT/profiles/default.json`. -## Running the Tests - -The `configurator` project includes a collection of tests focused on verifying plugin behavior and generating files. The tests do not include fetching information from any remote sources, can be ran with the following command: - -```bash -go test ./tests/generate_test.go --tags=all +```json +{ + "id": "default", + "description": "Makeshift default profile", + "data": { + "host": "localhost", + "path": "/test", + "server_root": "./test" + } +} ``` -## Known Issues +> [!TIP] +> Make sure that you store your custom profiles in `$MAKESHIFT_ROOT/profiles` and that you set the name you want to use for lookup with a `*.json` extension (e.g. `compute.json`). -- Adds a new `OAuthClient` with every token request -- Plugins are being loaded each time a file is generated +## TODO: Missing Features -## TODO +There are some features still missing that will be added later. -- Add group functionality to create by files by groups -- Extend SMD client functionality (or make extensible?) -- Handle authentication with `OAuthClient`'s correctly +1. Running `makeshift` locally with profiles and plugins +2. Plugin to add user data for one-time use without creating a profile +3. Optionally build plugins directly into the main driver +4. Protected routes that require authentication +5. Configuration file for persistent runs +6. `Dockerfile` and `docker-compose.yml` files to build containers \ No newline at end of file diff --git a/bin/compile-plugins.sh b/bin/compile-plugins.sh new file mode 100755 index 0000000..278bc7c --- /dev/null +++ b/bin/compile-plugins.sh @@ -0,0 +1,26 @@ +#!/usr/bin bash + + +function compile_default_plugins() { + makeshift_exe=./makeshift + go_exe=go + + # make sure go build tools are installed + if command -v $go_exe >/dev/null 2>&1; then + # make sure that MAKESHIFT_ROOT is set + if [[ ! -v MAKESHIFT_ROOT ]]; then + # Compile the default external plugins + go build + $makeshift_exe compile pkg/plugins/jinja2/jinja2.go -o $MAKESHIFT_ROOT/plugins/jinja2.go + $makeshift_exe compile pkg/plugins/smd/smd.go -o $MAKESHIFT_ROOT/plugins/smd.so + $makeshift_exe compile pkg/plugins/userdata/userdata.go -o $MAKESHIFT_ROOT/plugins/userdata.go + else + echo "requires MAKESHIFT_ROOT to be set" + fi + else + echo "Go build tools must be installed" + fi +} + + +compile_default_plugins \ No newline at end of file diff --git a/res/archlinux/PKGBUILD b/build/archlinux/PKGBUILD similarity index 58% rename from res/archlinux/PKGBUILD rename to build/archlinux/PKGBUILD index 5796121..3d9d030 100644 --- a/res/archlinux/PKGBUILD +++ b/build/archlinux/PKGBUILD @@ -1,15 +1,15 @@ -# Maintainer: David J. Allen -pkgname=configurator +# Maintainer: David J. Allen +pkgname=makeshift pkgver=v0.1.0alpha pkgrel=1 -pkgdesc="An extensible tool to dynamically generate config files from SMD with Jinja 2 templating support." +pkgdesc="Extensible file cobbler" arch=("x86_64") -url="https://github.com/OpenCHAMI/configurator" +url="https://git.towk2.me/towk/makeshift" license=('MIT') -groups=("openchami") -provides=('configurator') -conflicts=('configurator') -https://git.towk2.me/towk/configurator/releases/download/v0.1.0-alpha/configurator +# groups=("towk") +provides=('makeshift') +conflicts=('makeshift') +# https://git.towk2.me/towk/makeshift/releases/download/v0.1.0-alpha/makeshift source_x86_64=( "${url}/releases/download/v0.1.0-alpha/${pkgname}.tar.gz" ) @@ -27,7 +27,7 @@ package() { # install the binary to /usr/bin mkdir -p "${pkgdir}/usr/bin" mkdir -p "${pkgdir}/usr/lib/${pkgname}" - install -m755 configurator "${pkgdir}/usr/bin/configurator" + install -m755 makeshift "${pkgdir}/usr/bin/makeshift" # install plugins to /usr/lib install -m755 *.so "${pkgdir}/usr/lib/${pkgname}" diff --git a/cmd/delete.go b/cmd/delete.go new file mode 100644 index 0000000..4bdb767 --- /dev/null +++ b/cmd/delete.go @@ -0,0 +1,155 @@ +package cmd + +import ( + "fmt" + "net/http" + + "git.towk2.me/towk/makeshift/pkg/client" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var deleteCmd = &cobra.Command{ + Use: "delete", + Example: ` + # set up environment + export MAKESHIFT_HOST=http://localhost:5050 + export MAKESHIFT_PATH=test + + # delete a file or directory (cannot delete root) + makeshift delete -p help.txt + makeshift delete --host http://localhost:5555 --path templates +`, + Short: "Delete files and directories", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") + }, + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + paths, _ = cmd.Flags().GetStringSlice("path") + cacertPath, _ = cmd.Flags().GetString("cacert") + + c = client.New(host) + res *http.Response + query string + err error + ) + + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + for _, path := range paths { + if path == "" { + log.Warn().Msg("skipping empty path") + continue + } + + query = fmt.Sprintf("/delete/%s?", path) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodDelete, + }) + handleResponseError(res, host, query, err) + } + }, +} + +var deleteProfilesCmd = &cobra.Command{ + Use: "profiles", + Example: ` + # delete profile(s) by its ID + makeshift delete profiles kubernetes slurm compute +`, + Args: cobra.MinimumNArgs(1), + Short: "Delete profile(s)", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + cacertPath, _ = cmd.Flags().GetString("cacert") + + c = client.New(host) + res *http.Response + query string + err error + ) + + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + for _, profileID := range args { + if profileID == "default" { + log.Warn().Msg("cannot delete the default profile") + continue + } + + query = fmt.Sprintf("/profiles/%s", profileID) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodDelete, + }) + handleResponseError(res, host, query, err) + } + }, +} + +var deletePluginsCmd = &cobra.Command{ + Use: "plugins", + Example: ` + # delete plugin(s) by name + makeshift delete plugins weather slurm user +`, + Args: cobra.MinimumNArgs(1), + Short: "Delete plugin(s)", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + cacertPath, _ = cmd.Flags().GetString("cacert") + + c = client.New(host) + res *http.Response + query string + err error + ) + + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + for _, pluginName := range args { + query = fmt.Sprintf("/plugins/%s", pluginName) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodDelete, + }) + handleResponseError(res, host, query, err) + } + }, +} + +func init() { + deleteCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift server host (can be set with MAKESHIFT_HOST)") + deleteCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load") + deleteCmd.Flags().StringSliceP("path", "p", []string{}, "Set the paths to delete files and directories") + deleteCmd.AddCommand(deleteProfilesCmd, deletePluginsCmd) + + rootCmd.AddCommand(deleteCmd) +} diff --git a/cmd/download.go b/cmd/download.go index 0160d4f..03bf061 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -1,7 +1,305 @@ package cmd -import "github.com/spf13/cobra" +import ( + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "git.towk2.me/towk/makeshift/internal/archive" + "git.towk2.me/towk/makeshift/pkg/client" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) var downloadCmd = cobra.Command{ Use: "download", + Example: ` + # set up environment + export MAKESHIFT_HOST=http://localhost:5050 + export MAKESHIFT_PATH=test + + # download a file or directory (as archive) + makeshift download + makeshift download --host http://localhost:5050.com --path test + + # download a file or directory and run plugins with profile data + makeshift download --plugins smd,jinja2 --profile compute + curl $MAKESHIFT_HOST/download/test?plugins=smd,jinja2&profile=test + + # download directory and extract it's contents automatically + # then, remove the downloaded archive + makeshift download -xr +`, + Short: "Download and modify files with plugins", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") + }, + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + outputPath, _ = cmd.Flags().GetString("output") + cacertPath, _ = cmd.Flags().GetString("cacert") + pluginNames, _ = cmd.Flags().GetStringSlice("plugins") + profileIDs, _ = cmd.Flags().GetStringSlice("profiles") + extract, _ = cmd.Flags().GetBool("extract") + removeArchive, _ = cmd.Flags().GetBool("remove-archive") + + c = client.New(host) + res *http.Response + query string + body []byte + err error + ) + + query = fmt.Sprintf("/download/%s?", path) + if len(pluginNames) > 0 { + query += "plugins=" + url.QueryEscape(strings.Join(pluginNames, ",")) + } + if len(profileIDs) > 0 { + query += "&profiles=" + url.QueryEscape(strings.Join(profileIDs, ",")) + } + + log.Debug(). + Str("host", host). + Str("path", path). + Str("query", query). + Str("output", outputPath). + Strs("profiles", profileIDs). + Strs("plugins", pluginNames). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("path", path). + Str("output", outputPath). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Str("path", path). + Str("output", outputPath). + Msg("response returned bad status") + os.Exit(1) + } + + // determine if output path is an archive or file + switch res.Header.Get("FILETYPE") { + case "archive": + // write archive to disk with or without '-o' specified + if outputPath == "" { + outputPath = fmt.Sprintf("%s.tar.gz", path) + writeFiles(outputPath, body) + log.Debug().Str("path", outputPath).Msg("wrote archive to pre-determined path") + } else { + writeFiles(outputPath, body) + log.Debug().Str("path", outputPath).Msg("wrote archive to specified path") + } + + // extract files if '-x' flag is passed + if extract { + var ( + dir = filepath.Dir(outputPath) + base = strings.TrimSuffix(filepath.Base(outputPath), ".tar.gz") + ) + err = archive.Expand(outputPath, fmt.Sprintf("%s/%s", dir, base)) + if err != nil { + log.Error().Err(err). + Str("path", outputPath). + Msg("failed to expand archive") + os.Exit(1) + } + } + + // optionally, remove archive if '-r' flag is passed + // NOTE: this can only be used if `-x` flag is set + if removeArchive { + if !extract { + log.Warn().Msg("requires '-x/--extract' flag to be set to 'true'") + } else { + err = os.Remove(outputPath) + if err != nil { + log.Error().Err(err). + Str("path", outputPath). + Msg("failed to remove archive") + } + } + } + case "file": + // write to file if '-o' specified otherwise stdout + if outputPath != "" { + writeFiles(outputPath, body) + log.Debug().Str("path", outputPath).Msg("wrote file to specified path") + } else { + fmt.Println(string(body)) + } + } + }, +} + +var downloadProfileCmd = &cobra.Command{ + Use: "profile", + Example: ` + // download a profile + makeshift download profile default +`, + Args: cobra.ExactArgs(1), + Short: "Download a profile", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + outputPath, _ = cmd.Flags().GetString("output") + cacertPath, _ = cmd.Flags().GetString("cacert") + + c = client.New(host) + res *http.Response + body []byte + query string + err error + ) + + log.Debug(). + Str("host", host). + Str("output", outputPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + for _, profileID := range args { + query = fmt.Sprintf("/profiles/%s", profileID) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) + } + if outputPath != "" { + writeFiles(outputPath, body) + } else { + fmt.Println(string(body)) + } + } + }, +} + +var downloadPluginCmd = &cobra.Command{ + Use: "plugin", + Example: ` + // download a plugin + makeshift download plugin smd jinja2 +`, + Args: cobra.ExactArgs(1), + Short: "Download a raw plugin", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + outputPath, _ = cmd.Flags().GetString("output") + cacertPath, _ = cmd.Flags().GetString("cacert") + + c = client.New(host) + res *http.Response + query string + body []byte + err error + ) + + log.Debug(). + Str("host", host). + Str("output", outputPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + for _, pluginName := range args { + query = fmt.Sprintf("/plugins/%s/raw", pluginName) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("query", query). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) + } + if outputPath != "" { + writeFiles(outputPath, body) + } else { + writeFiles(fmt.Sprintf("%s.so", pluginName), body) + } + } + }, +} + +func init() { + downloadCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + downloadCmd.PersistentFlags().StringP("output", "o", "", "Set the output path to write files") + downloadCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load") + downloadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") + downloadCmd.Flags().StringSlice("profiles", []string{}, "Set the profile(s) to use to populate data store") + downloadCmd.Flags().StringSlice("plugins", []string{}, "Set the plugin(s) to run before downloading files") + downloadCmd.Flags().BoolP("extract", "x", false, "Set whether to extract archive locally after downloading") + downloadCmd.Flags().BoolP("remove-archive", "r", false, "Set whether to remove the archive after extracting (used with '--extract' flag)") + + downloadCmd.AddCommand(downloadProfileCmd, downloadPluginCmd) + + rootCmd.AddCommand(&downloadCmd) +} + +// helper to write downloaded files +func writeFiles(path string, body []byte) { + var err = os.WriteFile(path, body, 0o755) + if err != nil { + log.Error().Err(err).Msg("failed to write file(s) from download") + os.Exit(1) + } } diff --git a/cmd/init.go b/cmd/init.go new file mode 100644 index 0000000..1bac85d --- /dev/null +++ b/cmd/init.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "git.towk2.me/towk/makeshift/pkg/service" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var initCmd = &cobra.Command{ + Use: "init", + Example: ` + # create default files and directories at specified root path + # (must be set with positional argument) + makeshift init $MAKESHIFT_ROOT +`, + Args: cobra.ExactArgs(1), + Short: "Initialize directory with default files", + Run: func(cmd *cobra.Command, args []string) { + var ( + server *service.Service + err error + ) + + // create the server root files and directories + server = service.New() + server.RootPath = args[0] + err = server.Init() + if err != nil { + log.Error().Err(err). + Str("root", server.RootPath). + Msg("failed to initialize server root") + return + } + log.Debug(). + Str("root", server.RootPath). + Msg("initialize makeshift files at root path") + }, +} + +func init() { + rootCmd.AddCommand(initCmd) +} diff --git a/cmd/list.go b/cmd/list.go index e69de29..4c5898c 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -0,0 +1,216 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/client" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var listCmd = &cobra.Command{ + Use: "list", + Example: ` + # list files in a remote data directory + configurator list --path test + configurator list --host http://localhost:5050 --path test + + # list files using 'curl' + curl http://localhost:5050/list/test +`, + Args: cobra.NoArgs, + Short: "List all files in a remote data directory", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") + }, + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + cacertPath, _ = cmd.Flags().GetString("cacert") + + c = client.New(host) + body []byte + output []string + err error + ) + + log.Debug(). + Str("host", host). + Str("path", path). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + // make request to /list endpoint + _, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: fmt.Sprintf("/list/%s", path), + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("path", path). + Msg("failed to make request") + os.Exit(1) + } + + err = json.Unmarshal(body, &output) + if err != nil { + log.Error().Err(err).Msg("failed to unmarshal response body") + os.Exit(1) + } + + // show the list of files and directories + log.Info().Strs("output", output).Send() + }, +} + +var listPluginsCmd = &cobra.Command{ + Use: "plugins", + Example: ` + # show all plugins + makeshift list plugins + + # show details for specific plugins + makeshift list plugins smd jinja2 +`, + Short: "Show plugins information", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + cacertPath, _ = cmd.Flags().GetString("cacert") + + c = client.New(host) + res *http.Response + query string + plugins []string + body []byte + err error + ) + + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + if len(args) == 0 { + // make request to /list endpoint + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: "/plugins", + Method: http.MethodGet, + }) + handleResponseError(res, host, "/plugins", err) + err = json.Unmarshal(body, &plugins) + if err != nil { + log.Error().Err(err). + Msg("failed to unmarshal plugins") + return + } + } else { + for _, pluginName := range args { + // make request to /list endpoint + query = fmt.Sprintf("/plugins/%s/info", pluginName) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + handleResponseError(res, host, query, err) + + plugins = append(plugins, string(body)) + } + } + log.Info().Strs("plugins", plugins).Send() + }, +} + +var listProfilesCmd = &cobra.Command{ + Use: "profiles", + Example: ` + # list all profiles + makeshift list profiles + + # live individual profiles + makeshift list profiles default custom +`, + Short: "Show all available profiles", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + cacertPath, _ = cmd.Flags().GetString("cacert") + + c = client.New(host) + res *http.Response + profiles []makeshift.Profile + body []byte + query string + err error + ) + + log.Debug(). + Str("host", host). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + if len(args) == 0 { + // make request to /list endpoint + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: "/profiles", + Method: http.MethodGet, + }) + handleResponseError(res, host, "/profiles", err) + + err = json.Unmarshal(body, &profiles) + if err != nil { + log.Error().Err(err). + Msg("failed to unmarshal plugins") + return + } + } else { + for _, profileID := range args { + // make request to /list endpoint + query = fmt.Sprintf("/profiles/%s", profileID) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + handleResponseError(res, host, query, err) + var profile makeshift.Profile + err = json.Unmarshal(body, &profile) + if err != nil { + log.Error().Err(err). + Msg("failed to unmarshal plugin") + continue + } + profiles = append(profiles, profile) + } + } + log.Info().Any("plugins", profiles).Send() + }, +} + +func init() { + listCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the configurator remote host (can be set with MAKESHIFT_HOST)") + listCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load") + listCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") + + listCmd.AddCommand(listPluginsCmd, listProfilesCmd) + rootCmd.AddCommand(listCmd) +} diff --git a/cmd/plugins.go b/cmd/plugins.go new file mode 100644 index 0000000..99b8908 --- /dev/null +++ b/cmd/plugins.go @@ -0,0 +1,246 @@ +package cmd + +import ( + "fmt" + "io/fs" + "net/http" + "os" + "os/exec" + "path/filepath" + + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/client" + "git.towk2.me/towk/makeshift/pkg/service" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var pluginsCmd = &cobra.Command{ + Use: "plugins", + Short: "Manage, inspect, and compile plugins (requires Go build tools)", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + }, +} + +var pluginsCompileCmd = &cobra.Command{ + Use: "compile", + Example: ` + # compile plugin using Go build tools + go build -buildmode=plugin -o lib/myplugin.so src/plugins/myplugin.go + + # try to compile all plugins in current directory + cd src/plugins + makeshift plugin compile + + # try to compile all plugins in specified directory + makeshift plugin compile src/plugins + + # compile 'src/plugins/myplugin.go' and save to 'lib/myplugin.so' + makeshift plugin compile src/plugins/myplugin.go -o lib/myplugin.so +`, + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + var ( + outputPath, _ = cmd.Flags().GetString("output") + output []byte + fileInfo os.FileInfo + err error + ) + + // make the directory + err = os.MkdirAll(filepath.Dir(outputPath), 0o777) + if err != nil { + log.Fatal().Err(err).Msg("failed to make output directory") + } + + // one arg passed, so determine if it is file or directory + if len(args) > 0 { + if fileInfo, err = os.Stat(args[0]); err == nil { + if fileInfo.IsDir() { + err = compilePluginsDir(args[0], outputPath) + if err != nil { + log.Fatal().Err(err). + Bytes("output", output). + Msg("failed to compile plugin") + } + } else { + // not a directory so check if Go file so try and compile it + if filepath.Ext(args[0]) == ".go" { + output, err = compilePlugin(outputPath, args[0]) + if err != nil { + log.Fatal().Err(err). + Bytes("output", output). + Msg("failed to compile plugin") + } + } else { + log.Fatal().Msg("argument is not a valid plugin (must be Go file)") + } + } + } else if err != nil { + log.Fatal().Err(err).Msgf("failed to stat provided plugin path") + } + } else { + // no args passed, so use current directory + err = compilePluginsDir(".", outputPath) + if err != nil { + log.Fatal().Err(err). + Bytes("output", output). + Msg("failed to compile plugin") + } + } + }, +} + +var pluginsInspectCmd = &cobra.Command{ + Use: "inspect", + Args: cobra.MinimumNArgs(1), + Example: ` + # inspect a plugin and print its information + makeshift plugin inspect lib/jinja2.so +`, + Run: func(cmd *cobra.Command, args []string) { + for _, path := range args { + var ( + plugin makeshift.Plugin + err error + ) + plugin, err = service.LoadPluginFromFile(path) + if err != nil { + log.Error().Err(err). + Str("path", path). + Msg("failed to load plugin from file") + continue + } + + log.Info().Any("plugin", map[string]any{ + "name": plugin.Name(), + "version": plugin.Version(), + "description": plugin.Description(), + "metadata": plugin.Metadata(), + }).Send() + } + }, +} + +var pluginsInfoCmd = &cobra.Command{ + Use: "info", + Example: ` + # show information of a remote plugin + makeshift plugins info jinja2 smd + + # show information of a local plugin (same as 'makeshift inspect') + makeshift plugins info --local $MAKESHIFT_ROOT/plugins/jinja2.so +`, + Short: "Show plugin information", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + outputPath, _ = cmd.Flags().GetString("output") + local, _ = cmd.Flags().GetBool("local") + + c = client.New(host) + res *http.Response + query string + body []byte + err error + ) + + log.Debug(). + Str("host", host). + Str("output", outputPath). + Send() + + if local { + var ( + plugins []map[string]any + plugin makeshift.Plugin + err error + ) + for _, path := range args { + plugin, err = service.LoadPluginFromFile(path) + if err != nil { + log.Error().Err(err). + Str("path", path). + Msg("failed to load plugin from path") + continue + } + plugins = append(plugins, makeshift.PluginToMap(plugin)) + } + log.Info().Any("plugins", plugins).Send() + } else { + for _, pluginName := range args { + query = fmt.Sprintf("/plugins/%s/info", pluginName) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodGet, + }) + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("query", query). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + "body": string(body), + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) + } + if outputPath != "" { + writeFiles(outputPath, body) + } else { + fmt.Println(string(body)) + } + } + } + }, +} + +func init() { + pluginsCompileCmd.Flags().StringP("output", "o", "", "Set the path to save compiled plugin (matches source type, i.e. uses files or directory)") + pluginsInfoCmd.Flags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + pluginsInfoCmd.Flags().Bool("local", false, "Set whether to display information of a local plugin") + + pluginsCmd.AddCommand(pluginsCompileCmd, pluginsInspectCmd, pluginsInfoCmd) + rootCmd.AddCommand(pluginsCmd) +} + +func compilePlugin(outputPath string, srcPath string) ([]byte, error) { + var ( + commandArgs string + command *exec.Cmd + ) + // execute command to build the plugin + commandArgs = fmt.Sprintf("go build -buildmode=plugin -o=%s %s", outputPath, srcPath) + command = exec.Command("bash", "-c", commandArgs) + return command.CombinedOutput() +} + +func compilePluginsDir(dirpath string, outputPath string) error { + err := filepath.WalkDir(dirpath, func(path string, d fs.DirEntry, err error) error { + // not a directory and is Go file, so try and compile it + if !d.IsDir() && filepath.Ext(path) == ".go" { + var ( + localOutputPath string = outputPath + "/" + path + ) + output, err := compilePlugin(localOutputPath, path) + if err != nil { + log.Fatal().Err(err). + Bytes("output", output). + Str("path", localOutputPath). + Msg("failed to compile plugin") + return err + } + } + return nil + }) + return err +} diff --git a/cmd/render.go b/cmd/render.go deleted file mode 100644 index e69de29..0000000 diff --git a/cmd/root.go b/cmd/root.go index 2e70c6a..fb354f0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,22 +2,58 @@ package cmd import ( "fmt" + "net/http" "os" + logger "git.towk2.me/towk/makeshift/pkg/log" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) -var rootCmd = cobra.Command{ - Use: "configurator", - Run: func(cmd *cobra.Command, args []string) { +var ( + loglevel logger.LogLevel = logger.INFO +) +var rootCmd = cobra.Command{ + Use: "makeshift", + Short: "Extensible file cobbler", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + var ( + logFile string + err error + ) + + // initialize the logger + logFile, _ = cmd.Flags().GetString("log-file") + err = logger.InitWithLogLevel(loglevel, logFile) + if err != nil { + log.Error().Err(err).Msg("failed to initialize logger") + os.Exit(1) + } + }, + Run: func(cmd *cobra.Command, args []string) { + // try and set flags using env vars + setenv(cmd, "log-file", "MAKESHIFT_LOG_FILE") + setenv(cmd, "log-level", "MAKESHIFT_LOG_LEVEL") + if len(args) == 0 { + err := cmd.Help() + if err != nil { + log.Error().Err(err).Msg("failed to print help") + } + os.Exit(0) + } + }, + PostRun: func(cmd *cobra.Command, args []string) { + log.Debug().Msg("closing log file") + err := logger.LogFile.Close() + if err != nil { + log.Error().Err(err).Msg("failed to close log file") + } }, } func Execute() { - // run initialization code first - initEnv() - + // run the main program if err := rootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -25,13 +61,60 @@ func Execute() { } func init() { + cobra.OnInitialize( + initLogger, + ) // initialize the config a single time + rootCmd.PersistentFlags().VarP(&loglevel, "log-level", "l", "Set the log level output") + rootCmd.PersistentFlags().String("log-file", "", "Set the log file path (can be set with MAKESHIFT_LOG_FILE)") } -func initConfigFromFile(path string) { - +func setenv(cmd *cobra.Command, varname string, envvar string) { + if cmd.Flags().Changed(varname) { + return + } + val := os.Getenv(envvar) + if val != "" { + cmd.Flags().Set(varname, val) + } } -func initEnv() { - +func setenvp(cmd *cobra.Command, varname string, envvar string) { + if cmd.Flags().Changed(varname) { + return + } + val := os.Getenv(envvar) + if val != "" { + cmd.PersistentFlags().Set(varname, val) + } +} + +func initLogger() { + // initialize the logger + logfile, _ := rootCmd.PersistentFlags().GetString("log-file") + err := logger.InitWithLogLevel(loglevel, logfile) + if err != nil { + log.Error().Err(err).Msg("failed to initialize logger") + os.Exit(1) + } +} + +func handleResponseError(res *http.Response, host, query string, err error) { + if err != nil { + log.Error().Err(err). + Str("host", host). + Str("query", query). + Msg("failed to make request") + os.Exit(1) + } + if res.StatusCode != http.StatusOK { + log.Error(). + Any("status", map[string]any{ + "code": res.StatusCode, + "message": res.Status, + }). + Str("host", host). + Msg("response returned bad status") + os.Exit(1) + } } diff --git a/cmd/run.go b/cmd/run.go new file mode 100644 index 0000000..ab698e7 --- /dev/null +++ b/cmd/run.go @@ -0,0 +1,28 @@ +package cmd + +import "github.com/spf13/cobra" + +var runCmd = &cobra.Command{ + Use: "run", + Example: ` + NOTE: This command is not implemented yet! + + # set up environment + export MAKESHIFT_HOST=http://localhost:5050 + export MAKESHIFT_PATH=help.txt + export MAKESHIFT_ROOT=/opt/makeshift + + # run locally similar to 'download' + makeshift run --plugins jinja2 --profiles default + makeshift run --root $HOME/apps/makeshift -p help.txt --plugins jinja2 --profiles default +`, + Args: cobra.NoArgs, + Short: "Run locally with plugins and profiles", + Run: func(cmd *cobra.Command, args []string) { + + }, +} + +func init() { + rootCmd.AddCommand(runCmd) +} diff --git a/cmd/serve.go b/cmd/serve.go new file mode 100644 index 0000000..9c8c046 --- /dev/null +++ b/cmd/serve.go @@ -0,0 +1,101 @@ +package cmd + +import ( + "net/url" + "time" + + "git.towk2.me/towk/makeshift/pkg/service" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var serveCmd = &cobra.Command{ + Use: "serve", + Example: ` + # start the service in current directory + makeshift serve + + # start the service with root path and initialize + makeshift serve --root ./test --init -l debug +`, + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "root", "MAKESHIFT_ROOT") + setenv(cmd, "timeout", "MAKESHIFT_TIMEOUT") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") + setenv(cmd, "keyfile", "MAKESHIFT_KEYFILE") + }, + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + rootPath, _ = cmd.Flags().GetString("root") + cacertPath, _ = cmd.Flags().GetString("cacert") + keyfile, _ = cmd.Flags().GetString("keyfile") + timeout, _ = cmd.Flags().GetInt("timeout") + + parsed *url.URL + server *service.Service + err error + ) + + // parse the host to remove scheme if needed + parsed, err = url.Parse(host) + if err != nil { + log.Warn().Err(err). + Str("host", host). + Msg("could not parse host") + } + + // set the server values + server = service.New() + server.Addr = parsed.Host + server.RootPath = rootPath + server.CACertFile = cacertPath + server.CACertKeyfile = keyfile + server.Timeout = time.Duration(timeout) * time.Second + + // show some debugging information + log.Debug(). + Str("host", parsed.Host). + Any("paths", map[string]string{ + "root": rootPath, + "cacert": cacertPath, + "keyfile": keyfile, + "data": server.PathForData(), + "profiles": server.PathForProfiles(), + "plugins": server.PathForPlugins(), + "metadata": server.PathForMetadata(), + }). + Send() + + // make the default directories and files if flag is passed + if cmd.Flags().Changed("init") { + err = server.Init() + if err != nil { + log.Error().Err(err). + Str("host", parsed.Host). + Str("root", rootPath). + Msg("failed to initialize server root") + return + } + } + + // serve and log why the server closed + err = server.Serve() + log.Error().Err(err).Msg("server closed") + }, +} + +func init() { + serveCmd.Flags().Bool("init", false, "Initializes default files at specified with the '--root' flag") + serveCmd.Flags().String("host", "localhost:5050", "Set the configurator server host (can be set with MAKESHIFT_HOST)") + serveCmd.Flags().String("root", "./", "Set the root path to serve files (can be set with MAKESHIFT_ROOT)") + serveCmd.Flags().IntP("timeout", "t", 60, "Set the timeout in seconds for requests (can be set with MAKESHIFT_TIMEOUT)") + serveCmd.Flags().String("cacert", "", "Set the CA certificate path to load (can be set with MAKESHIFT_CACERT)") + serveCmd.Flags().String("keyfile", "", "Set the CA key file to use (can be set with MAKESHIFT_KEYFILE)") + + serveCmd.MarkFlagsRequiredTogether("cacert", "keyfile") + + rootCmd.AddCommand(serveCmd) +} diff --git a/cmd/upload.go b/cmd/upload.go index 9fc1f12..b4d09cb 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -1,14 +1,462 @@ package cmd -import "github.com/spf13/cobra" +import ( + "bufio" + "encoding/json" + "fmt" + "io/fs" + "maps" + "net/http" + "os" + "path/filepath" + "strings" + + "git.towk2.me/towk/makeshift/internal/format" + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/client" + "git.towk2.me/towk/makeshift/pkg/service" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +var ( + inputFormat format.DataFormat = format.JSON +) var uploadCmd = &cobra.Command{ Use: "upload", - Run: func(cmd *cobra.Command, args []string) { + Example: ` + # upload a single file in root directory + makeshift upload -d @compute-base.yaml + + # upload a directory + makeshift upload -d @setup/ + # upload an archive (extracted and saved on server) + makeshift upload -d @setup.tar.gz -t archive + + # upload multiple files with a specific path (used to set remote location) + makeshift upload -d @kubernetes.json -p nodes/kubernetes.json + makeshift upload -d @slurm.json -d @compute.json -p nodes +`, + Short: "Upload files and directories", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + setenv(cmd, "host", "MAKESHIFT_HOST") + setenv(cmd, "path", "MAKESHIFT_PATH") + setenv(cmd, "cacert", "MAKESHIFT_CACERT") + }, + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + path, _ = cmd.Flags().GetString("path") + cacertPath, _ = cmd.Flags().GetString("cacert") + dataArgs, _ = cmd.Flags().GetStringArray("data") + + inputData = processFiles(dataArgs) + useDirectoryPath = len(inputData) > 1 + c = client.New(host) + res *http.Response + query string + err error + ) + + log.Debug(). + Str("host", host). + Str("path", path). + Str("query", query). + Str("cacert", cacertPath). + Any("input", inputData). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + for inputPath, contents := range inputData { + log.Debug().Str("path", path).Int("size", len(contents)).Send() + if useDirectoryPath { + query = path + "/" + filepath.Clean(inputPath) + } else { + // use flag value if supplied + if cmd.Flags().Changed("path") { + query = path + } else { + query = inputPath + } + } + + query = fmt.Sprintf("/upload/%s", query) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodPost, + Body: contents, + }) + handleResponseError(res, host, query, err) + } + }, +} + +var uploadProfilesCmd = &cobra.Command{ + Use: "profile [profile_id]", + Example: ` + # upload a new profile + makeshift upload profile -d @compute.json kubernetes.json + + # upload a new profile with a specific path + makeshift upload profile -d @kubernetes.json + makeshift upload profile -d '{"id": "custom", "data": {}}' kubernetes.json +`, + Short: "Upload a new profile", + Run: func(cmd *cobra.Command, args []string) { + var ( + host, _ = cmd.Flags().GetString("host") + dataArgs, _ = cmd.Flags().GetStringArray("data") + cacertPath, _ = cmd.Flags().GetString("cacert") + profiles = processProfiles(dataArgs) + + c = client.New(host) + res *http.Response + query string + body []byte + err error + ) + + log.Debug(). + Str("host", host). + Str("query", query). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + // load files from args + for i, path := range args { + body, err = os.ReadFile(path) + if err != nil { + log.Error().Err(err). + Int("index", i). + Str("path", path). + Msg("failed to read profile file") + continue + } + var profile *makeshift.Profile + err = json.Unmarshal(body, &profile) + if err != nil { + log.Error().Err(err). + Int("index", i). + Str("path", path). + Msg("failed to unmarshal profile") + } + profiles = append(profiles, profile) + } + + // send each loaded profile to server + for _, profile := range profiles { + if profile == nil { + continue + } + + body, err = json.Marshal(profile) + if err != nil { + log.Error().Err(err).Msg("failed to marshal profile") + continue + } + + query = fmt.Sprintf("/profiles/%s", profile.ID) + res, body, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodPost, + Body: body, + }) + handleResponseError(res, host, query, err) + } + }, +} + +var uploadPluginsCmd = &cobra.Command{ + Use: "plugin [plugin_name]", + Example: ` + # upload a new plugin + makeshift upload plugin -d @slurm.so + makeshift upload plugin slurm.so +`, + Args: cobra.ExactArgs(1), + Short: "Upload a new plugin", + Run: func(cmd *cobra.Command, args []string) { + // make one request be host positional argument (restricted to 1 for now) + // temp := append(handleArgs(args), processDataArgs(dataArgs)...) + var ( + host, _ = cmd.Flags().GetString("host") + dataArgs, _ = cmd.Flags().GetStringArray("data") + cacertPath, _ = cmd.Flags().GetString("cacert") + + plugins = processFiles(dataArgs) + c = client.New(host) + res *http.Response + query string + body []byte + plugin makeshift.Plugin + err error + ) + + log.Debug(). + Str("host", host). + Str("query", query). + Str("cacert", cacertPath). + Send() + + if cacertPath != "" { + c.LoadCertificateFromPath(cacertPath) + } + + // load files from args + for i, path := range args { + body, err = os.ReadFile(path) + if err != nil { + log.Error().Err(err). + Int("index", i). + Str("path", path). + Msg("failed to read plugin file") + continue + } + + plugins[path] = body + } + + for path, contents := range plugins { + plugin, err = service.LoadPluginFromFile(path) + if err != nil { + log.Error().Err(err). + Str("path", path). + Msg("failed to load plugin from file") + } + + query = fmt.Sprintf("/plugins/%s", plugin.Name()) + res, _, err = c.MakeRequest(client.HTTPEnvelope{ + Path: query, + Method: http.MethodPost, + Body: contents, + }) + handleResponseError(res, host, query, err) + } }, } func init() { + uploadCmd.PersistentFlags().String("host", "http://localhost:5050", "Set the makeshift remote host (can be set with MAKESHIFT_HOST)") + uploadCmd.PersistentFlags().StringArrayP("data", "d", []string{}, "Set the data to send to specified host (prepend @ for files)") + uploadCmd.PersistentFlags().String("cacert", "", "Set the CA certificate path to load") + + uploadCmd.Flags().StringP("path", "p", ".", "Set the path to list files (can be set with MAKESHIFT_PATH)") + + uploadProfilesCmd.Flags().VarP(&inputFormat, "format", "F", "Set the input format for profile") + + uploadCmd.AddCommand(uploadProfilesCmd, uploadPluginsCmd) rootCmd.AddCommand(uploadCmd) } + +func processFiles(args []string) map[string][]byte { + // load data either from file or directly from args + var collection = make(map[string][]byte, len(args)) + for _, arg := range args { + // skip empty string args + if len(arg) > 0 { + // determine if we're reading from file to load contents + if strings.HasPrefix(arg, "@") { + var path string = strings.TrimLeft(arg, "@") + + // process sub-directories recursively + newCollection, err := processDir(path) + if err != nil { + log.Warn(). + Err(err). + Str("path", path). + Msg("failed to process directory at path") + } + log.Trace(). + Str("path", path). + Msg("new collection added at path") + maps.Copy(collection, newCollection) + + } else { + log.Warn().Msg("only files can be uploaded (add @ before the path with '--data' flag)") + + continue + } + } + } + return collection +} + +// processProfiles takes a slice of strings that check for the @ symbol and loads +// the contents from the file specified in place (which replaces the path). +// +// NOTE: The purpose is to make the input arguments uniform for our request. This +// function is meant to handle data passed with the `-d/--data` flag and positional +// args from the CLI. +func processProfiles(args []string) []*makeshift.Profile { + // load data either from file or directly from args + var collection = make([]*makeshift.Profile, len(args)) + for i, arg := range args { + // if arg is empty string, then skip and continue + if len(arg) > 0 { + // determine if we're reading from file to load contents + if strings.HasPrefix(arg, "@") { + var ( + path string = strings.TrimLeft(arg, "@") + contents []byte + data *makeshift.Profile + err error + ) + contents, err = os.ReadFile(path) + if err != nil { + log.Error(). + Err(err). + Str("path", path). + Msg("failed to read file") + continue + } + + // skip empty files + if len(contents) == 0 { + log.Warn(). + Str("path", path). + Msg("file is empty") + continue + } + + // convert/validate input data + data, err = parseProfile(contents, format.DataFormatFromFileExt(path, inputFormat)) + if err != nil { + log.Error(). + Err(err). + Str("path", path). + Msg("failed to validate input from file") + } + + // add loaded data to collection of all data + collection = append(collection, data) + } else { + // input should be a valid JSON + var ( + data *makeshift.Profile + input = []byte(arg) + err error + ) + if !json.Valid(input) { + log.Error().Msgf("argument %d not a valid JSON", i) + continue + } + err = json.Unmarshal(input, &data) + if err != nil { + log.Error(). + Err(err). + Msgf("failed to unmarshal input for argument %d", i) + } + return []*makeshift.Profile{data} + } + } + } + return collection +} + +func processDir(path string) (map[string][]byte, error) { + var ( + collection = map[string][]byte{} + fileInfo os.FileInfo + contents []byte + err error + ) + // determine if path is directory + if fileInfo, err = os.Stat(path); err == nil { + if fileInfo.IsDir() { + filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + contents, err = os.ReadFile(path) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("failed to read file") + return nil + } + + // skip empty files + if len(contents) == 0 { + log.Warn().Str("path", path).Msg("file is empty") + return nil + } + + log.Debug(). + Str("path", path). + Msg("file added to collection") + + // add loaded data to collection of all data + collection[path] = contents + } else { + // process sub-directories recursively + newCollection, err := processDir(path) + if err != nil { + return fmt.Errorf("failed to process directory at path '%s': %v", path, err) + } + log.Trace(). + Str("path", path). + Msg("new collection added from nested directory") + maps.Copy(collection, newCollection) + } + return nil + }) + } else { + contents, err = os.ReadFile(path) + if err != nil { + return collection, fmt.Errorf("failed to read file at path '%s': %v", path, err) + } + + // skip empty files + if len(contents) == 0 { + return collection, fmt.Errorf("file is empty") + } + + log.Debug(). + Str("path", path). + Msg("file added to collection") + + // add loaded data to collection of all data + collection[path] = contents + } + } else { + return nil, fmt.Errorf("failed to stat file: %v", err) + } + return collection, nil +} + +func parseProfile(contents []byte, dataFormat format.DataFormat) (*makeshift.Profile, error) { + var ( + data *makeshift.Profile + err error + ) + + // convert/validate JSON input format + err = format.Unmarshal(contents, &data, dataFormat) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal profile: %v", err) + } + return data, nil +} + +// ReadStdin reads all of standard input and returns the bytes. If an error +// occurs during scanning, it is returned. +func ReadStdin() ([]byte, error) { + var b []byte + input := bufio.NewScanner(os.Stdin) + for input.Scan() { + b = append(b, input.Bytes()...) + b = append(b, byte('\n')) + if len(b) == 0 { + break + } + } + if err := input.Err(); err != nil { + return b, fmt.Errorf("failed to read stdin: %w", err) + } + return b, nil +} diff --git a/examples/plugin/test.go b/examples/plugin/test.go deleted file mode 100644 index 6289e1c..0000000 --- a/examples/plugin/test.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/generator" -) - -type TestGenerator struct{} - -func (g *TestGenerator) GetName() string { return "test" } -func (g *TestGenerator) GetVersion() string { return "v1.0.0" } -func (g *TestGenerator) GetDescription() string { - return "This is a plugin creating for running tests." -} -func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) { - return generator.ApplyTemplates(generator.Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - }, params.Templates) -} - -var Generator TestGenerator diff --git a/examples/templates/conman.jinja b/examples/templates/conman.jinja deleted file mode 100644 index 69e779c..0000000 --- a/examples/templates/conman.jinja +++ /dev/null @@ -1,23 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# -SERVER keepalive=ON -SERVER logdir="/var/log/conman" -SERVER logfile="/var/log/conman.log" -SERVER loopback=ON -SERVER pidfile="/var/run/conman.pid" -SERVER resetcmd="/usr/bin/powerman -0 \%N; sleep 5; /usr/bin/powerman -1 \%N" -SERVER tcpwrappers=ON -#SERVER timestamp=1h - -GLOBAL seropts="115200,8n1" -GLOBAL log="/var/log/conman/console.\%N" -GLOBAL logopts="sanitize,timestamp" - -{{ consoles }} diff --git a/examples/templates/dhcpd.jinja b/examples/templates/dhcpd.jinja deleted file mode 100644 index 264e590..0000000 --- a/examples/templates/dhcpd.jinja +++ /dev/null @@ -1,51 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# -allow booting; -allow bootp; -ddns-update-style interim; -authoritative; - -option space ipxe; - -# Tell iPXE to not wait for ProxyDHCP requests to speed up boot. -option ipxe.no-pxedhcp code 176 = unsigned integer 8; -option ipxe.no-pxedhcp 1; - -option architecture-type code 93 = unsigned integer 16; - -if exists user-class and option user-class = "iPXE" { - filename "http://%{IPADDR}/WW/ipxe/cfg/${mac}"; -} else { - if option architecture-type = 00:0B { - filename "/warewulf/ipxe/bin-arm64-efi/snp.efi"; - } elsif option architecture-type = 00:0A { - filename "/warewulf/ipxe/bin-arm32-efi/placeholder.efi"; - } elsif option architecture-type = 00:09 { - filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi"; - } elsif option architecture-type = 00:07 { - filename "/warewulf/ipxe/bin-x86_64-efi/snp.efi"; - } elsif option architecture-type = 00:06 { - filename "/warewulf/ipxe/bin-i386-efi/snp.efi"; - } elsif option architecture-type = 00:00 { - filename "/warewulf/ipxe/bin-i386-pcbios/undionly.kpxe"; - } -} - -subnet %{NETWORK} netmask %{NETMASK} { - not authoritative; - # option interface-mtu 9000; - option subnet-mask %{NETMASK}; -} - -# Compute Nodes (WIP - see the dhcpd generator plugin) -{{ compute_nodes }} - -# Node entries will follow below -{{ node_entries }} \ No newline at end of file diff --git a/examples/templates/dnsmasq.jinja b/examples/templates/dnsmasq.jinja deleted file mode 100644 index 1cc4dab..0000000 --- a/examples/templates/dnsmasq.jinja +++ /dev/null @@ -1,10 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# -{{ dhcp_hosts }} diff --git a/examples/templates/powerman.jinja b/examples/templates/powerman.jinja deleted file mode 100644 index 44c13e4..0000000 --- a/examples/templates/powerman.jinja +++ /dev/null @@ -1,18 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# -include "/etc/powerman/ipmipower.dev" -include "/etc/powerman/ipmi.dev" - - -# list of devices -{{ devices }} - -# create nodes based on found nodes in hostfile -{{ nodes }} \ No newline at end of file diff --git a/examples/templates/test.j2 b/examples/templates/test.j2 deleted file mode 100644 index 17bd6b6..0000000 --- a/examples/templates/test.j2 +++ /dev/null @@ -1,16 +0,0 @@ -# -# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: -# Name: {{ plugin_name }} -# Version: {{ plugin_version }} -# Description: {{ plugin_description }} -# -# Source code: https://github.com/OpenCHAMI/configurator -# Creating plugins: https://git.towk2.me/towk/configurator/blob/main/README.md#creating-generator-plugins -# - -# TODO: test variables - -# TODO: test if/else statements - -# TODO: test for loops - diff --git a/go.mod b/go.mod index dcb6e87..4b47295 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,21 @@ -module git.towk2.me/towk/configurator +module git.towk2.me/towk/makeshift -go 1.21.5 +go 1.24.4 + +toolchain go1.24.6 require ( github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 + github.com/cavaliergopher/grab/v3 v3.0.1 github.com/go-chi/chi/v5 v5.1.0 github.com/lestrrat-go/jwx/v2 v2.1.1 - github.com/nikolalohinski/gonja/v2 v2.2.0 - github.com/openchami/chi-middleware/auth v0.0.0-20240812224658-b16b83c70700 - github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700 - github.com/rodaine/table v1.2.0 - github.com/rs/zerolog v1.33.0 + github.com/nikolalohinski/gonja/v2 v2.3.5 + github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.8.0 - golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 - gopkg.in/yaml.v2 v2.4.0 + github.com/tidwall/sjson v1.2.5 + go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -22,7 +24,7 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kr/text v0.1.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect @@ -36,7 +38,10 @@ require ( github.com/segmentio/asm v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tidwall/gjson v1.14.2 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.23.0 // indirect ) diff --git a/go.sum b/go.sum index 8452dee..91a82d0 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,11 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk= github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8= +github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= +github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -13,27 +16,26 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= @@ -51,34 +53,27 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/nikolalohinski/gonja/v2 v2.2.0 h1:tAs3BDHNjvPj48F2BL5t7iVhN32HhgeldAl3EmdsLh8= -github.com/nikolalohinski/gonja/v2 v2.2.0/go.mod h1:l9DuWJvT/BddBr2SsmEimESD6msSqRw7u5HzI2Um+sc= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= -github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= -github.com/openchami/chi-middleware/auth v0.0.0-20240812224658-b16b83c70700 h1:XADGipD2FZ9swuFUqeL7h63j3voiq9qA7P0aKsqgZKg= -github.com/openchami/chi-middleware/auth v0.0.0-20240812224658-b16b83c70700/go.mod h1:kswb9kU5cZAFRAvf1dAUJRWbQyjDEb0qkxW4ncDdEXg= -github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700 h1:Gzt5f6RK39CHvY3SJudzBb/RK4tVh/S3CpJ0eQlbNdg= -github.com/openchami/chi-middleware/log v0.0.0-20240812224658-b16b83c70700/go.mod h1:UuXvr2loD4MtvZeKr57W0WpBs+gm0KM1kdtcXrE8M6s= +github.com/nikolalohinski/gonja/v2 v2.3.5 h1:7ukCnsokmOIGXOjgW/WrM+xqgwjsQcU0ejFrrz4HQXk= +github.com/nikolalohinski/gonja/v2 v2.3.5/go.mod h1:UIzXPVuOsr5h7dZ5DUbqk3/Z7oFA/NLGQGMjqT4L2aU= +github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= +github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= +github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rodaine/table v1.2.0 h1:38HEnwK4mKSHQJIkavVj+bst1TEY7j9zhLMWu4QJrMA= -github.com/rodaine/table v1.2.0/go.mod h1:wejb/q/Yd4T/SVmBSRMr7GCq3KlcZp3gyNYdLSBhkaE= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= -github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= @@ -89,38 +84,43 @@ github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyh github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..4fadf22 --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,220 @@ +package archive + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + makeshift "git.towk2.me/towk/makeshift/pkg" +) + +func Create(filenames []string, buf io.Writer, hooks []makeshift.Hook) error { + // Create new Writers for gzip and tar + // These writers are chained. Writing to the tar writer will + // write to the gzip writer which in turn will write to + // the "buf" writer + gw := gzip.NewWriter(buf) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + + // Iterate over files and add them to the tar archive + for _, filename := range filenames { + err := addToArchive(tw, filename, hooks) + if err != nil { + return err + } + } + + return nil +} + +func Expand(tarname, xpath string) error { + tarfile, err := os.Open(tarname) + if err != nil { + return err + } + defer tarfile.Close() + // absPath, err := filepath.Abs(xpath) + // if err != nil { + // return err + // } + tr := tar.NewReader(tarfile) + if strings.HasSuffix(tarname, ".gz") { + gz, err := gzip.NewReader(tarfile) + if err != nil { + return fmt.Errorf("failed to create new gzip reader: %v", err) + } + defer gz.Close() + tr = tar.NewReader(gz) + } + + // untar each segment + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to get next tar header: %v", err) + } + + // determine proper file path info + var ( + fileinfo = header.FileInfo() + filename = header.Name + file *os.File + abspath string + dirpath string + ) + + // absFileName := filepath.Join(absPath, filename) // if a dir, create it, then go to next segment + if fileinfo.Mode().IsDir() { + if err := os.MkdirAll(filename, 0o755); err != nil { + return fmt.Errorf("failed to make directory '%s': %v", filename, err) + } + continue + } + + dirpath = filepath.Dir(filename) + if err = os.MkdirAll(dirpath, 0o777); err != nil { + return fmt.Errorf("failed to make directory '%s': %v", err) + } + + // create new file with original file mode + abspath, err = filepath.Abs(filename) + if err != nil { + return fmt.Errorf("failed to get absolute path: %v", err) + } + file, err = os.OpenFile( + abspath, + os.O_RDWR|os.O_CREATE|os.O_TRUNC, + fileinfo.Mode().Perm(), + ) + if err != nil { + return fmt.Errorf("failed to open file: %v", err) + } + // fmt.Printf("x %s\n", filename) + + // copy the contents to the new file + n, err := io.Copy(file, tr) + if err != nil { + return fmt.Errorf("failed to copy file: %v", err) + } + if err = file.Close(); err != nil { + return fmt.Errorf("failed to close file: %v", err) + } + if n != fileinfo.Size() { + return fmt.Errorf("wrote %d, want %d", n, fileinfo.Size()) + } + } + return nil +} + +func addToArchive(tw *tar.Writer, filename string, hooks []makeshift.Hook) error { + var ( + tempfile = fmt.Sprintf("%s.tmp", filename) + file *os.File + contents []byte + data any + err error + ) + // run pre-hooks to modify the contents of the file + // before archiving using plugins + for _, hook := range hooks { + // set the file in the data store before running hook + contents, err = os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read '%s' to download: %v", filename, err) + } + hook.Data.Set("file", contents) + + err = hook.Init() + if err != nil { + return err + } + err = hook.Run() + if err != nil { + return err + } + err = hook.Cleanup() + if err != nil { + return err + } + + // create temporary file to use to add to archive + hook = hooks[len(hooks)-1] + data, err = hook.Data.Get("out") + if err != nil { + return fmt.Errorf("failed to get output data from '%s' plugin: %v", hook.Plugin.Name(), err) + } + + err = os.WriteFile(tempfile, data.([]byte), 0o777) + if err != nil { + return fmt.Errorf("failed to write temporary file: %v", err) + } + } + + // use original file if no hooks to write archive + if len(hooks) == 0 { + file, err = os.Open(filename) + } else { + file, err = os.Open(tempfile) + } + if err != nil { + return fmt.Errorf("failed to open archive file: %v", err) + } + defer file.Close() + + // get FileInfo for file size, mode, etc. + info, err := file.Stat() + if err != nil { + return err + } + + // skip file if it's a directory + if info.IsDir() { + return nil + } + + // create a tar Header from the FileInfo data + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return fmt.Errorf("failed to create FileInfoHeader: %v", err) + } + + // use full path as name (FileInfoHeader only takes the basename) to + // preserve directory structure + // see for more info: https://golang.org/src/archive/tar/common.go?#L626 + header.Name = filename + + // Write file header to the tar archive + err = tw.WriteHeader(header) + if err != nil { + return err + } + + // copy file content to tar archive + if len(hooks) == 0 { + _, err = io.Copy(tw, file) + } else { + _, err = io.Copy(tw, strings.NewReader(string(data.([]byte)))) + } + if err != nil { + return err + } + + // delete the temporary file since we're done with it + if len(hooks) != 0 { + err = os.Remove(tempfile) + if err != nil { + return err + } + } + + return nil +} diff --git a/internal/format/format.go b/internal/format/format.go new file mode 100644 index 0000000..b753268 --- /dev/null +++ b/internal/format/format.go @@ -0,0 +1,104 @@ +package format + +import ( + "encoding/json" + "fmt" + "path/filepath" + + "gopkg.in/yaml.v3" +) + +type DataFormat string + +const ( + List DataFormat = "list" + JSON DataFormat = "json" + YAML DataFormat = "yaml" +) + +func (df DataFormat) String() string { + return string(df) +} + +func (df *DataFormat) Set(v string) error { + switch DataFormat(v) { + case List, JSON, YAML: + *df = DataFormat(v) + return nil + default: + return fmt.Errorf("must be one of %v", []DataFormat{ + List, JSON, YAML, + }) + } +} + +func (df DataFormat) Type() string { + return "DataFormat" +} + +// MarshalData marshals arbitrary data into a byte slice formatted as outFormat. +// If a marshalling error occurs or outFormat is unknown, an error is returned. +// +// Supported values are: json, list, yaml +func Marshal(data interface{}, outFormat DataFormat) ([]byte, error) { + switch outFormat { + case JSON: + if bytes, err := json.MarshalIndent(data, "", " "); err != nil { + return nil, fmt.Errorf("failed to marshal data into JSON: %w", err) + } else { + return bytes, nil + } + case YAML: + if bytes, err := yaml.Marshal(data); err != nil { + return nil, fmt.Errorf("failed to marshal data into YAML: %w", err) + } else { + return bytes, nil + } + case List: + return nil, fmt.Errorf("this data format cannot be marshaled") + default: + return nil, fmt.Errorf("unknown data format: %s", outFormat) + } +} + +// UnmarshalData unmarshals a byte slice formatted as inFormat into an interface +// v. If an unmarshalling error occurs or inFormat is unknown, an error is +// returned. +// +// Supported values are: json, list, yaml +func Unmarshal(data []byte, v interface{}, inFormat DataFormat) error { + switch inFormat { + case JSON: + if err := json.Unmarshal(data, v); err != nil { + return fmt.Errorf("failed to unmarshal data into JSON: %w", err) + } + case YAML: + if err := yaml.Unmarshal(data, v); err != nil { + return fmt.Errorf("failed to unmarshal data into YAML: %w", err) + } + case List: + return fmt.Errorf("this data format cannot be unmarshaled") + default: + return fmt.Errorf("unknown data format: %s", inFormat) + } + + return nil +} + +// DataFormatFromFileExt determines the type of the contents +// (JSON or YAML) based on the filname extension. The default +// format is passed in, so if it doesn't match one of the cases, +// that's what we will use. The defaultFmt value takes into account +// both the standard default format (JSON) and any command line +// change to that provided by options. +func DataFormatFromFileExt(path string, defaultFmt DataFormat) DataFormat { + switch filepath.Ext(path) { + case ".json", ".JSON": + // The file is a JSON file + return JSON + case ".yaml", ".yml", ".YAML", ".YML": + // The file is a YAML file + return YAML + } + return defaultFmt +} diff --git a/cmd/profiles.go b/lib/.gitkeep similarity index 100% rename from cmd/profiles.go rename to lib/.gitkeep diff --git a/main.go b/main.go index 0828d29..ad4d69e 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ package main -import "git.towk2.me/towk/configurator/cmd" +import "git.towk2.me/towk/makeshift/cmd" func main() { cmd.Execute() diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 0000000..028fced --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,157 @@ +package client + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "mime/multipart" + "net" + "net/http" + "os" + "strings" + "time" + + "git.towk2.me/towk/makeshift/pkg/util" + "github.com/cavaliergopher/grab/v3" +) + +type HTTPBody []byte +type HTTPHeader map[string]string +type HTTPEnvelope struct { + Path string + Method string + Header HTTPHeader + Body HTTPBody + CACert string +} + +type Client struct { + http.Client + BaseURI string + AccessToken string +} + +func New(uri string) Client { + return Client{ + BaseURI: strings.TrimSuffix(uri, "/"), + } +} + +func NewHTTPEnvelope() HTTPEnvelope { + return HTTPEnvelope{ + Path: "", + Method: http.MethodGet, + Header: nil, + Body: nil, + CACert: "", + } +} + +func (c *Client) MakeRequest(env HTTPEnvelope) (*http.Response, []byte, error) { + return util.MakeRequest(c.BaseURI+env.Path, env.Method, env.Body, env.Header) +} + +func (c *Client) Download(out string, env HTTPEnvelope) (*grab.Response, error) { + if out == "" { + return grab.Get(out, c.BaseURI+env.Path) + } + return grab.Get(out, c.BaseURI+env.Path) +} + +func (c *Client) UploadMultipartFile(uri, key, path string) (*http.Response, error) { + body, writer := io.Pipe() + + req, err := http.NewRequest(http.MethodPost, uri, body) + if err != nil { + return nil, err + } + + mwriter := multipart.NewWriter(writer) + req.Header.Add("Content-Type", mwriter.FormDataContentType()) + + errchan := make(chan error) + + go func() { + defer close(errchan) + defer writer.Close() + defer mwriter.Close() + + w, err := mwriter.CreateFormFile(key, path) + if err != nil { + errchan <- err + return + } + + in, err := os.Open(path) + if err != nil { + errchan <- err + return + } + defer in.Close() + + if written, err := io.Copy(w, in); err != nil { + errchan <- fmt.Errorf("error copying %s (%d bytes written): %v", path, written, err) + return + } + + if err := mwriter.Close(); err != nil { + errchan <- err + return + } + }() + + resp, err := c.Do(req) + merr := <-errchan + + if err != nil || merr != nil { + return resp, fmt.Errorf("http error: %v, multipart error: %v", err, merr) + } + + return resp, nil +} + +func (c *Client) LoadCertificateFromPath(path string) error { + cacert, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read certificate at path: %s", path) + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + err = c.LoadCertificateFromPool(certPool) + if err != nil { + return fmt.Errorf("could not initialize certificate from pool: %v", err) + } + return nil +} + +func (c *Client) LoadCertificateFromPool(certPool *x509.CertPool) error { + // make sure we have a valid cert pool + if certPool == nil { + return fmt.Errorf("invalid cert pool") + } + + // make sure that we can access the internal client + c.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: false, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + return nil +} + +func mustOpen(f string) *os.File { + r, err := os.Open(f) + if err != nil { + panic(err) + } + return r +} diff --git a/pkg/generator/conman.go b/pkg/generator/conman.go deleted file mode 100644 index be8e882..0000000 --- a/pkg/generator/conman.go +++ /dev/null @@ -1,56 +0,0 @@ -package generator - -import ( - "fmt" - - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Conman struct{} - -func (g *Conman) GetName() string { - return "conman" -} - -func (g *Conman) GetVersion() string { - return util.GitCommit() -} - -func (g *Conman) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Conman) Generate(config *config.Config, params Params) (FileMap, error) { - var ( - smdClient = client.NewSmdClient(params.ClientOpts...) - eps = []configurator.RedfishEndpoint{} - err error = nil - consoles = "" - ) - - // fetch required data from SMD to create config - eps, err = smdClient.FetchRedfishEndpoints(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch redfish endpoints with client: %v", err) - } - - // format output to write to config file - consoles = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" - for _, ep := range eps { - consoles += fmt.Sprintf("CONSOLE name=%s dev=ipmi:%s-bmc ipmiopts=U:%s,P:%s,W:solpayloadsize\n", ep.Name, ep.Name, ep.User, ep.Password) - } - consoles += "# =====================================================================" - - // apply template substitutions and return output as byte array - return ApplyTemplates(Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - "server_opts": "", - "global_opts": "", - "consoles": consoles, - }, params.Templates) -} diff --git a/pkg/generator/coredhcp.go b/pkg/generator/coredhcp.go deleted file mode 100644 index ccc79cd..0000000 --- a/pkg/generator/coredhcp.go +++ /dev/null @@ -1,26 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type CoreDhcp struct{} - -func (g *CoreDhcp) GetName() string { - return "coredhcp" -} - -func (g *CoreDhcp) GetVersion() string { - return util.GitCommit() -} - -func (g *CoreDhcp) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s' to generate config files. (WIP)", g.GetName()) -} - -func (g *CoreDhcp) Generate(config *config.Config, params Params) (FileMap, error) { - return nil, fmt.Errorf("plugin does not implement generation function") -} diff --git a/pkg/generator/dhcpd.go b/pkg/generator/dhcpd.go deleted file mode 100644 index 75a41bb..0000000 --- a/pkg/generator/dhcpd.go +++ /dev/null @@ -1,64 +0,0 @@ -package generator - -import ( - "fmt" - - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type DHCPd struct{} - -func (g *DHCPd) GetName() string { - return "dhcpd" -} - -func (g *DHCPd) GetVersion() string { - return util.GitCommit() -} - -func (g *DHCPd) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *DHCPd) Generate(config *config.Config, params Params) (FileMap, error) { - var ( - smdClient = client.NewSmdClient(params.ClientOpts...) - eths = []configurator.EthernetInterface{} - computeNodes = "" - err error = nil - ) - - // - eths, err = smdClient.FetchEthernetInterfaces(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %w", err) - } - - // check if we have the required params first - if eths == nil { - return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") - } - if len(eths) <= 0 { - return nil, fmt.Errorf("no ethernet interfaces found") - } - - // format output to write to config file - computeNodes = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" - for _, eth := range eths { - if len(eth.IpAddresses) == 0 { - continue - } - computeNodes += fmt.Sprintf("host %s { hardware ethernet %s; fixed-address %s} ", eth.ComponentId, eth.MacAddress, eth.IpAddresses[0]) - } - computeNodes += "# =====================================================================" - return ApplyTemplates(Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - "compute_nodes": computeNodes, - "node_entries": "", - }, params.Templates) -} diff --git a/pkg/generator/dnsmasq.go b/pkg/generator/dnsmasq.go deleted file mode 100644 index 16ea6c9..0000000 --- a/pkg/generator/dnsmasq.go +++ /dev/null @@ -1,71 +0,0 @@ -package generator - -import ( - "fmt" - - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type DNSMasq struct{} - -func (g *DNSMasq) GetName() string { - return "dnsmasq" -} - -func (g *DNSMasq) GetVersion() string { - return util.GitCommit() -} - -func (g *DNSMasq) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *DNSMasq) Generate(config *config.Config, params Params) (FileMap, error) { - // make sure we have a valid config first - if config == nil { - return nil, fmt.Errorf("invalid config (config is nil)") - } - - // set all the defaults for variables - var ( - smdClient = client.NewSmdClient(params.ClientOpts...) - eths = []configurator.EthernetInterface{} - err error = nil - ) - - // if we have a client, try making the request for the ethernet interfaces - eths, err = smdClient.FetchEthernetInterfaces(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) - } - - // check if we have the required params first - if eths == nil { - return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") - } - if len(eths) <= 0 { - return nil, fmt.Errorf("no ethernet interfaces found") - } - - // format output to write to config file - output := "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" - for _, eth := range eths { - if eth.Type == "NodeBMC" { - output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n" - } else { - output += "dhcp-host=" + eth.MacAddress + "," + eth.ComponentId + "," + eth.IpAddresses[0].IpAddress + "\n" - } - } - output += "# =====================================================================" - - // apply template substitutions and return output as byte array - return ApplyTemplates(Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - "dhcp_hosts": output, - }, params.Templates) -} diff --git a/pkg/generator/example.go b/pkg/generator/example.go deleted file mode 100644 index 12091ba..0000000 --- a/pkg/generator/example.go +++ /dev/null @@ -1,31 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Example struct { - Message string -} - -func (g *Example) GetName() string { - return "example" -} - -func (g *Example) GetVersion() string { - return util.GitCommit() -} - -func (g *Example) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Example) Generate(config *config.Config, params Params) (FileMap, error) { - g.Message = ` - This is an example generator plugin. See the file in 'internal/generator/plugins/example/example.go' on - information about constructing plugins and plugin requirements.` - return FileMap{"example": []byte(g.Message)}, nil -} diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go deleted file mode 100644 index 3667130..0000000 --- a/pkg/generator/generator.go +++ /dev/null @@ -1,264 +0,0 @@ -package generator - -import ( - "fmt" - "io/fs" - "os" - "path/filepath" - "plugin" - - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" - "github.com/rs/zerolog/log" -) - -type ( - Mappings map[string]any - FileMap map[string][]byte - FileList [][]byte - - // Generator interface used to define how files are created. Plugins can - // be created entirely independent of the main driver program. - Generator interface { - GetName() string - GetVersion() string - GetDescription() string - Generate(config *config.Config, params Params) (FileMap, error) - } -) - -var DefaultGenerators = createDefaultGenerators() - -func createDefaultGenerators() map[string]Generator { - var ( - generatorMap = map[string]Generator{} - generators = []Generator{ - &Conman{}, &DHCPd{}, &DNSMasq{}, &Warewulf{}, &Example{}, &CoreDhcp{}, - } - ) - for _, g := range generators { - generatorMap[g.GetName()] = g - } - return generatorMap -} - -// Converts the file outputs from map[string][]byte to map[string]string. -func ConvertContentsToString(f FileMap) map[string]string { - n := make(map[string]string, len(f)) - for k, v := range f { - n[k] = string(v) - } - return n -} - -// Loads files without applying any Jinja 2 templating. -func LoadFiles(paths ...string) (FileMap, error) { - var outputs = FileMap{} - for _, path := range paths { - expandedPaths, err := filepath.Glob(path) - if err != nil { - return nil, fmt.Errorf("failed to glob path: %w", err) - } - for _, expandedPath := range expandedPaths { - info, err := os.Stat(expandedPath) - if err != nil { - fmt.Println(err) - return nil, fmt.Errorf("failed to stat file or directory: %w", err) - } - // skip any directories found - if info.IsDir() { - continue - } - b, err := os.ReadFile(expandedPath) - if err != nil { - return nil, fmt.Errorf("failed to read file: %w", err) - } - - outputs[expandedPath] = b - } - } - - return outputs, nil -} - -// Loads a single generator plugin given a single file path. -func LoadPlugin(path string) (Generator, error) { - // skip loading plugin if path is a directory with no error - if isDir, err := util.IsDirectory(path); err == nil && isDir { - return nil, nil - } else if err != nil { - return nil, fmt.Errorf("failed to test if plugin path is directory: %w", err) - } - - // try and open the plugin - p, err := plugin.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open plugin: %w", err) - } - - // load the "Generator" symbol from plugin - symbol, err := p.Lookup("Generator") - if err != nil { - return nil, fmt.Errorf("failed to look up symbol at path '%s': %w", path, err) - } - - // assert that the plugin loaded has a valid generator - gen, ok := symbol.(Generator) - if !ok { - return nil, fmt.Errorf("failed to load the correct symbol type at path '%s'", path) - } - return gen, nil -} - -// Loads all generator plugins in a given directory. -// -// Returns a map of generators. Each generator can be accessed by the name -// returned by the generator.GetName() implemented. -func LoadPlugins(dirpath string, opts ...Option) (map[string]Generator, error) { - // check if verbose option is supplied - var ( - generators = make(map[string]Generator) - params = ToParams(opts...) - ) - - // - err := filepath.Walk(dirpath, func(path string, info fs.FileInfo, err error) error { - // skip trying to load generator plugin if directory or error - if info.IsDir() || err != nil { - return nil - } - - // only try loading if file has .so extension - if filepath.Ext(path) != ".so" { - return nil - } - - // load the generator plugin from current path - gen, err := LoadPlugin(path) - if err != nil { - return fmt.Errorf("failed to load generator in directory '%s': %w", path, err) - } - - // show the plugins found if verbose flag is set - if params.Verbose { - log.Info().Str("plugin_name", gen.GetName()).Msg("found plugin") - } - - // map each generator plugin by name for lookup - generators[gen.GetName()] = gen - return nil - }) - - if err != nil { - return nil, fmt.Errorf("failed to walk directory: %w", err) - } - - return generators, nil -} - -// Generate() is the main function to generate a collection of files and returns them as a map. -// This function only expects a path to a plugin and paths to a collection of templates to -// be used. This function will only load the plugin on-demand and fetch resources as needed. -// -// This function requires that a target and plugin path be set at minimum. -func Generate(config *config.Config, plugin string, params Params) (FileMap, error) { - var ( - gen Generator - ok bool - err error - ) - - // check if generator is built-in first before loading external plugin - log.Debug().Any("generators", DefaultGenerators).Msg("available generators") - gen, ok = DefaultGenerators[plugin] - if !ok { - // only load the plugin needed for this target if we don't find default - log.Error().Str("plugin", plugin).Msg("could not find target in default generators") - gen, err = LoadPlugin(plugin) - if err != nil { - return nil, fmt.Errorf("failed to load plugin from file: %v", err) - } - } - - return gen.Generate(config, params) -} - -// Main function to generate a collection of files as a map with the path as the key and -// the contents of the file as the value. This function currently expects a list of plugin -// paths to load all plugins within a directory. Then, each plugin's generator.GenerateWithTarget() -// function is called for each target specified. -// -// This function is the corresponding implementation for the "generate" CLI subcommand. -// It is also call when running the configurator as a service with the "/generate" route. -// -// TODO: Separate loading plugins so we can load them once when running as a service. -func GenerateWithTarget(config *config.Config, target string) (FileMap, error) { - // load generator plugins to generate configs or to print - var ( - opts []client.Option - targetInfo configurator.Target - generator Generator - params Params - err error - ok bool - ) - - // check if a target is supplied - if target == "" { - return nil, fmt.Errorf("must specify a target") - } - - // load target information from config - targetInfo, ok = config.Targets[target] - if !ok { - log.Warn().Str("target", target).Msg("target not found in config") - } - - // if no plugin supplied in config target, then using the target supplied - if targetInfo.Plugin == "" { - targetInfo.Plugin = target - } - - // check if generator is built-in first before loading - generator, ok = DefaultGenerators[target] - if !ok { - // only load the plugin needed for this target if we don't find default - log.Warn().Str("target", target).Msg("could not find target in default generators") - generator, err = LoadPlugin(targetInfo.Plugin) - if err != nil { - return nil, fmt.Errorf("failed to load plugin: %v", err) - } - } - - // check if there's at least one template available - if len(targetInfo.TemplatePaths) <= 0 { - return nil, fmt.Errorf("expects at least one template to be available") - } - - // prepare params to pass into generator - params.Templates = map[string]Template{} - for _, templatePath := range targetInfo.TemplatePaths { - template := Template{} - template.LoadFromFile(templatePath) - params.Templates[templatePath] = template - } - - // set the client options - if config.AccessToken != "" { - params.ClientOpts = append(opts, client.WithAccessToken(config.AccessToken)) - } - if config.CertPath != "" { - params.ClientOpts = append(opts, client.WithCertPoolFile(config.CertPath)) - } - - // load files that are not to be copied - params.Files, err = LoadFiles(targetInfo.FilePaths...) - if err != nil { - return nil, fmt.Errorf("failed to load files to copy: %v", err) - } - - // run the generator plugin from target passed - return generator.Generate(config, params) -} diff --git a/pkg/generator/hostfile.go b/pkg/generator/hostfile.go deleted file mode 100644 index a9f92e5..0000000 --- a/pkg/generator/hostfile.go +++ /dev/null @@ -1,26 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Hostfile struct{} - -func (g *Hostfile) GetName() string { - return "hostfile" -} - -func (g *Hostfile) GetVersion() string { - return util.GitCommit() -} - -func (g *Hostfile) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Hostfile) Generate(config *config.Config, opts ...Option) (FileMap, error) { - return nil, fmt.Errorf("plugin does not implement generation function") -} diff --git a/pkg/generator/params.go b/pkg/generator/params.go deleted file mode 100644 index e9037ec..0000000 --- a/pkg/generator/params.go +++ /dev/null @@ -1,43 +0,0 @@ -package generator - -import ( - configurator "git.towk2.me/towk/configurator/pkg" - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" -) - -type ( - // Params used by the generator - Params struct { - Templates map[string]Template - Files map[string][]byte - ClientOpts []client.Option - Verbose bool - } - Option func(Params) -) - -func ToParams(opts ...Option) Params { - params := Params{} - for _, opt := range opts { - opt(params) - } - return params -} - -func WithClientOpts(opts ...client.Option) Option { - return func(p Params) { - p.ClientOpts = opts - } -} - -func WithTemplates(templates map[string]Template) Option { - return func(p Params) { - p.Templates = templates - } -} - -// Helper function to get the target in generator.Generate() plugin implementations. -func GetTarget(config *config.Config, key string) configurator.Target { - return config.Targets[key] -} diff --git a/pkg/generator/powerman.go b/pkg/generator/powerman.go deleted file mode 100644 index d8e376f..0000000 --- a/pkg/generator/powerman.go +++ /dev/null @@ -1,26 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Powerman struct{} - -func (g *Powerman) GetName() string { - return "powerman" -} - -func (g *Powerman) GetVersion() string { - return util.GitCommit() -} - -func (g *Powerman) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Powerman) Generate(config *config.Config, opts ...Option) (FileMap, error) { - return nil, fmt.Errorf("plugin does not implement generation function") -} diff --git a/pkg/generator/syslog.go b/pkg/generator/syslog.go deleted file mode 100644 index 450dcbc..0000000 --- a/pkg/generator/syslog.go +++ /dev/null @@ -1,26 +0,0 @@ -package generator - -import ( - "fmt" - - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" -) - -type Syslog struct{} - -func (g *Syslog) GetName() string { - return "syslog" -} - -func (g *Syslog) GetVersion() string { - return util.GitCommit() -} - -func (g *Syslog) GetDescription() string { - return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) -} - -func (g *Syslog) Generate(config *config.Config, opts ...Option) (FileMap, error) { - return nil, fmt.Errorf("plugin does not implement generation function") -} diff --git a/pkg/generator/templates.go b/pkg/generator/templates.go deleted file mode 100644 index 3028949..0000000 --- a/pkg/generator/templates.go +++ /dev/null @@ -1,98 +0,0 @@ -package generator - -import ( - "bytes" - "fmt" - "os" - - "git.towk2.me/towk/configurator/pkg/util" - "github.com/nikolalohinski/gonja/v2" - "github.com/nikolalohinski/gonja/v2/exec" - "github.com/rs/zerolog/log" -) - -type Template struct { - Contents []byte `json:"contents"` -} - -func (t *Template) LoadFromFile(path string) error { - // skip loading template if path is a directory with no error - if isDir, err := util.IsDirectory(path); err == nil && isDir { - return nil - } else if err != nil { - return fmt.Errorf("failed to test if template path is directory: %w", err) - } - - // try and read the contents of the file - // NOTE: we don't care if this is actually a Jinja template - // or not...at least for now. - contents, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read file: %v", err) - } - t.Contents = contents - return nil -} - -func (t *Template) IsEmpty() bool { - return len(t.Contents) <= 0 -} - -// Wrapper function to slightly abstract away some of the nuances with using gonja -// into a single function call. This function is *mostly* for convenience and -// simplication. If no paths are supplied, then no templates will be applied and -// there will be no output. -// -// The "FileList" returns a slice of byte arrays in the same order as the argument -// list supplied, but with the Jinja templating applied. -func ApplyTemplates(mappings Mappings, templates map[string]Template) (FileMap, error) { - var ( - data = exec.NewContext(mappings) - outputs = FileMap{} - ) - - for path, template := range templates { - // load jinja template from file - t, err := gonja.FromBytes(template.Contents) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %w", err) - } - - // execute/render jinja template - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %w", err) - } - outputs[path] = b.Bytes() - } - - log.Debug().Any("templates", templates).Any("outputs", outputs).Any("mappings", mappings).Msg("apply templates") - - return outputs, nil -} - -// Wrapper function similiar to "ApplyTemplates" but takes file paths as arguments. -// This function will load templates from a file instead of using file contents. -func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) { - var ( - data = exec.NewContext(mappings) - outputs = FileMap{} - ) - - for _, path := range paths { - // load jinja template from file - t, err := gonja.FromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %w", err) - } - - // execute/render jinja template - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %w", err) - } - outputs[path] = b.Bytes() - } - - return outputs, nil -} diff --git a/pkg/generator/warewulf.go b/pkg/generator/warewulf.go deleted file mode 100644 index 5a7c803..0000000 --- a/pkg/generator/warewulf.go +++ /dev/null @@ -1,78 +0,0 @@ -package generator - -import ( - "fmt" - "maps" - "strings" - - "git.towk2.me/towk/configurator/pkg/client" - "git.towk2.me/towk/configurator/pkg/config" - "git.towk2.me/towk/configurator/pkg/util" - "github.com/rs/zerolog/log" -) - -type Warewulf struct{} - -func (g *Warewulf) GetName() string { - return "warewulf" -} - -func (g *Warewulf) GetVersion() string { - return util.GitCommit() -} - -func (g *Warewulf) GetDescription() string { - return "Configurator generator plugin for 'warewulf' config files." -} - -func (g *Warewulf) Generate(config *config.Config, params Params) (FileMap, error) { - var ( - smdClient = client.NewSmdClient(params.ClientOpts...) - outputs = make(FileMap, len(params.Templates)) - nodeEntries = "" - paths = []string{} - ) - - // if we have a client, try making the request for the ethernet interfaces - eths, err := smdClient.FetchEthernetInterfaces(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) - } - - // check if we have the required params first - if eths == nil { - return nil, fmt.Errorf("invalid ethernet interfaces (variable is nil)") - } - if len(eths) <= 0 { - return nil, fmt.Errorf("no ethernet interfaces found") - } - - // fetch redfish endpoints and handle errors - eps, err := smdClient.FetchRedfishEndpoints(params.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to fetch redfish endpoints: %v", err) - } - if len(eps) <= 0 { - return nil, fmt.Errorf("no redfish endpoints found") - } - - templates, err := ApplyTemplates(Mappings{ - "node_entries": nodeEntries, - }, params.Templates) - if err != nil { - return nil, fmt.Errorf("failed to load templates: %v", err) - } - - maps.Copy(outputs, params.Files) - maps.Copy(outputs, templates) - - // print message if verbose param is found - if params.Verbose { - for path, _ := range outputs { - paths = append(paths, path) - } - } - log.Info().Str("paths", strings.Join(paths, ":")).Msg("templates and files loaded: \n") - - return outputs, err -} diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..90ffa8d --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,119 @@ +package log + +import ( + "fmt" + "io" + "os" + "slices" + "strings" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +// string representation that directly corresponds to zerolog.Level +type LogLevel string +type LogLevelList []LogLevel +type LogFilter string + +const ( + DEBUG LogLevel = "debug" + INFO LogLevel = "info" + WARN LogLevel = "warn" + ERROR LogLevel = "error" + DISABLED LogLevel = "disabled" + TRACE LogLevel = "trace" +) + +var Levels = [6]LogLevel{DEBUG, INFO, WARN, ERROR, DISABLED, TRACE} +var LogFile *os.File + +func (ll LogLevel) String() string { + return string(ll) +} + +func (ll *LogLevel) Set(v string) error { + switch LogLevel(v) { + case DEBUG, INFO, WARN, ERROR, DISABLED, TRACE: + *ll = LogLevel(v) + return nil + default: + return fmt.Errorf("must be one of %v", []LogLevel{ + DEBUG, + INFO, + WARN, + ERROR, + DISABLED, + TRACE, + }) + } +} + +func (df LogLevel) Type() string { + return "LogLevel" +} + +func InitWithLogLevel(logLevel LogLevel, logPath string) error { + var ( + logger zerolog.Logger + level zerolog.Level + writer zerolog.LevelWriter + writers []io.Writer + err error + ) + + // set the logging level + level, err = strToLogLevel(logLevel) + if err != nil { + return fmt.Errorf("failed to convert log level: %v", err) + } + + // add the default stderr writer + writers = append(writers, &zerolog.FilteredLevelWriter{ + Writer: &zerolog.LevelWriterAdapter{os.Stderr}, + Level: level, + }) + + // add another writer to write to a log file + if logPath != "" { + LogFile, err = os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664) + if err != nil { + return fmt.Errorf("failed to open log file: %v", err) + } + + // add another write to write to the specified log file + writers = append(writers, &zerolog.FilteredLevelWriter{ + Writer: zerolog.LevelWriterAdapter{LogFile}, + Level: level, + }) + } + writer = zerolog.MultiLevelWriter(writers...) + logger = zerolog.New(writer).Level(level).With().Timestamp().Caller().Logger() + zerolog.SetGlobalLevel(level) + log.Logger = logger + return nil +} + +func strToLogLevel(ll LogLevel) (zerolog.Level, error) { + var tostr = func(lls []LogLevel) []string { + s := []string{} + for _, l := range lls { + s = append(s, string(l)) + } + return s + } + + if index := slices.Index(Levels[:], ll); index >= 0 { + // handle special cases to map index to DISABLED and TRACE + switch index { + case 4: + return zerolog.Disabled, nil + case 5: + return zerolog.TraceLevel, nil + } + return zerolog.Level(index), nil + } + return -100, fmt.Errorf( + "invalid log level (options: %s)", strings.Join(tostr(Levels[:]), ", "), + ) // use 'info' by default +} diff --git a/pkg/models.go b/pkg/models.go new file mode 100644 index 0000000..ff2e823 --- /dev/null +++ b/pkg/models.go @@ -0,0 +1,52 @@ +package makeshift + +import ( + "git.towk2.me/towk/makeshift/pkg/storage" +) + +type ProfileMap map[string]*Profile +type Profile struct { + ID string `json:"id"` // profile ID + Description string `json:"description,omitempty"` // profile description + Tags []string `json:"tags,omitempty"` // tags used for ... + Data map[string]any `json:"data,omitempty"` // include render data +} + +type Plugin interface { + Name() string + Version() string + Description() string + Metadata() Metadata + + Init() error + Run(data storage.KVStore, args []string) error + Cleanup() error +} +type Metadata map[string]any +type Hook struct { + Data storage.KVStore + Args []string + Plugin Plugin +} + +func (h *Hook) Init() error { + return h.Plugin.Init() +} + +func (h *Hook) Run() error { + return h.Plugin.Run(h.Data, h.Args) +} + +func (h *Hook) Cleanup() error { + return h.Plugin.Cleanup() +} + +func PluginToMap(p Plugin) map[string]any { + return map[string]any{ + "name": p.Name(), + "version": p.Version(), + "description": p.Description(), + "metadata": p.Metadata(), + } + +} diff --git a/pkg/plugin.go b/pkg/plugin.go deleted file mode 100644 index 602cfce..0000000 --- a/pkg/plugin.go +++ /dev/null @@ -1,14 +0,0 @@ -package configurator - -type Plugin interface { - // plugin data - Name() string - Version() string - Description() string - Metadata() map[string]string - - // run the plugin - Init() error - Run() error - Cleanup() error -} diff --git a/pkg/plugins/jinja2/jinja2.go b/pkg/plugins/jinja2/jinja2.go new file mode 100644 index 0000000..daf6b20 --- /dev/null +++ b/pkg/plugins/jinja2/jinja2.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/storage" + "github.com/nikolalohinski/gonja/v2" + "github.com/nikolalohinski/gonja/v2/exec" + "github.com/rs/zerolog/log" +) + +type Jinja2 struct{} + +func (p *Jinja2) Name() string { return "jinja2" } +func (p *Jinja2) Version() string { return "v0.0.1-alpha" } +func (p *Jinja2) Description() string { return "Renders Jinja 2 templates" } +func (p *Jinja2) Metadata() makeshift.Metadata { + return makeshift.Metadata{ + "author": map[string]any{ + "name": "David J. Allen", + "email": "davidallendj@gmail.com", + "links": []string{ + "https://github.com/davidallendj", + "https://git.towk2.me/towk", + }, + }, + } +} + +func (p *Jinja2) Init() error { + // nothing to initialize + log.Debug().Str("plugin", p.Name()).Msg("jinja2.Init()") + return nil +} + +func (p *Jinja2) Run(store storage.KVStore, args []string) error { + // render the files using Jinja 2 from args + var ( + mappings struct { + Data map[string]any `json:"data"` + } + context *exec.Context + template *exec.Template + profiles any // makeshift.ProfileMap + input any // []byte + output bytes.Buffer + err error + ) + log.Debug(). + Str("plugin", p.Name()). + Any("store", store). + Strs("args", args). + Int("arg_count", len(args)). + Msg("(jinja2) Run()") + + profiles, err = store.Get("profiles") + if err != nil { + return fmt.Errorf("(jinja2) failed to get profiles: %v", err) + } + + input, err = store.Get("file") + if err != nil { + return fmt.Errorf("(jinja2) failed to get input data: %v", err) + } + + // get the templates provided as args to the plugin + template, err = gonja.FromBytes(input.([]byte)) + if err != nil { + return fmt.Errorf("(jinja2) failed to get template from args: %v", err) + } + + // get mappings from shared data (optional) + shared, err := store.Get("shared") + if err != nil { + log.Warn().Err(err).Msg("(jinja2) could not retrieve shared data") + } else { + err = json.Unmarshal(shared.([]byte), &mappings) + if err != nil { + return fmt.Errorf("(jinja2) failed to unmarshal mappings from shared data: %v", err) + } + } + + var ps = make(map[string]any) + for profileID, profile := range profiles.(makeshift.ProfileMap) { + ps[profileID] = map[string]any{ + "id": profile.ID, + "description": profile.Description, + "data": profile.Data, + } + } + + // inject profiles and plugin-specific mapping + mappings.Data = map[string]any{ + "makeshift": map[string]any{ + "profiles": ps, + "plugin": map[string]any{ + "name": p.Name(), + "version": p.Version(), + "description": p.Description(), + "metadata": p.Metadata(), + }, + }, + } + + log.Debug().Any("mappings", mappings).Send() + + // use the provided data in the store to render templates + // NOTE: this may be changed to specifically use "shared" data instead + context = exec.NewContext(mappings.Data) + if err = template.Execute(&output, context); err != nil { // Prints: Hello Bob! + return fmt.Errorf("(jinja2) failed to render template: %v", err) + } + + // write render templates to data store output + store.Set("out", output.Bytes()) + return nil +} + +func (p *Jinja2) Cleanup() error { + // nothing to clean up + log.Debug().Str("plugin", p.Name()).Msg("(jinja2) Cleanup()") + return nil +} + +var Makeshift Jinja2 diff --git a/pkg/plugins/mapper/mapper.go b/pkg/plugins/mapper/mapper.go new file mode 100644 index 0000000..1fded1f --- /dev/null +++ b/pkg/plugins/mapper/mapper.go @@ -0,0 +1,30 @@ +package main + +import "git.towk2.me/towk/makeshift/pkg/storage" + +type Mapper struct{} + +func (p *Mapper) Name() string { return "jinja2" } +func (p *Mapper) Version() string { return "test" } +func (p *Mapper) Description() string { return "Renders Jinja 2 templates" } +func (p *Mapper) Metadata() map[string]string { + return map[string]string{ + "author.name": "David J. Allen", + "author.email": "davidallendj@gmail.com", + } +} + +func (p *Mapper) Init() error { + // nothing to initialize + return nil +} + +func (p *Mapper) Run(data storage.KVStore, args []string) error { + return nil +} + +func (p *Mapper) Clean() error { + return nil +} + +var Makeshift Mapper diff --git a/pkg/plugins/smd/smd.go b/pkg/plugins/smd/smd.go new file mode 100644 index 0000000..0d177be --- /dev/null +++ b/pkg/plugins/smd/smd.go @@ -0,0 +1,279 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/storage" + "github.com/rs/zerolog/log" +) + +// An struct that's meant to extend functionality of the base HTTP client by +// adding commonly made requests to SMD. The implemented functions are can be +// used in generator plugins to fetch data when it is needed to substitute +// values for the Jinja templates used. +type SmdClient struct { + http.Client `json:"-" yaml:"-"` + Host string `yaml:"host"` + Port int `yaml:"port"` + AccessToken string `yaml:"access-token"` + + RedfishEndpoints []RedfishEndpoint `json:"redfish_endpoints"` + EthernetInterfaces []EthernetInterface `json:"ethernet_interfaces"` + Components []Component `json:"components"` +} + +type IPAddr struct { + IpAddress string `json:"IPAddress"` + Network string `json:"Network"` +} + +type EthernetInterface struct { + ID string `json:"ID"` + Description string `json:"Description"` + MacAddress string `json:"MACAddr"` + LastUpdate string `json:"LastUpdate"` + ComponentID string `json:"ComponentID"` + Type string `json:"Type"` + IpAddresses []IPAddr `json:"IPAddresses"` +} + +type Component struct { + ID string `json:"ID"` + Type string `json:"Type"` + State string `json:"State,omitempty"` + Flag string `json:"Flag,omitempty"` + Enabled *bool `json:"Enabled,omitempty"` + SwStatus string `json:"SoftwareStatus,omitempty"` + Role string `json:"Role,omitempty"` + SubRole string `json:"SubRole,omitempty"` + NID json.Number `json:"NID,omitempty"` + Subtype string `json:"Subtype,omitempty"` + NetType string `json:"NetType,omitempty"` + Arch string `json:"Arch,omitempty"` + Class string `json:"Class,omitempty"` + ReservationDisabled bool `json:"ReservationDisabled,omitempty"` + Locked bool `json:"Locked,omitempty"` +} + +type RedfishEndpoint struct { + ID string `json:"ID"` + Type string `json:"Type"` + Name string `json:"Name,omitempty"` // user supplied descriptive name + Hostname string `json:"Hostname"` + Domain string `json:"Domain"` + FQDN string `json:"FQDN"` + Enabled bool `json:"Enabled"` + UUID string `json:"UUID,omitempty"` + User string `json:"User"` + Password string `json:"Password"` // Temporary until more secure method + UseSSDP bool `json:"UseSSDP,omitempty"` + MACRequired bool `json:"MACRequired,omitempty"` + MACAddr string `json:"MACAddr,omitempty"` + IPAddr string `json:"IPAddress,omitempty"` +} + +func (p *SmdClient) Name() string { return "smd" } +func (p *SmdClient) Version() string { return "v0.0.1-alpha" } +func (p *SmdClient) Description() string { return "Fetchs data from SMD and writes to store" } +func (p *SmdClient) Metadata() makeshift.Metadata { + return makeshift.Metadata{ + "author": map[string]any{ + "name": "David J. Allen", + "email": "davidallendj@gmail.com", + "links": []string{ + "https://github.com/davidallendj", + "https://git.towk2.me/towk", + }, + }, + } +} + +func (p *SmdClient) Init() error { + log.Debug().Str("plugin", p.Name()).Msg("(smd) Init()") + return nil +} + +func (p *SmdClient) Run(store storage.KVStore, args []string) error { + // set all the defaults for variables + var ( + client SmdClient + bytes []byte + err error + ) + + log.Debug(). + Str("plugin", p.Name()). + Strs("args", args). + Int("arg_count", len(args)). + Any("store", store). + Msg("(smd) Run()") + + // if we have a client, try making the request for the ethernet interfaces + err = client.FetchEthernetInterfaces() + if err != nil { + return fmt.Errorf("(smd) failed to fetch ethernet interfaces with client: %v", err) + } + + err = client.FetchRedfishEndpoints() + if err != nil { + return fmt.Errorf("(smd) failed to fetch redfish endpoints with client: %v", err) + } + err = client.FetchComponents() + if err != nil { + return fmt.Errorf("(smd) failed to fetch components with client: %v", err) + } + + // write data back to shared data store to be used by other plugins + bytes, err = json.Marshal(client) + if err != nil { + return fmt.Errorf("(smd) failed to marshal SMD client: %v", err) + } + store.Set("shared", bytes) + + // apply template substitutions and return output as byte array + return nil +} + +func (p *SmdClient) Cleanup() error { + log.Debug().Str("plugin", p.Name()).Msg("(smd) Init()") + return nil +} + +// Fetch the ethernet interfaces from SMD service using its API. An access token may be required if the SMD +// service SMD_JWKS_URL envirnoment variable is set. +func (client *SmdClient) FetchEthernetInterfaces() error { + var ( + bytes []byte + err error + ) + // make request to SMD endpoint + bytes, err = client.makeRequest("/Inventory/EthernetInterfaces") + if err != nil { + return fmt.Errorf("failed to read HTTP response: %v", err) + } + + // unmarshal response body JSON and extract in object + err = json.Unmarshal(bytes, &client.EthernetInterfaces) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + + // print what we got if verbose is set + log.Debug().Str("ethernet_interfaces", string(bytes)).Msg("found interfaces") + + return nil +} + +// Fetch the components from SMD using its API. An access token may be required if the SMD +// service SMD_JWKS_URL envirnoment variable is set. +func (client *SmdClient) FetchComponents() error { + var ( + bytes []byte + err error + ) + // make request to SMD endpoint + bytes, err = client.makeRequest("/State/Components") + if err != nil { + return fmt.Errorf("failed to make HTTP request: %v", err) + } + + // make sure our response is actually JSON first + if !json.Valid(bytes) { + return fmt.Errorf("expected valid JSON response: %v", string(bytes)) + } + + // unmarshal response body JSON and extract in object + var tmp map[string]any + err = json.Unmarshal(bytes, &tmp) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + bytes, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) + if err != nil { + return fmt.Errorf("failed to marshal JSON: %v", err) + } + err = json.Unmarshal(bytes, &client.Components) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + + // print what we got if verbose is set + log.Debug().Str("components", string(bytes)).Msg("found components") + + return nil +} + +// TODO: improve implementation of this function +func (client *SmdClient) FetchRedfishEndpoints() error { + var ( + store map[string]any + rfeps []RedfishEndpoint + body []byte + err error + ) + + // make initial request to get JSON with 'RedfishEndpoints' as property + body, err = client.makeRequest("/Inventory/RedfishEndpoints") + if err != nil { + return fmt.Errorf("failed to make HTTP resquest: %v", err) + } + // make sure response is in JSON + if !json.Valid(body) { + return fmt.Errorf("expected valid JSON response: %s", string(body)) + } + err = json.Unmarshal(body, &store) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + + // marshal RedfishEndpoint JSON back to makeshift.RedfishEndpoint + body, err = json.Marshal(store["RedfishEndpoints"].([]any)) + if err != nil { + return fmt.Errorf("failed to marshal JSON: %v", err) + } + err = json.Unmarshal(body, &rfeps) + if err != nil { + return fmt.Errorf("failed to unmarshal response: %v", err) + } + + // show the final result + log.Debug().Bytes("redfish_endpoints", body).Msg("found redfish endpoints") + + client.RedfishEndpoints = rfeps + return nil +} + +func (client *SmdClient) makeRequest(endpoint string) ([]byte, error) { + if client == nil { + return nil, fmt.Errorf("client is nil") + } + + // fetch DHCP related information from SMD's endpoint: + url := fmt.Sprintf("%s/hsm/v2%s", client.Host, endpoint) + req, err := http.NewRequest(http.MethodGet, url, bytes.NewBuffer([]byte{})) + if err != nil { + return nil, fmt.Errorf("failed to create new HTTP request: %v", err) + } + + // include access token in authorzation header if found + // NOTE: This shouldn't be needed for this endpoint since it's public + if client.AccessToken != "" { + req.Header.Add("Authorization", "Bearer "+client.AccessToken) + } + + // make the request to SMD + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to make request: %v", err) + } + + // read the contents of the response body + return io.ReadAll(res.Body) +} + +var Makeshift SmdClient diff --git a/pkg/service/constants.go b/pkg/service/constants.go new file mode 100644 index 0000000..68c2953 --- /dev/null +++ b/pkg/service/constants.go @@ -0,0 +1,77 @@ +package service + +const ( + RELPATH_PLUGINS = "/plugins" + RELPATH_PROFILES = "/profiles" + RELPATH_DATA = "/data" + RELPATH_METADATA = "/.makeshift" + RELPATH_HELP = RELPATH_DATA + "/index.html" + RELPATH_PROFILE = RELPATH_PROFILES + "/default.json" + + PATH_CONFIG = "$HOME/.config/makeshift/config.yaml" + + DEFAULT_TIMEOUT_IN_SECS = 60 + DEFAULT_PLUGINS_MAX_COUNT = 32 + DEFAULT_PROFILES_MAX_COUNT = 256 + + FILE_METADATA = `` + FILE_HOME_PAGE = ` + + + +

+ Plugin Information: + Name: {{ makeshift.plugin.name }} + Version: {{ makeshift.plugin.version }} + Description: {{ makeshift.plugin.description }} + Author: {{ makeshift.plugin.metadata.name }} ({{ makeshift.plugin.metadata.email }}) + + Profile Information: + ID: {{ makeshift.profiles.default.id }} + Description: {{ makeshift.profiles.default.description }} + + # setup environment variables
+ export MAKESHIFT_HOST={{ makeshift.profiles.default.data.host }}
+ export MAKESHIFT_PATH={{ makeshift.profiles.default.data.path }}
+ export MAKESHIFT_ROOT={{ makeshift.profiles.default.data.server_root }}

+ + # start the service
+ makeshift serve --root ./tests --init -l debug

+ + # download a file or directory (as archive)
+ makeshift download
+ makeshift download --host http://localhost:5050 --path help.txt

+ + # download files with rendering using plugins
+ makeshift download --plugins smd,jinja2 --profile compute
+ makeshift download -p templates --plugins jinja --profile io
+ curl $MAKESHIFT_HOST/download/help.txt?plugins=smd,jinja2

+ + # upload a file or directory (recursively)
+ makeshift upload
+ makeshift upload --host http://localhost:5050 --path help.txt

+ + # list the files in a directory
+ makeshift list --path help.txt
+ makeshift list --host http://localhost:5050 --path help.txt
+ curl http://localhost:5050/list/test
+

+ + +` + FILE_DEFAULT_PROFILE = ` +{ + "id": "default", + "description": "Makeshift default profile", + "data": { + "host": "localhost", + "path": "/test", + "server_root": "./test" + } +} +` +) + +// makeshift.host: https://localhost:5050 +// makeshift.path: test +// makeshift.server.root: $HOME/apps/makeshift diff --git a/pkg/service/plugins.go b/pkg/service/plugins.go new file mode 100644 index 0000000..29d2f16 --- /dev/null +++ b/pkg/service/plugins.go @@ -0,0 +1,141 @@ +package service + +import ( + "encoding/json" + "io" + "net/http" + "os" + + makeshift "git.towk2.me/towk/makeshift/pkg" + "github.com/go-chi/chi/v5" + "github.com/rs/zerolog/log" +) + +func (s *Service) ListPlugins() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + plugins map[string]makeshift.Plugin + names []string + body []byte + err error + ) + + plugins, err = LoadPluginsFromDir(s.PathForPlugins()) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + for name := range plugins { + names = append(names, name) + } + + body, err = json.Marshal(names) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(body) + } +} + +func (s *Service) GetPluginInfo() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + pluginName = chi.URLParam(r, "name") + path = s.PathForPluginWithName(pluginName) + plugin makeshift.Plugin + body []byte + err error + ) + + plugin, err = LoadPluginFromFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + body, err = json.Marshal(makeshift.PluginToMap(plugin)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(body) + } +} + +func (s *Service) GetPluginRaw() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + pluginName = chi.URLParam(r, "name") + path = s.PathForPluginWithName(pluginName) + rawPlugin []byte + err error + ) + + rawPlugin, err = os.ReadFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(rawPlugin) + } +} + +func (s *Service) CreatePlugin() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + pluginName = chi.URLParam(r, "name") + body []byte + path string + err error + ) + + // helper to check for valid plugin name + var hasValidName = func(name string) bool { + return name != "" && len(name) < 64 + } + + // check for a valid plugin name + if !hasValidName(pluginName) { + http.Error(w, "invalid name for plugin", http.StatusBadRequest) + return + } + + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + path = s.PathForPluginWithName(pluginName) + err = os.WriteFile(path, body, 0o777) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + w.WriteHeader(http.StatusOK) + } +} + +func (s *Service) DeletePlugin() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + pluginName = chi.URLParam(r, "name") + path = s.PathForPluginWithName(pluginName) + err error + ) + + log.Debug().Str("path", path).Send() + err = os.Remove(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + } +} diff --git a/pkg/service/profile.go b/pkg/service/profile.go deleted file mode 100644 index cbfe5c0..0000000 --- a/pkg/service/profile.go +++ /dev/null @@ -1,184 +0,0 @@ -package service - -import ( - "encoding/json" - "fmt" - "io" - "io/fs" - "net/http" - "os" - "path/filepath" - - "github.com/go-chi/chi/v5" -) - -type Profile struct { - ID string `json:"id"` // profile ID - Description string `json:"description"` // profile description - Tags []string `json:"tags"` // tags used for ... - Paths []string `json:"paths"` // paths to download - Plugins []string `json:"plugins"` // plugins to run - Data map[string]any `json:"data"` // include render data -} - -func (s *Service) GetProfiles() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ( - path = s.RootPath + PROFILES_RELPATH - profiles []*Profile - contents []byte - err error - ) - - // walk profiles directory to load all profiles - err = filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - // skip directories - if info.IsDir() { - return nil - } - - // read file contents - var profile *Profile - profile, err = LoadProfile(path) - if err != nil { - return err - } - - profiles = append(profiles, profile) - - fmt.Println(path, info.Size()) - return nil - }) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - // marshal and send all the profiles - contents, err = json.Marshal(profiles) - if err != nil { - http.Error(w, fmt.Sprintf("failed to marshal profiles: %v", err), http.StatusInternalServerError) - } - - _, err = w.Write(contents) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - -// func (s *Service) CreateProfiles() http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) { -// var ( -// path = chi.URLParam(r, "path") -// err error -// ) - -// } -// } - -func (s *Service) GetProfile() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ( - id = chi.URLParam(r, "id") - path = s.BuildProfilePath(id) - profile *Profile - contents []byte - err error - ) - - profile, err = LoadProfile(path) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - contents, err = json.Marshal(profile) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - _, err = w.Write(contents) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - -func (s *Service) CreateProfile() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - type input struct { - path string `json:"path"` - profile *Profile `json:"profile"` - } - var ( - body []byte - in input - err error - ) - - body, err = io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // use the request info to build profile - err = json.Unmarshal(body, &in) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // create a new profile on disk - os.WriteFile() - - w.WriteHeader(http.StatusOK) - } -} - -func (s *Service) CreateProfileVar() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) DeleteProfileVar() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) GetProfileData() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) GetProfileVar() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) CreateProfilePath() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) DeleteProfilePath() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) GetProfilePath() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) GetPlugins() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) CreatePlugins() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} - -func (s *Service) DeletePlugins() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) {} -} diff --git a/pkg/service/profiles.go b/pkg/service/profiles.go new file mode 100644 index 0000000..e951e26 --- /dev/null +++ b/pkg/service/profiles.go @@ -0,0 +1,278 @@ +package service + +import ( + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + + makeshift "git.towk2.me/towk/makeshift/pkg" + "github.com/go-chi/chi/v5" + "github.com/tidwall/sjson" +) + +func (s *Service) ListProfiles() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + path = s.RootPath + RELPATH_PROFILES + profiles []*makeshift.Profile + contents []byte + err error + ) + + // walk profiles directory to load all profiles + err = filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + // skip directories + if info.IsDir() { + return nil + } + + // read file contents + var profile *makeshift.Profile + profile, err = LoadProfileFromFile(path) + if err != nil { + return err + } + + profiles = append(profiles, profile) + + fmt.Println(path, info.Size()) + return nil + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + + // marshal and send all the profiles + contents, err = json.Marshal(profiles) + if err != nil { + http.Error(w, fmt.Sprintf("failed to marshal profiles: %v", err), http.StatusInternalServerError) + } + + _, err = w.Write(contents) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +func (s *Service) GetProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + id = chi.URLParam(r, "id") + path = s.PathForProfileWithID(id) + contents []byte + err error + ) + + contents, err = loadProfileContents(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + + _, err = w.Write(contents) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +func (s *Service) CreateProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + body, contents []byte + path string + profile *makeshift.Profile + err error + ) + + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // use the request info to build profile + err = json.Unmarshal(body, &profile) + if err != nil { + http.Error(w, fmt.Sprintf("failed to unmarshal profile: %v", err.Error()), http.StatusBadRequest) + return + } + + // serialize just the profile part + contents, err = json.Marshal(profile) + if err != nil { + http.Error(w, fmt.Sprintf("failed to marshal profile: %v", err.Error()), http.StatusBadRequest) + return + } + + // create a new profile on disk + path = s.PathForProfileWithID(profile.ID) + err = os.WriteFile(path, contents, os.ModePerm) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + } +} + +func (s *Service) DeleteProfile() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + profileID = chi.URLParam(r, "id") + path string + err error + ) + + if profileID == "default" { + http.Error(w, "cannot delete the default profile", http.StatusBadRequest) + return + } + + path = s.PathForProfileWithID(profileID) + err = os.Remove(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + } +} + +func (s *Service) SetProfileData() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + body, contents []byte + newContents string + profile *makeshift.Profile + path string + err error + ) + + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + err = json.Unmarshal(body, &profile) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // make sure the request data sets an ID + if profile.ID == "" { + http.Error(w, "ID must be set to a non-empty value", http.StatusBadRequest) + return + } + + // read the contents the file with profile ID + path = s.PathForProfileWithID(profile.ID) + contents, err = os.ReadFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // modify the data of the profile's contents + newContents, err = sjson.Set(string(contents), "data", profile.Data) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // write only the data to the file with ID + err = os.WriteFile(path, []byte(newContents), os.ModePerm) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +func (s *Service) DeleteProfileData() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + id = chi.URLParam(r, "id") + path = s.PathForProfileWithID(id) + profile *makeshift.Profile + err error + ) + + // get the profile + profile, err = LoadProfileFromFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + + // delete the profile data + profile.Data = map[string]any{} + + // save the profile back to the file to update + SaveProfileToFile(path, profile) + + } +} + +func (s *Service) GetProfileData() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + id = chi.URLParam(r, "id") + path = s.PathForProfileWithID(id) + profile *makeshift.Profile + body []byte + err error + ) + + profile, err = LoadProfileFromFile(path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // only marshal the profile data and not entire profile + body, err = json.Marshal(profile.Data) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // write body to response + _, err = w.Write(body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +func loadProfileContents(path string) ([]byte, error) { + var ( + contents []byte + profile *makeshift.Profile + err error + ) + profile, err = LoadProfileFromFile(path) + if err != nil { + return nil, fmt.Errorf("failed to load profile from file: %v", err) + } + + contents, err = json.Marshal(profile) + if err != nil { + return nil, fmt.Errorf("failed to marshal profile: %v", err) + } + return contents, nil +} diff --git a/pkg/service/routes.go b/pkg/service/routes.go index f0e55ab..7d371c9 100644 --- a/pkg/service/routes.go +++ b/pkg/service/routes.go @@ -3,36 +3,374 @@ package service import ( "encoding/json" "fmt" + "io" + "io/fs" "net/http" + "os" + "path/filepath" + "strings" + + "git.towk2.me/towk/makeshift/internal/archive" + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/storage" + "github.com/rs/zerolog/log" ) func (s *Service) Download() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var ( + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/download") + pluginNames = strings.Split(r.URL.Query().Get("plugins"), ",") + profileIDs = strings.Split(r.URL.Query().Get("profiles"), ",") + fileInfo os.FileInfo + out *os.File + store *storage.MemoryStorage = new(storage.MemoryStorage) + hooks []makeshift.Hook + contents []byte + errs []error + err error + ) + + // initialize storage + store.Init() + + log.Debug(). + Str("path", path). + Str("client_host", r.Host). + Strs("plugins", pluginNames). + Strs("profiles", profileIDs). + Any("query", r.URL.Query()). + Msg("Service.Download()") + + // prepare profiles + errs = s.loadProfiles(profileIDs, store, errs) + if len(errs) > 0 { + log.Error().Errs("errs", errs).Msg("errors occurred loading profiles") + errs = []error{} + } + + // determine if path is directory, file, or exists + if fileInfo, err = os.Stat(path); err == nil { + if fileInfo.IsDir() { + // get the final archive path + archivePath := fmt.Sprintf("%s.tar.gz", path) + + log.Debug(). + Str("archive_path", archivePath). + Str("type", "directory"). + Msg("Service.Download()") + + out, err = os.Create(archivePath) + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to create named file: %v", err), http.StatusInternalServerError) + return + } + + // get a list of filenames to archive + filenamesToArchive := []string{} + filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + filenamesToArchive = append(filenamesToArchive, path) + } + return nil + }) + log.Debug().Strs("files", filenamesToArchive).Send() + + // prepare plugins + hooks, errs = s.loadPlugins(pluginNames, store, nil, errs) + if len(errs) > 0 { + log.Error().Errs("errs", errs).Msg("errors occurred loading plugins") + errs = []error{} + } + + // create an archive of the directory, run hooks, and download + err = archive.Create(filenamesToArchive, out, hooks) + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to create archive: %v", err.Error()), http.StatusInternalServerError) + return + } + + // load the final archive + contents, err = os.ReadFile(archivePath) + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to read archive contents: %v", err.Error()), http.StatusInternalServerError) + return + } + + // send the archive back as response + w.Header().Add("FILETYPE", "archive") + w.Write(contents) + + // clean up the temporary archive + err = os.Remove(archivePath) + if err != nil { + log.Error().Err(err).Msg("failed to remove temporary archive") + return + } + + } else { + // download individual file + log.Debug(). + Str("type", "file"). + Msg("Service.Download()") + + contents, err = os.ReadFile(path) + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to read file to download: %v", err), http.StatusInternalServerError) + return + } + + // prepare plugins + + store.Set("file", contents) + hooks, errs = s.loadPlugins(pluginNames, store, nil, errs) + if len(errs) > 0 { + log.Error().Errs("errs", errs).Msg("errors occurred loading plugins") + errs = []error{} + } + if len(hooks) > 0 { + // run pre-hooks to modify the contents of the file before archiving + log.Debug().Int("hook_count", len(hooks)).Msg("running hooks") + for _, hook := range hooks { + log.Debug().Any("hook", map[string]any{ + "store": hook.Data, + "args": hook.Args, + "plugin": map[string]string{ + "name": hook.Plugin.Name(), + "description": hook.Plugin.Description(), + "version": hook.Plugin.Version(), + }, + }).Send() + err = hook.Init() + if err != nil { + log.Error(). + Err(err). + Str("plugin", hook.Plugin.Name()). + Msg("failed to initialize plugin") + continue + } + err = hook.Run() + if err != nil { + log.Error(). + Err(err). + Str("plugin", hook.Plugin.Name()). + Msg("failed to run plugin") + continue + } + err = hook.Cleanup() + if err != nil { + log.Error(). + Err(err). + Str("plugin", hook.Plugin.Name()). + Msg("failed to cleanup plugin") + continue + } + } + + // take the contents from the last hook and update files + var ( + hook = hooks[len(hooks)-1] + data any + ) + data, err = hook.Data.Get("out") + if err != nil { + s.writeErrorResponse(w, fmt.Sprintf("failed to get data from hook: %v", err), http.StatusInternalServerError) + return + } + + // send processed (with plugins) file back as response + w.Header().Add("FILETYPE", "file") + w.Write(data.([]byte)) + } else { + // send non-processed file back as response + w.Header().Add("FILETYPE", "file") + w.Write(contents) + } + + } + } else { + s.writeErrorResponse(w, err.Error(), http.StatusBadRequest) + return + } } } func (s *Service) Upload() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var ( + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/upload") + body []byte + dirpath string + err error + ) + // show what we're uploading + log.Debug(). + Str("path", path). + Msg("Service.Upload()") + + // take the provided path and store the file contents + dirpath = filepath.Dir(path) + err = os.MkdirAll(dirpath, 0o777) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // write file to disk + body, err = io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + err = os.WriteFile(path, body, 0o777) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) } } func (s *Service) List() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var ( + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/list") + entries []string + body []byte + err error + ) + // show what we're listing + log.Debug().Str("path", path).Msg("Service.List()") + + // walk directory and show all entries "ls" + err = filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + entries = append(entries, d.Name()) + return nil + }) + if err != nil { + switch err { + case fs.ErrNotExist, fs.ErrInvalid: + http.Error(w, "No such file or directory...", http.StatusBadRequest) + case fs.ErrPermission: + http.Error(w, "Invalid permissions...", http.StatusForbidden) + default: + http.Error(w, "Something went wrong (file or directory *probably* does not exist)...", http.StatusInternalServerError) + } + return + } + + body, err = json.Marshal(entries) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(body) + } +} + +func (s *Service) Delete() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + path = s.PathForData() + strings.TrimPrefix(r.URL.Path, "/delete") + err error + ) + + err = os.RemoveAll(path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) } } func (s *Service) GetStatus(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - data := map[string]any{ - "code": 200, - "message": "Configurator is healthy", - } - err := json.NewEncoder(w).Encode(data) + err := json.NewEncoder(w).Encode(map[string]any{ + "code": http.StatusOK, + "message": "The makeshift server is healthy", + }) if err != nil { - fmt.Printf("failed to encode JSON: %v\n", err) + fmt.Printf("failed to encode JSON response body: %v\n", err) return } } + +func (s *Service) loadProfiles(profileIDs []string, store storage.KVStore, errs []error) []error { + // load data from profiles into the data store + var profiles = make(makeshift.ProfileMap, len(profileIDs)) + for i, profileID := range profileIDs { + var ( + profilePath = s.PathForProfileWithID(profileID) + profile *makeshift.Profile + err error + ) + if i > s.ProfilesMaxCount { + log.Warn().Msg("max profiles count reached...stopping") + return errs + } + if profileID == "" { + log.Warn().Msg("profile ID is empty...skipping") + continue + } + log.Debug(). + Str("id", profileID). + Str("path", profilePath). + Msg("load profile") + profile, err = LoadProfileFromFile(profilePath) + if err != nil { + errs = append(errs, err) + continue + } + profiles[profileID] = profile + } + store.Set("profiles", profiles) + + return errs +} + +func (s *Service) loadPlugins(pluginNames []string, store storage.KVStore, args []string, errs []error) ([]makeshift.Hook, []error) { + // create hooks to run from provided plugins specified + var hooks []makeshift.Hook + for i, pluginName := range pluginNames { + var ( + pluginPath string = s.PathForPluginWithName(pluginName) + plugin makeshift.Plugin + err error + ) + if i > s.PluginsMaxCount { + log.Warn().Msg("max plugins count reached or exceeded...stopping") + return hooks, errs + } + if pluginName == "" { + log.Warn().Msgf("no plugin name found with index %d...skipping", i) + continue + } + log.Debug(). + Str("name", pluginName). + Str("path", pluginPath). + Msg("load plugin") + + // load the plugin from disk + plugin, err = LoadPluginFromFile(pluginPath) + if err != nil { + errs = append(errs, err) + continue + } + hooks = append(hooks, makeshift.Hook{ + Data: store, + Args: args, + Plugin: plugin, + }) + } + return hooks, errs +} diff --git a/pkg/service/service.go b/pkg/service/service.go index 6c3467d..7197876 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -3,43 +3,83 @@ package service import ( "encoding/json" "fmt" + "io/fs" + "net/http" "os" + "path/filepath" + "plugin" + "slices" "time" - configurator "git.towk2.me/towk/configurator/pkg" - "github.com/go-chi/chi/middleware" + makeshift "git.towk2.me/towk/makeshift/pkg" + "git.towk2.me/towk/makeshift/pkg/util" "github.com/go-chi/chi/v5" -) - -const ( - PLUGINS_RELPATH = "/plugins" - TEMPLATES_RELPATH = "/templates" - PROFILES_RELPATH = "/profiles" + "github.com/go-chi/chi/v5/middleware" + "github.com/rs/zerolog/log" ) type Service struct { - RootPath string `yaml:"root,omitempty"` - Environment map[string]string + Addr string + RootPath string `yaml:"root,omitempty"` + CACertFile string `yaml:"cacert,omitempty"` + CACertKeyfile string `yaml:"keyfile,omitempty"` // max counts PluginsMaxCount int ProfilesMaxCount int + Timeout time.Duration } -// New creates the directories at specified path +// New creates a new Service instance with default values func New() *Service { return &Service{ - RootPath: ".", - Environment: map[string]string{ - "CONFIGURATOR_HOST_URI": "", - "ACCESS_TOKEN": "", - }, - PluginsMaxCount: 64, - ProfilesMaxCount: 256, + Addr: ":5050", + RootPath: "./", + PluginsMaxCount: DEFAULT_PLUGINS_MAX_COUNT, + ProfilesMaxCount: DEFAULT_PROFILES_MAX_COUNT, + Timeout: DEFAULT_TIMEOUT_IN_SECS, } } -// Serve() starts the configurator service and waits for requests. +// Init() sets up the default files and directories for the service +func (s *Service) Init() error { + // create the default directories + var err error + err = os.MkdirAll(s.RootPath, 0o777) + if err != nil { + return fmt.Errorf("failed to make service root path: %v", err) + } + err = os.MkdirAll(s.PathForPlugins(), 0o777) + if err != nil { + return fmt.Errorf("failed to make service plugin path: %v", err) + } + err = os.MkdirAll(s.PathForProfiles(), 0o777) + if err != nil { + return fmt.Errorf("failed to make service profile path: %v", err) + } + err = os.MkdirAll(s.PathForData(), 0o777) + if err != nil { + return fmt.Errorf("failed to make service data path: %v", err) + } + + // create the default files + err = os.WriteFile(s.PathForMetadata(), []byte(FILE_METADATA), 0o777) + if err != nil { + return fmt.Errorf("failed to make service metadata file: %v", err) + } + err = os.WriteFile(s.PathForHome(), []byte(FILE_HOME_PAGE), 0o777) + if err != nil { + return fmt.Errorf("failed to make service home page file: %v", err) + } + err = os.WriteFile(s.PathForProfileWithID("default"), []byte(FILE_DEFAULT_PROFILE), 0o777) + if err != nil { + return fmt.Errorf("failed to make service default profile file: %v", err) + } + + return nil +} + +// Serve() starts the makeshift service and waits for requests. func (s *Service) Serve() error { router := chi.NewRouter() router.Use(middleware.RequestID) @@ -47,38 +87,41 @@ func (s *Service) Serve() error { router.Use(middleware.Logger) router.Use(middleware.Recoverer) router.Use(middleware.StripSlashes) - router.Use(middleware.Timeout(60 * time.Second)) + router.Use(middleware.Timeout(s.Timeout * time.Second)) if s.requireAuth() { } else { // general - router.Get("/download", s.Download()) - router.Post("/upload", s.Upload()) - router.Get("/list", s.List()) + router.Get("/download/*", s.Download()) + router.Post("/upload/*", s.Upload()) + router.Get("/list/*", s.List()) + router.Delete("/delete/*", s.Delete()) // profiles - router.Get("/profiles", s.GetProfiles()) - // router.Post("/profiles", s.CreateProfiles()) - router.Get("/profile/{id}", s.GetProfile()) - router.Post("/profile/{id}", s.CreateProfile()) - router.Post("/profile/{id}/data/{varname}", s.CreateProfileVar()) - router.Delete("/profile/{id}/data/{varname}", s.DeleteProfileVar()) - router.Get("/profile/{id}/data", s.GetProfileData()) - router.Get("/profile/{id}/data/{varname}", s.GetProfileVar()) - router.Post("/profile/{id}/paths/{path}", s.CreateProfilePath()) - router.Delete("/profile/{id}/paths/{path}", s.DeleteProfilePath()) - router.Get("/profile/{id}/paths/{path}", s.GetProfilePath()) + router.Get("/profiles", s.ListProfiles()) + router.Get("/profiles/{id}", s.GetProfile()) + router.Post("/profiles/{id}", s.CreateProfile()) + router.Delete("/profiles/{id}", s.DeleteProfile()) + router.Get("/profiles/{id}/data", s.GetProfileData()) + router.Post("/profiles/{id}/data", s.SetProfileData()) + router.Delete("/profiles/{id}/data", s.DeleteProfileData()) // plugins - router.Get("/plugins", s.GetPlugins()) - router.Post("/plugins", s.CreatePlugins()) - router.Delete("/plugins/{id}", s.DeletePlugins()) + router.Get("/plugins", s.ListPlugins()) + router.Get("/plugins/{name}/info", s.GetPluginInfo()) + router.Get("/plugins/{name}/raw", s.GetPluginRaw()) + router.Post("/plugins/{name}", s.CreatePlugin()) + router.Delete("/plugins/{name}", s.DeletePlugin()) } // always available public routes go here router.HandleFunc("/status", s.GetStatus) - return nil + if s.CACertFile != "" && s.CACertKeyfile != "" { + return http.ListenAndServeTLS(s.Addr, s.CACertFile, s.CACertKeyfile, router) + } else { + return http.ListenAndServe(s.Addr, router) + } } func (s *Service) requireAuth() bool { @@ -89,15 +132,96 @@ func (s *Service) FetchJwks(uri string) { } -func LoadProfile(path string) (*Profile, error) { - return LoadFromJSONFile[Profile](path) +func LoadProfileFromFile(path string) (*makeshift.Profile, error) { + return loadFromJSONFile[makeshift.Profile](path) } -func LoadPlugin(path string) (*configurator.Plugin, error) { - return LoadFromJSONFile[configurator.Plugin](path) +// LoadPluginFromFile loads a single plugin given a single file path +func LoadPluginFromFile(path string) (makeshift.Plugin, error) { + var ( + isDir bool + err error + loadedPlugin *plugin.Plugin + ) + // skip loading plugin if path is a directory with no error + isDir, err = util.IsDirectory(path) + if err != nil { + return nil, fmt.Errorf("failed to test if plugin path is directory: %v", err) + } else if isDir { + return nil, fmt.Errorf("path is a directory") + } + + // try and open the plugin + loadedPlugin, err = plugin.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open plugin at path '%s': %v", path, err) + } + + // load the "Target" symbol from plugin + symbol, err := loadedPlugin.Lookup("Makeshift") + if err != nil { + return nil, fmt.Errorf("failed to look up symbol at path '%s': %v", path, err) + } + + // assert that the plugin is a valid makeshift.Plugin + target, ok := symbol.(makeshift.Plugin) + if !ok { + return nil, fmt.Errorf("failed to assert the correct symbol type at path '%s'", path) + } + return target, nil } -func LoadFromJSONFile[T any](path string) (*T, error) { +// LoadPluginsFromDir loads all plugins in a given directory. +// +// Returns a map of plugins. Each plugin can be accessed by the name +// returned by the plugin.GetName() implemented. +func LoadPluginsFromDir(dirpath string) (map[string]makeshift.Plugin, error) { + // check if verbose option is supplied + var ( + cps = make(map[string]makeshift.Plugin) + err error + ) + + // helper to check for valid extensions + var hasValidExt = func(path string) bool { + return slices.Contains([]string{".so", ".dylib", ".dll"}, filepath.Ext(path)) + } + + // walk all files in directory only loading *valid* plugins + err = filepath.Walk(dirpath, func(path string, info fs.FileInfo, err error) error { + // skip trying to load generator plugin if directory or error + // only try loading if file has .so extension + if info.IsDir() || err != nil || !hasValidExt(path) { + return nil + } + + // load the plugin from current path + p, err := LoadPluginFromFile(path) + if err != nil { + return fmt.Errorf("failed to load plugin in directory '%s': %v", path, err) + } + + // map each plugin by name for lookup + cps[p.Name()] = p + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to walk directory '%s': %v", dirpath, err) + } + + return cps, nil +} + +func SaveProfileToFile(path string, profile *makeshift.Profile) error { + return saveToJSONFile(path, profile) +} + +func SavePluginToFile(path string, plugin *makeshift.Plugin) error { + return saveToJSONFile(path, plugin) +} + +func loadFromJSONFile[T any](path string) (*T, error) { var ( res *T contents []byte @@ -106,17 +230,63 @@ func LoadFromJSONFile[T any](path string) (*T, error) { contents, err = os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("failed to read plugin file: %v", err) + return nil, fmt.Errorf("failed to read file: %v", err) } err = json.Unmarshal(contents, &res) if err != nil { - return nil, fmt.Errorf("failed to unmarshal plugin: %v", err) + return nil, fmt.Errorf("failed to unmarshal contents from JSON: %v", err) } return res, err } -func (s *Service) BuildProfilePath(id string) string { - return s.RootPath + PLUGINS_RELPATH + "/" + id +func saveToJSONFile[T any](path string, data T) error { + var ( + contents []byte + err error + ) + contents, err = json.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data to JSON: %v", err) + } + err = os.WriteFile(path, contents, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to write JSON to file: %v", err) + } + + return nil +} + +func (s *Service) PathForProfileWithID(id string) string { + return s.RootPath + RELPATH_PROFILES + "/" + id + ".json" +} + +func (s *Service) PathForPluginWithName(name string) string { + return s.RootPath + RELPATH_PLUGINS + "/" + name + ".so" +} + +func (s *Service) PathForProfiles() string { + return s.RootPath + RELPATH_PROFILES + "/" +} + +func (s *Service) PathForPlugins() string { + return s.RootPath + RELPATH_PLUGINS + "/" +} + +func (s *Service) PathForData() string { + return s.RootPath + RELPATH_DATA +} + +func (s *Service) PathForMetadata() string { + return s.RootPath + RELPATH_METADATA +} + +func (s *Service) PathForHome() string { + return s.RootPath + RELPATH_HELP +} + +func (s *Service) writeErrorResponse(w http.ResponseWriter, message string, code int) { + http.Error(w, message, code) + log.Error().Msg(message) } diff --git a/pkg/storage/disk.go b/pkg/storage/disk.go new file mode 100644 index 0000000..46d3f1b --- /dev/null +++ b/pkg/storage/disk.go @@ -0,0 +1,23 @@ +package storage + +type DiskStorage struct{} + +func (ds DiskStorage) Init() error { + return nil +} + +func (ds DiskStorage) Cleanup() error { + return nil +} + +func (ds DiskStorage) Get(k string) error { + return nil +} + +func (ds DiskStorage) Set(k string, v any) error { + return nil +} + +func (ds DiskStorage) GetData() any { + return nil +} diff --git a/pkg/storage/memory.go b/pkg/storage/memory.go new file mode 100644 index 0000000..e554fbd --- /dev/null +++ b/pkg/storage/memory.go @@ -0,0 +1,33 @@ +package storage + +import "fmt" + +type MemoryStorage struct { + Data map[string]any `json:"data"` +} + +func (ms *MemoryStorage) Init() error { + ms.Data = map[string]any{} + return nil +} + +func (ms *MemoryStorage) Cleanup() error { + return nil +} + +func (ms *MemoryStorage) Get(k string) (any, error) { + v, ok := ms.Data[k] + if ok { + return v, nil + } + return nil, fmt.Errorf("value '%s' does not exist", k) +} + +func (ms *MemoryStorage) Set(k string, v any) error { + ms.Data[k] = v + return nil +} + +func (ms *MemoryStorage) GetData() any { + return ms.Data +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go new file mode 100644 index 0000000..3bc9877 --- /dev/null +++ b/pkg/storage/storage.go @@ -0,0 +1,19 @@ +package storage + +type KVStore interface { + Init() error + Cleanup() error + + Get(k string) (any, error) + Set(k string, v any) error + GetData() any +} + +type KVStaticStore[T any] interface { + Init() error + Cleanup() error + + Get(k string) (T, error) + Set(k string, v T) error + GetData() T +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 6ff13b0..fc53b67 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1,10 +1,8 @@ package util import ( - "archive/tar" "bytes" "cmp" - "compress/gzip" "crypto/tls" "fmt" "io" @@ -99,64 +97,3 @@ func CopyIf[T comparable](s []T, condition func(t T) bool) []T { } return f } - -func CreateArchive(files []string, buf io.Writer) error { - // Create new Writers for gzip and tar - // These writers are chained. Writing to the tar writer will - // write to the gzip writer which in turn will write to - // the "buf" writer - gw := gzip.NewWriter(buf) - defer gw.Close() - tw := tar.NewWriter(gw) - defer tw.Close() - - // Iterate over files and add them to the tar archive - for _, file := range files { - err := addToArchive(tw, file) - if err != nil { - return err - } - } - - return nil -} - -func addToArchive(tw *tar.Writer, filename string) error { - // open file to write to archive - file, err := os.Open(filename) - if err != nil { - return err - } - defer file.Close() - - // get FileInfo for file size, mode, etc. - info, err := file.Stat() - if err != nil { - return err - } - - // create a tar Header from the FileInfo data - header, err := tar.FileInfoHeader(info, info.Name()) - if err != nil { - return err - } - - // use full path as name (FileInfoHeader only takes the basename) to - // preserve directory structure - // see for more info: https://golang.org/src/archive/tar/common.go?#L626 - header.Name = filename - - // Write file header to the tar archive - err = tw.WriteHeader(header) - if err != nil { - return err - } - - // copy file content to tar archive - _, err = io.Copy(tw, file) - if err != nil { - return err - } - - return nil -} diff --git a/tests/01-download-localhost.hurl b/tests/01-download-localhost.hurl new file mode 100644 index 0000000..1c1d0c0 --- /dev/null +++ b/tests/01-download-localhost.hurl @@ -0,0 +1,11 @@ +# download single file +GET http://localhost:5050/downloads + +# download directory as archive +GET + +# download single file using plugins + +# download directory as archive using plugins + + diff --git a/tests/02-upload-localhost.hurl b/tests/02-upload-localhost.hurl new file mode 100644 index 0000000..ff0163a --- /dev/null +++ b/tests/02-upload-localhost.hurl @@ -0,0 +1,8 @@ +# upload a single new file +POST http://localhost:5050/upload + +# upload a new directory + +# upload a new plugin + +# upload a new profile \ No newline at end of file diff --git a/tests/03-list-localhost.hurl b/tests/03-list-localhost.hurl new file mode 100644 index 0000000..bfc2558 --- /dev/null +++ b/tests/03-list-localhost.hurl @@ -0,0 +1,2 @@ +GET http://localhost:5050/list +GET http://localhost:5050/status \ No newline at end of file diff --git a/tests/04-profiles-localhost.hurl b/tests/04-profiles-localhost.hurl new file mode 100644 index 0000000..82786b2 --- /dev/null +++ b/tests/04-profiles-localhost.hurl @@ -0,0 +1,6 @@ +GET http://localhost:5050/profiles +GET http://localhost:5050/profiles/test +POST http://localhost:5050/profiles/test +GET http://localhost:5050/profiles/test/data +POST http://localhost:5050/profiles/test/data +DELETE http://localhost:5050/profiles/test/data diff --git a/tests/05-plugins-localhost.hurl b/tests/05-plugins-localhost.hurl new file mode 100644 index 0000000..e52e383 --- /dev/null +++ b/tests/05-plugins-localhost.hurl @@ -0,0 +1,6 @@ +GET http://localhost:5050/plugins +GET http://localhost:5050/plugin/test +POST http://localhost:5050/plugins/test +DELETE http://localhost:5050/plugin/test + +