From 699ff76e4271f5f750a4609ae588da7d0d36fca8 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 10:36:16 -0600 Subject: [PATCH 001/102] Added Dockerfile and Makefile rule --- Dockerfile | 14 ++++++++++++++ Makefile | 23 ++++++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..97d906e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM cgr.dev/chainguard/wolfi-base + +RUN apk add --no-cache tini bash + +# nobody 65534:65534 +USER 65534:65534 + +# copy the binary and all of the default plugins +COPY configurator /configurator +COPY lib/* /lib/* + +CMD ["/configurator"] + +ENTRYPOINT [ "/sbin/tini", "--" ] \ No newline at end of file diff --git a/Makefile b/Makefile index 0ca212f..49f7065 100644 --- a/Makefile +++ b/Makefile @@ -5,21 +5,26 @@ all: plugins exe test # build the main executable to make configs main: exe driver: exe +binaries: exe exe: go build --tags=all -o configurator + +docker: binaries plugins + docker build -t configurator:latest . + # build all of the generators into plugins plugins: mkdir -p lib - go build -buildmode=plugin -o lib/conman.so internal/generator/plugins/conman/conman.go - go build -buildmode=plugin -o lib/coredhcp.so internal/generator/plugins/coredhcp/coredhcp.go - go build -buildmode=plugin -o lib/dhcpd.so internal/generator/plugins/dhcpd/dhcpd.go - go build -buildmode=plugin -o lib/dnsmasq.so internal/generator/plugins/dnsmasq/dnsmasq.go - go build -buildmode=plugin -o lib/example.so internal/generator/plugins/example/example.go - go build -buildmode=plugin -o lib/hostfile.so internal/generator/plugins/hostfile/hostfile.go - go build -buildmode=plugin -o lib/powerman.so internal/generator/plugins/powerman/powerman.go - go build -buildmode=plugin -o lib/syslog.so internal/generator/plugins/syslog/syslog.go - go build -buildmode=plugin -o lib/warewulf.so internal/generator/plugins/warewulf/warewulf.go + go build -buildmode=plugin -o lib/conman.so pkg/generator/plugins/conman/conman.go + go build -buildmode=plugin -o lib/coredhcp.so pkg/generator/plugins/coredhcp/coredhcp.go + go build -buildmode=plugin -o lib/dhcpd.so pkg/generator/plugins/dhcpd/dhcpd.go + go build -buildmode=plugin -o lib/dnsmasq.so pkg/generator/plugins/dnsmasq/dnsmasq.go + go build -buildmode=plugin -o lib/example.so pkg/generator/plugins/example/example.go + go build -buildmode=plugin -o lib/hostfile.so pkg/generator/plugins/hostfile/hostfile.go + go build -buildmode=plugin -o lib/powerman.so pkg/generator/plugins/powerman/powerman.go + go build -buildmode=plugin -o lib/syslog.so pkg/generator/plugins/syslog/syslog.go + go build -buildmode=plugin -o lib/warewulf.so pkg/generator/plugins/warewulf/warewulf.go # remove executable and all built plugins clean: From 6f027fa7fb7813beffa061efc8ad60de4168d55a Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 10:47:39 -0600 Subject: [PATCH 002/102] Added goreleaser and GitHub workflow --- .github/workflows/main.yml | 39 +++++++++++++++++++++++++++++++++++ .goreleaser.yaml | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100644 .goreleaser.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..0b8ae48 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,39 @@ +name: Release with goreleaser + +on: + workflow_dispatch: + push: + tags: + - v* + +permissions: write-all # Necessary for the generate-build-provenance action with containers + +jobs: + + build: + + + runs-on: ubuntu-latest + + steps: + - name: Set up Go 1.21 + uses: actions/setup-go@v5 + with: + go-version: 1.21 + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-tags: 1 + fetch-depth: 0 + - name: Release with goreleaser + uses: goreleaser/goreleaser-action@v6 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + version: latest + args: release --clean + id: goreleaser + - name: Attest Binaries + uses: actions/attest-build-provenance@v1 + with: + subject-path: '${{ github.workspace }}/dist/configurator_linux_amd64_v1/configurator' \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..d5592d1 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,42 @@ +version: 2 + +before: + hooks: + - go mod download +builds: + - env: + - CGO_ENABLED=1 + goos: + - linux + goarch: + - amd64 +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of uname. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + files: + - LICENSE + - CHANGELOG.md + - README.md + - configurator +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' +release: + github: + name_template: "{{.Version}}" + prerelease: auto + mode: append \ No newline at end of file From dbea108f74f0c011a9e960618313616d15d3146b Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 11:29:19 -0600 Subject: [PATCH 003/102] Updated Dockerfile and Makefile --- Dockerfile | 6 ++++-- Makefile | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 97d906e..b36b37d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,15 @@ FROM cgr.dev/chainguard/wolfi-base RUN apk add --no-cache tini bash +RUN mkdir -p /configurator +RUN mkdir -p /configurator/lib # nobody 65534:65534 USER 65534:65534 # copy the binary and all of the default plugins -COPY configurator /configurator -COPY lib/* /lib/* +COPY configurator /configurator/configurator +COPY lib/* /configurator/lib/* CMD ["/configurator"] diff --git a/Makefile b/Makefile index 49f7065..f58451d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ exe: docker: binaries plugins - docker build -t configurator:latest . + docker build . --build-arg REGISTRY_HOST=${REGISTRY_HOST} --no-cache --pull --tag '${NAME}:${VERSION}' # build all of the generators into plugins plugins: From a0ee615d30cca7962abc96dbbf42551f74c22730 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 11:32:21 -0600 Subject: [PATCH 004/102] Added local Docker container rule for testing locally --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index f58451d..b5eba96 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,9 @@ exe: docker: binaries plugins docker build . --build-arg REGISTRY_HOST=${REGISTRY_HOST} --no-cache --pull --tag '${NAME}:${VERSION}' +docker-testing: binaries plugins + docker build . --tag configurator:testing + # build all of the generators into plugins plugins: mkdir -p lib From 80ade5bf6fa6238bf08095627f7c9977faa13b55 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 1 Aug 2024 11:02:42 -0600 Subject: [PATCH 005/102] Updated README.md to include Docker section --- README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3fe72a3..acfcb36 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,30 @@ curl http://127.0.0.1:3334/generate?target=dnsmasq -H "Authorization: Bearer $AC This will do the same thing as the `generate` subcommand, but remotely. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set. The `ACCESS_TOKEN` environment variable passed to `curl` and it's corresponding CLI argument both expects a token as a JWT. +### Docker + +New images can be built and tested using the `Dockerfile` provided in the project. However, the binary executable and the generator plugins must first be built before building the image since the Docker build copies the binary over. Therefore, build all of the binaries first by following the first section of ["Building and Usage"](#building-and-usage). If you run the `make docker`, this will be done for you. Otherwise, run the `docker build` command after building the executable and libraries. + +```bash +docker build -t configurator:testing path/to/configurator/Dockerfile +# ...or +make docker +``` + +Keep in mind that all plugins included in the project are build in the `lib/` directory and copied from there. If you want to easily include your own external generator plugins, you can build it and copy the `lib.so` file to that location. Make sure that the `Generator` interface is implemented correct as described in the ["Creating Generator Plugins"](#creating-generator-plugins) or the plugin will not load. Additionally, the name string returned from the `GetName()` method is used for looking up the plugin after all plugins have been loaded by the main driver. + +Alternatively, pull the latest existing image/container from the GitHub container repository. + +```bash +docker pull ghcr.io/openchami/configurator:latest +``` + +Then, run the container similarly to the binary. + +``` +docker run ghcr.io/openchami/configurator:latest configurator generate --config config.yaml --target dnsmasq +``` + ### Creating Generator Plugins The `configurator` uses generator plugins to define how config files are generated using a `Generator` interface. The interface is defined like so: @@ -64,7 +88,7 @@ type Generator interface { } ``` -A new plugin can be created by implementing the methods from interface and exporting a symbol with `Generator` as the name and the plugin struct as the type. The `GetName()` function returns the name that is used for looking up the corresponding template set in your config file. It can also be included in the templated files with the default plugins using the `{{ plugin_name }}` in your template. The `GetVersion()` and `GetDescription()` functions returns the version and description of the plugin which can be included in the templated files using `{{ plugin_version }}` and `{{ plugin_description }}` respectively with the default plugins. The `Generate` function is where the magic happens to build the config file from a template. +A new plugin can be created by implementing the methods from interface and exporting a symbol with `Generator` as the name and the plugin struct as the type. The `GetName()` function returns the name that is used for looking up the corresponding target set in your config file. It can also be included in the templated files with the default plugins using the `{{ plugin_name }}` in your template. The `GetVersion()` and `GetDescription()` functions returns the version and description of the plugin which can be included in the templated files using `{{ plugin_version }}` and `{{ plugin_description }}` respectively with the default plugins. The `Generate` function is where the magic happens to build the config file from a template. ```go package main From 73ca17dce6b0b7261e27248be48b79a07ff0130a Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 1 Aug 2024 15:47:24 -0600 Subject: [PATCH 006/102] Updated Makefile with recommended changes --- Makefile | 50 +++++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index b5eba96..5783752 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,51 @@ +# Unless set otherwise, the container runtime is Docker +DOCKER ?= docker + +prog ?= configurator +sources := main.go $(wildcard cmd/*.go) +plugin_source_prefix := pkg/generator/plugins +plugin_sources := $(filter-out %_test.go,$(wildcard $(plugin_source_prefix)/*/*.go)) +plugin_binaries := $(addprefix lib/,$(patsubst %.go,%.so,$(notdir $(plugin_sources)))) # build everything at once +.PHONY: all all: plugins exe test # build the main executable to make configs +.PHONY: main driver binaries exe main: exe driver: exe binaries: exe -exe: - go build --tags=all -o configurator +exe: $(prog) +# build named executable from go sources +$(prog): $(sources) + go build --tags=all -o $(prog) -docker: binaries plugins - docker build . --build-arg REGISTRY_HOST=${REGISTRY_HOST} --no-cache --pull --tag '${NAME}:${VERSION}' +.PHONY: container +container: binaries plugins + $(DOCKER) build . --build-arg --no-cache --pull --tag 'configurator:testing' -docker-testing: binaries plugins - docker build . --tag configurator:testing +.PHONY: container-testing +container-testing: binaries plugins + $(DOCKER) build . --tag configurator:testing # build all of the generators into plugins -plugins: +.PHONY: plugins +plugins: $(plugin_binaries) + +# how to make each plugin +lib/%.so: pkg/generator/plugins/%/*.go mkdir -p lib - go build -buildmode=plugin -o lib/conman.so pkg/generator/plugins/conman/conman.go - go build -buildmode=plugin -o lib/coredhcp.so pkg/generator/plugins/coredhcp/coredhcp.go - go build -buildmode=plugin -o lib/dhcpd.so pkg/generator/plugins/dhcpd/dhcpd.go - go build -buildmode=plugin -o lib/dnsmasq.so pkg/generator/plugins/dnsmasq/dnsmasq.go - go build -buildmode=plugin -o lib/example.so pkg/generator/plugins/example/example.go - go build -buildmode=plugin -o lib/hostfile.so pkg/generator/plugins/hostfile/hostfile.go - go build -buildmode=plugin -o lib/powerman.so pkg/generator/plugins/powerman/powerman.go - go build -buildmode=plugin -o lib/syslog.so pkg/generator/plugins/syslog/syslog.go - go build -buildmode=plugin -o lib/warewulf.so pkg/generator/plugins/warewulf/warewulf.go + go build -buildmode=plugin -o $@ $< # remove executable and all built plugins +.PHONY: clean clean: - rm configurator - rm lib/* + rm -f configurator + rm -f lib/* # run all of the unit tests +.PHONY: test test: - go test ./tests/generate_test.go --tags=all + go test ./tests/generate_test.go --tags=all \ No newline at end of file From 7a1b57931e29445dbf41091fdf15002f72836e91 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 1 Aug 2024 16:02:58 -0600 Subject: [PATCH 007/102] Updated Makefile with changes to container rules --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5783752..900909f 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ DOCKER ?= docker prog ?= configurator +git_tag := $(git describe --abbrev=0 --tags) sources := main.go $(wildcard cmd/*.go) plugin_source_prefix := pkg/generator/plugins plugin_sources := $(filter-out %_test.go,$(wildcard $(plugin_source_prefix)/*/*.go)) @@ -24,11 +25,11 @@ $(prog): $(sources) .PHONY: container container: binaries plugins - $(DOCKER) build . --build-arg --no-cache --pull --tag 'configurator:testing' + $(DOCKER) build . --build-arg --no-cache --pull --tag '$(prog):$(git_tag)-dirty' .PHONY: container-testing container-testing: binaries plugins - $(DOCKER) build . --tag configurator:testing + $(DOCKER) build . --tag $(prog):testing # build all of the generators into plugins .PHONY: plugins From 49fd6fb8926aa1dc3de432ff5125c3029096af4a Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 1 Aug 2024 16:10:35 -0600 Subject: [PATCH 008/102] Changed space indentations to tabs in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 900909f..5273190 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ exe: $(prog) # build named executable from go sources $(prog): $(sources) - go build --tags=all -o $(prog) + go build --tags=all -o $(prog) .PHONY: container container: binaries plugins From dcff41dd4350e52422d82b05638e3892735094e0 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 1 Aug 2024 16:14:47 -0600 Subject: [PATCH 009/102] Add shell directive to git_tag --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5273190..af99754 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ DOCKER ?= docker prog ?= configurator -git_tag := $(git describe --abbrev=0 --tags) +git_tag := $(shell git describe --abbrev=0 --tags) sources := main.go $(wildcard cmd/*.go) plugin_source_prefix := pkg/generator/plugins plugin_sources := $(filter-out %_test.go,$(wildcard $(plugin_source_prefix)/*/*.go)) From be9db173a395d1f22bdc4211ccfeb3403bbb4cda Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 1 Aug 2024 16:40:28 -0600 Subject: [PATCH 010/102] Made building pluging and executable prereqs for test rule --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index af99754..e3bb1a4 100644 --- a/Makefile +++ b/Makefile @@ -48,5 +48,5 @@ clean: # run all of the unit tests .PHONY: test -test: +test: $(prog) $(plugin_binaries) go test ./tests/generate_test.go --tags=all \ No newline at end of file From cd57c36d7ea421448966d6fe590517fe749ad0be Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 18 Sep 2024 21:27:10 -0600 Subject: [PATCH 011/102] Added PKGBUILD to install configurator binary with plugins --- dist/archlinux/PKGBUILD | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 dist/archlinux/PKGBUILD diff --git a/dist/archlinux/PKGBUILD b/dist/archlinux/PKGBUILD new file mode 100644 index 0000000..56159b3 --- /dev/null +++ b/dist/archlinux/PKGBUILD @@ -0,0 +1,34 @@ +# Maintainer: David J. Allen +pkgname=configurator +pkgver=v0.1.0alpha +pkgrel=1 +pkgdesc="An extensible tool to dynamically generate config files from SMD with Jinja 2 templating support." +arch=("x86_64") +url="https://github.com/OpenCHAMI/configurator" +license=('MIT') +groups=("openchami") +provides=('configurator') +conflicts=('configurator') +https://github.com/OpenCHAMI/configurator/releases/download/v0.1.0-alpha/configurator +source_x86_64=( + "${url}/releases/download/v0.1.0-alpha/${pkgname}.tar.gz" +) +sha256sums_x86_64=('28e10f1e39757bbdc3a503de74dd4d8c610d9c78e89665fb42012e8ef7834d0f') + +# we don't need to set pkgver just yet for the pre-release version... +# pkgver() { +# cd "$srcdir" || exit +# printf "%s" "$(git describe --tags --abbrev=0)" +# } + +package() { + cd "$srcdir/" || exit + + # install the binary to /usr/bin + mkdir -p "${pkgdir}/usr/bin" + mkdir -p "${pkgdir}/usr/lib/${pkgname}" + install -m755 configurator "${pkgdir}/usr/bin/configurator" + + # install plugins to /usr/lib + install -m755 *.so "${pkgdir}/usr/lib/${pkgname}" +} From c822531fde57b8bf6c3e7d111ab9e7bbf94cd58c Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 20 Sep 2024 16:50:45 -0600 Subject: [PATCH 012/102] Added check to remove duplicates in 'inspect' cmd --- cmd/inspect.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index 2635b70..735fa7b 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/OpenCHAMI/configurator/pkg/generator" + "github.com/OpenCHAMI/configurator/pkg/util" "github.com/rodaine/table" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -26,11 +27,16 @@ var inspectCmd = &cobra.Command{ return strings.ToUpper(fmt.Sprintf(format, vals...)) } - // TODO: remove duplicate args from CLI + // remove duplicate clean paths from CLI + paths := make([]string, len(args)) + for _, path := range args { + paths = append(paths, filepath.Clean(path)) + } + paths = util.RemoveDuplicates(paths) // load specific plugins from positional args var generators = make(map[string]generator.Generator) - for _, path := range args { + for _, path := range paths { err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { if err != nil { return err From 67854029286fe1eb7f551246e7da5c12eb0ae451 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 20 Sep 2024 16:51:18 -0600 Subject: [PATCH 013/102] Removed vars from fetch cmd --- cmd/fetch.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/fetch.go b/cmd/fetch.go index ef7e16e..c2ba4e5 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -13,12 +13,6 @@ import ( "github.com/spf13/cobra" ) -var ( - accessToken string - remoteHost string - remotePort int -) - var fetchCmd = &cobra.Command{ Use: "fetch", Short: "Fetch a config file from a remote instance of configurator", From 41c0e24c06ee10ff8a5301ef183c1adbcc07b931 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 20 Sep 2024 16:52:00 -0600 Subject: [PATCH 014/102] Changed logic for RunTargets in 'generate' cmd --- cmd/generate.go | 75 ++++++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 15c5e02..390f564 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -12,12 +12,14 @@ import ( configurator "github.com/OpenCHAMI/configurator/pkg" "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) var ( tokenFetchRetries int - pluginPaths []string + templatePaths []string + pluginPath string cacertPath string ) @@ -27,8 +29,6 @@ var generateCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { // make sure that we have a token present before trying to make request if config.AccessToken == "" { - // TODO: make request to check if request will need token - // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead accessToken := os.Getenv("ACCESS_TOKEN") if accessToken != "" { @@ -42,16 +42,10 @@ var generateCmd = &cobra.Command{ } // use cert path from cobra if empty - // TODO: this needs to be checked for the correct desired behavior if config.CertPath == "" { config.CertPath = cacertPath } - // use config plugins if none supplied via CLI - if len(pluginPaths) <= 0 { - pluginPaths = append(pluginPaths, config.PluginDirs...) - } - // show config as JSON and generators if verbose if verbose { b, err := json.MarshalIndent(config, "", " ") @@ -61,8 +55,22 @@ var generateCmd = &cobra.Command{ fmt.Printf("%v\n", string(b)) } - RunTargets(&config, args, targets...) + // run all of the target recursively until completion if provided + if len(targets) > 0 { + RunTargets(&config, args, targets...) + } else { + if pluginPath == "" { + fmt.Printf("no plugin path specified") + return + } + // run generator.Generate() with just plugin path and templates provided + generator.Generate(&config, generator.Params{ + PluginPath: pluginPath, + TemplatePaths: templatePaths, + }) + + } }, } @@ -75,22 +83,20 @@ var generateCmd = &cobra.Command{ func RunTargets(config *configurator.Config, args []string, targets ...string) { // generate config with each supplied target for _, target := range targets { - params := generator.Params{ - Args: args, - PluginPaths: pluginPaths, - Target: target, - Verbose: verbose, - } - outputBytes, err := generator.GenerateWithTarget(config, params) + outputBytes, err := generator.GenerateWithTarget(config, generator.Params{ + Args: args, + PluginPath: pluginPath, + Target: target, + Verbose: verbose, + }) if err != nil { - fmt.Printf("failed to generate config: %v\n", err) + log.Error().Err(err).Msg("failed to generate config") os.Exit(1) } - outputMap := generator.ConvertContentsToString(outputBytes) - // if we have more than one target and output is set, create configs in directory var ( + outputMap = generator.ConvertContentsToString(outputBytes) targetCount = len(targets) templateCount = len(outputMap) ) @@ -110,16 +116,16 @@ func RunTargets(config *configurator.Config, args []string, targets ...string) { for _, contents := range outputBytes { err := os.WriteFile(outputPath, contents, 0o644) if err != nil { - fmt.Printf("failed to write config to file: %v", err) + log.Error().Err(err).Msg("failed to write config to file") os.Exit(1) } - fmt.Printf("wrote file to '%s'\n", outputPath) + log.Info().Msgf("wrote file to '%s'\n", outputPath) } } else if outputPath != "" && targetCount > 1 || templateCount > 1 { // write multiple files in directory using template name err := os.MkdirAll(filepath.Clean(outputPath), 0o755) if err != nil { - fmt.Printf("failed to make output directory: %v\n", err) + log.Error().Err(err).Msg("failed to make output directory") os.Exit(1) } for path, contents := range outputBytes { @@ -127,15 +133,17 @@ func RunTargets(config *configurator.Config, args []string, targets ...string) { cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) err := os.WriteFile(cleanPath, contents, 0o755) if err != nil { - fmt.Printf("failed to write config to file: %v\n", err) + log.Error().Err(err).Msg("failed to write config to file") os.Exit(1) } - fmt.Printf("wrote file to '%s'\n", cleanPath) + log.Info().Msgf("wrote file to '%s'\n", cleanPath) } } // remove any targets that are the same as current to prevent infinite loop - nextTargets := util.CopyIf(config.Targets[target].RunTargets, func(t string) bool { return t != target }) + nextTargets := util.CopyIf(config.Targets[target].RunTargets, func(nextTarget string) bool { + return nextTarget != target + }) // ...then, run any other targets that the current target has RunTargets(config, args, nextTargets...) @@ -143,11 +151,20 @@ func RunTargets(config *configurator.Config, args []string, targets ...string) { } func init() { - generateCmd.Flags().StringSliceVar(&targets, "target", []string{}, "set the target configs to make") - generateCmd.Flags().StringSliceVar(&pluginPaths, "plugins", []string{}, "set the generator plugins directory path") + generateCmd.Flags().StringSliceVar(&targets, "target", []string{}, "set the targets to run pre-defined config") + generateCmd.Flags().StringSliceVar(&templatePaths, "template", []string{}, "set the paths for the Jinja 2 templates to use") + generateCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugin path") generateCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") - generateCmd.Flags().StringVar(&cacertPath, "ca-cert", "", "path to CA cert. (defaults to system CAs)") + generateCmd.Flags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") generateCmd.Flags().IntVar(&tokenFetchRetries, "fetch-retries", 5, "set the number of retries to fetch an access token") + generateCmd.Flags().StringVar(&remoteHost, "host", "http://localhost", "set the remote host") + generateCmd.Flags().IntVar(&remotePort, "port", 80, "set the remote port") + + // requires either 'target' by itself or 'plugin' and 'templates' together + // generateCmd.MarkFlagsOneRequired("target", "plugin") + generateCmd.MarkFlagsMutuallyExclusive("target", "plugin") + generateCmd.MarkFlagsMutuallyExclusive("target", "template") + generateCmd.MarkFlagsRequiredTogether("plugin", "template") rootCmd.AddCommand(generateCmd) } From b488c32195b765f0a65659192fa872a1c84c98a6 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 20 Sep 2024 16:52:31 -0600 Subject: [PATCH 015/102] Updated vars in 'root' cmd --- cmd/root.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index cbbfab7..26ccba7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,11 +10,14 @@ import ( ) var ( - configPath string - config configurator.Config - verbose bool - targets []string - outputPath string + configPath string + config configurator.Config + verbose bool + targets []string + outputPath string + accessToken string + remoteHost string + remotePort int ) var rootCmd = &cobra.Command{ From e044d4b5edaeeccdeb1a77694a2d5fec64228625 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 20 Sep 2024 16:53:30 -0600 Subject: [PATCH 016/102] Changed from using multiple plugin paths to just one --- cmd/serve.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index ee54f6c..f7e4401 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -35,11 +35,6 @@ var serveCmd = &cobra.Command{ } } - // use config plugins if none supplied via CLI - if len(pluginPaths) <= 0 { - pluginPaths = append(pluginPaths, config.PluginDirs...) - } - // show config as JSON and generators if verbose if verbose { b, err := json.MarshalIndent(config, "", " ") @@ -60,15 +55,19 @@ var serveCmd = &cobra.Command{ Retries: config.Server.Jwks.Retries, }, GeneratorParams: generator.Params{ - Args: args, - PluginPaths: pluginPaths, + Args: args, + PluginPath: pluginPath, // Target: target, // NOTE: targets are set via HTTP requests (ex: curl http://configurator:3334/generate?target=dnsmasq) Verbose: verbose, }, } + + // start listening with the server err := server.Serve() if errors.Is(err, http.ErrServerClosed) { - fmt.Printf("Server closed.") + if verbose { + fmt.Printf("Server closed.") + } } else if err != nil { fmt.Errorf("failed to start server: %v", err) os.Exit(1) @@ -79,7 +78,7 @@ var serveCmd = &cobra.Command{ func init() { serveCmd.Flags().StringVar(&config.Server.Host, "host", config.Server.Host, "set the server host") serveCmd.Flags().IntVar(&config.Server.Port, "port", config.Server.Port, "set the server port") - serveCmd.Flags().StringSliceVar(&pluginPaths, "plugins", nil, "set the generator plugins directory path") + serveCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugins directory path") serveCmd.Flags().StringVar(&config.Server.Jwks.Uri, "jwks-uri", config.Server.Jwks.Uri, "set the JWKS url to fetch public key") serveCmd.Flags().IntVar(&config.Server.Jwks.Retries, "jwks-fetch-retries", config.Server.Jwks.Retries, "set the JWKS fetch retry count") rootCmd.AddCommand(serveCmd) From b922dbdbdac38361512652751e728193c0f27e30 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 20 Sep 2024 16:54:25 -0600 Subject: [PATCH 017/102] Removed VerifyClaims --- pkg/auth.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/pkg/auth.go b/pkg/auth.go index 58aea04..c857102 100644 --- a/pkg/auth.go +++ b/pkg/auth.go @@ -8,26 +8,9 @@ import ( "slices" "github.com/OpenCHAMI/jwtauth/v5" - "github.com/lestrrat-go/jwx/jwk" + "github.com/lestrrat-go/jwx/v2/jwk" ) -func VerifyClaims(testClaims []string, r *http.Request) (bool, error) { - // extract claims from JWT - _, claims, err := jwtauth.FromContext(r.Context()) - if err != nil { - return false, fmt.Errorf("failed to get claims(s) from token: %v", err) - } - - // verify that each one of the test claims are included - for _, testClaim := range testClaims { - _, ok := claims[testClaim] - if !ok { - return false, fmt.Errorf("failed to verify claim(s) from token: %s", testClaim) - } - } - return true, nil -} - func VerifyScope(testScopes []string, r *http.Request) (bool, error) { // extract the scopes from JWT var scopes []string @@ -112,3 +95,7 @@ func FetchPublicKeyFromURL(url string) (*jwtauth.JWTAuth, error) { return tokenAuth, nil } + +func LoadAccessToken() { + +} From 601089672c7d441b1ed8730dcd65905f7fb631e8 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 20 Sep 2024 16:55:31 -0600 Subject: [PATCH 018/102] Changed how plugins and tempates are loaded --- pkg/generator/generator.go | 264 ++++++++++++++++++++++++++----------- 1 file changed, 188 insertions(+), 76 deletions(-) diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 27a032e..9a00545 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -3,7 +3,7 @@ package generator import ( "bytes" "fmt" - "maps" + "io/fs" "os" "path/filepath" "plugin" @@ -17,6 +17,7 @@ import ( type Mappings map[string]any type FileMap map[string][]byte type FileList [][]byte +type Template []byte // Generator interface used to define how files are created. Plugins can // be created entirely independent of the main driver program. @@ -29,11 +30,12 @@ type Generator interface { // Params defined and used by the "generate" subcommand. type Params struct { - Args []string - PluginPaths []string - Generators map[string]Generator - Target string - Verbose bool + Args []string + Generators map[string]Generator + TemplatePaths []string + PluginPath string + Target string + Verbose bool } // Converts the file outputs from map[string][]byte to map[string]string. @@ -51,13 +53,13 @@ func LoadFiles(paths ...string) (FileMap, error) { for _, path := range paths { expandedPaths, err := filepath.Glob(path) if err != nil { - return nil, fmt.Errorf("failed to glob path: %v", err) + return nil, fmt.Errorf("failed to glob path: %w", err) } for _, expandedPath := range expandedPaths { info, err := os.Stat(expandedPath) if err != nil { fmt.Println(err) - return nil, fmt.Errorf("failed to stat file or directory: %v", err) + return nil, fmt.Errorf("failed to stat file or directory: %w", err) } // skip any directories found if info.IsDir() { @@ -65,7 +67,7 @@ func LoadFiles(paths ...string) (FileMap, error) { } b, err := os.ReadFile(expandedPath) if err != nil { - return nil, fmt.Errorf("failed to read file: %v", err) + return nil, fmt.Errorf("failed to read file: %w", err) } outputs[expandedPath] = b @@ -81,19 +83,19 @@ func LoadPlugin(path string) (Generator, error) { if isDir, err := util.IsDirectory(path); err == nil && isDir { return nil, nil } else if err != nil { - return nil, fmt.Errorf("failed to test if path is directory: %v", err) + return nil, fmt.Errorf("failed to test if plugin path is directory: %w", err) } // try and open the plugin p, err := plugin.Open(path) if err != nil { - return nil, fmt.Errorf("failed to open plugin: %v", err) + return nil, fmt.Errorf("failed to open plugin: %w", err) } // load the "Generator" symbol from plugin symbol, err := p.Lookup("Generator") if err != nil { - return nil, fmt.Errorf("failed to look up symbol at path '%s': %v", path, err) + return nil, fmt.Errorf("failed to look up symbol at path '%s': %w", path, err) } // assert that the plugin loaded has a valid generator @@ -111,45 +113,123 @@ func LoadPlugin(path string) (Generator, error) { func LoadPlugins(dirpath string, opts ...util.Option) (map[string]Generator, error) { // check if verbose option is supplied var ( - gens = make(map[string]Generator) - params = util.GetParams(opts...) + generators = make(map[string]Generator) + params = util.ToDict(opts...) ) - items, _ := os.ReadDir(dirpath) - for _, item := range items { - if item.IsDir() { - subitems, _ := os.ReadDir(item.Name()) - for _, subitem := range subitems { - if !subitem.IsDir() { - gen, err := LoadPlugin(subitem.Name()) - if err != nil { - fmt.Printf("failed to load generator in directory '%s': %v\n", item.Name(), err) - continue - } - if verbose, ok := params["verbose"].(bool); ok { - if verbose { - fmt.Printf("-- found plugin '%s'\n", item.Name()) - } - } - gens[gen.GetName()] = gen - } + // + err := filepath.Walk(dirpath, func(path string, info fs.FileInfo, err error) error { + // skip trying to load generator plugin if directory or error + if info.IsDir() || err != nil { + return nil + } + + // load the generator plugin from current path + gen, err := LoadPlugin(path) + if err != nil { + return fmt.Errorf("failed to load generator in directory '%s': %w", path, err) + } + + // show the plugins found if verbose flag is set + if params.GetVerbose() { + fmt.Printf("-- found plugin '%s'\n", gen.GetName()) + } + + // map each generator plugin by name for lookup + generators[gen.GetName()] = gen + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to walk directory: %w", err) + } + + // items, _ := os.ReadDir(dirpath) + // for _, item := range items { + // if item.IsDir() { + // subitems, _ := os.ReadDir(item.Name()) + // for _, subitem := range subitems { + // if !subitem.IsDir() { + // gen, err := LoadPlugin(subitem.Name()) + // if err != nil { + // fmt.Printf("failed to load generator in directory '%s': %v\n", item.Name(), err) + // continue + // } + // if verbose, ok := params["verbose"].(bool); ok { + // if verbose { + // fmt.Printf("-- found plugin '%s'\n", item.Name()) + // } + // } + // gens[gen.GetName()] = gen + // } + // } + // } else { + // gen, err := LoadPlugin(dirpath + item.Name()) + // if err != nil { + // fmt.Printf("failed to load plugin: %v\n", err) + // continue + // } + // if verbose, ok := params["verbose"].(bool); ok { + // if verbose { + // fmt.Printf("-- found plugin '%s'\n", dirpath+item.Name()) + // } + // } + // gens[gen.GetName()] = gen + // } + // } + + return generators, nil +} + +func LoadTemplate(path string) (Template, error) { + // skip loading template if path is a directory with no error + if isDir, err := util.IsDirectory(path); err == nil && isDir { + return nil, nil + } else if err != nil { + return nil, fmt.Errorf("failed to test if template path is directory: %w", err) + } + + // try and read the contents of the file + // NOTE: we don't care if this is actually a Jinja template + // or not...at least for now. + return os.ReadFile(path) +} + +func LoadTemplates(paths []string, opts ...util.Option) (map[string]Template, error) { + var ( + templates = make(map[string]Template) + params = util.ToDict(opts...) + ) + + for _, path := range paths { + err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + // skip trying to load generator plugin if directory or error + if info.IsDir() || err != nil { + return nil } - } else { - gen, err := LoadPlugin(dirpath + item.Name()) + + // load the contents of the template + template, err := LoadTemplate(path) if err != nil { - fmt.Printf("failed to load plugin: %v\n", err) - continue + return fmt.Errorf("failed to load generator in directory '%s': %w", path, err) } - if verbose, ok := params["verbose"].(bool); ok { - if verbose { - fmt.Printf("-- found plugin '%s'\n", dirpath+item.Name()) - } + + // show the templates loaded if verbose flag is set + if params.GetVerbose() { + fmt.Printf("-- loaded tempalte '%s'\n", path) } - gens[gen.GetName()] = gen + + // map each template by the path it was loaded from + templates[path] = template + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to walk directory: %w", err) } } - return gens, nil + return templates, nil } // Option to specify "target" in parameter map. This is used to set which generator @@ -171,6 +251,31 @@ func WithType(_type string) util.Option { } } +// Option to the plugin to load +func WithPlugin(path string) util.Option { + return func(p util.Params) { + if p != nil { + plugin, err := LoadPlugin(path) + if err != nil { + return + } + p["plugin"] = plugin + } + } +} + +func WithTemplates(paths []string) util.Option { + return func(p util.Params) { + if p != nil { + templates, err := LoadTemplates(paths) + if err != nil { + + } + p["templates"] = templates + } + } +} + // Option to a specific client to include in implementing plugin generator.Generate(). // // NOTE: This may be changed to pass some kind of client interface as an argument in @@ -217,13 +322,13 @@ func ApplyTemplates(mappings Mappings, contents ...[]byte) (FileList, error) { // load jinja template from file t, err := gonja.FromBytes(b) if err != nil { - return nil, fmt.Errorf("failed to read template from file: %v", err) + return nil, fmt.Errorf("failed to read template from file: %w", err) } // execute/render jinja template b := bytes.Buffer{} if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %v", err) + return nil, fmt.Errorf("failed to execute: %w", err) } outputs = append(outputs, b.Bytes()) } @@ -243,13 +348,13 @@ func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) // load jinja template from file t, err := gonja.FromFile(path) if err != nil { - return nil, fmt.Errorf("failed to read template from file: %v", err) + return nil, fmt.Errorf("failed to read template from file: %w", err) } // execute/render jinja template b := bytes.Buffer{} if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %v", err) + return nil, fmt.Errorf("failed to execute: %w", err) } outputs[path] = b.Bytes() } @@ -257,6 +362,23 @@ func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) return outputs, nil } +// Generate() is the main function to generate a collection of files and returns them as a map. +// This function only expects a path to a plugin and paths to a collection of templates to +// be used. This function will only load the plugin on-demand and fetch resources as needed. +func Generate(config *configurator.Config, params Params) (FileMap, error) { + var ( + gen Generator + client = configurator.NewSmdClient() + ) + + return gen.Generate( + config, + WithPlugin(params.PluginPath), + WithTemplates(params.TemplatePaths), + WithClient(client), + ) +} + // Main function to generate a collection of files as a map with the path as the key and // the contents of the file as the value. This function currently expects a list of plugin // paths to load all plugins within a directory. Then, each plugin's generator.GenerateWithTarget() @@ -269,8 +391,7 @@ func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) func GenerateWithTarget(config *configurator.Config, params Params) (FileMap, error) { // load generator plugins to generate configs or to print var ( - generators = make(map[string]Generator) - client = configurator.NewSmdClient( + client = configurator.NewSmdClient( configurator.WithHost(config.SmdClient.Host), configurator.WithPort(config.SmdClient.Port), configurator.WithAccessToken(config.AccessToken), @@ -278,41 +399,32 @@ func GenerateWithTarget(config *configurator.Config, params Params) (FileMap, er ) ) - // load all plugins from supplied arguments - for _, path := range params.PluginPaths { - if params.Verbose { - fmt.Printf("loading plugins from '%s'\n", path) - } - plugins, err := LoadPlugins(path) - if err != nil { - fmt.Printf("failed to load plugins: %v\n", err) - err = nil - continue - } - - // add loaded generator plugins to set - maps.Copy(generators, plugins) + // check if a target is supplied + if len(params.Args) == 0 && params.Target == "" { + return nil, fmt.Errorf("must specify a target") } - // copy all generators supplied from arguments - maps.Copy(generators, params.Generators) + // load target information from config + target, ok := config.Targets[params.Target] + if !ok { + return nil, fmt.Errorf("target not found in config") + } - // show available targets then exit - if len(params.Args) == 0 && params.Target == "" { - for g := range generators { - fmt.Printf("-- found generator plugin \"%s\"\n", g) - } - return nil, nil + // if plugin path specified from CLI, use that instead + if params.PluginPath != "" { + target.PluginPath = params.PluginPath + } + + // only load the plugin needed for this target + generator, err := LoadPlugin(target.PluginPath) + if err != nil { + return nil, fmt.Errorf("failed to load plugin: %w", err) } // run the generator plugin from target passed - gen := generators[params.Target] - if gen == nil { - return nil, fmt.Errorf("invalid generator target (%s)", params.Target) - } - return gen.Generate( + return generator.Generate( config, - WithTarget(gen.GetName()), + WithTarget(generator.GetName()), WithClient(client), ) } From 751a2facdbea29d882a5d37f9c2f950b717cc6c8 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 20 Sep 2024 16:56:15 -0600 Subject: [PATCH 019/102] Minor changes to util functions --- pkg/util/params.go | 19 ++++++++++++++++--- pkg/util/util.go | 11 +++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pkg/util/params.go b/pkg/util/params.go index 6b342cd..322d852 100644 --- a/pkg/util/params.go +++ b/pkg/util/params.go @@ -11,7 +11,7 @@ type Params map[string]any type Option func(Params) // Extract all parameters from the options passed as map[string]any. -func GetParams(opts ...Option) Params { +func ToDict(opts ...Option) Params { params := Params{} for _, opt := range opts { opt(params) @@ -45,8 +45,8 @@ func WithDefault[T any](v T) Option { } } -// Syntactic sugar generic function to get parameter from util.Params. -func Get[T any](params Params, key string, opts ...Option) *T { +// Sugary generic function to get parameter from util.Params. +func Get[T any](params Params, key string) *T { if v, ok := params[key].(T); ok { return &v } @@ -55,3 +55,16 @@ func Get[T any](params Params, key string, opts ...Option) *T { } return nil } + +func GetOpt[T any](opts []Option, key string) *T { + return Get[T](ToDict(opts...), "required_claims") +} + +func (p Params) GetVerbose() bool { + if verbose, ok := p["verbose"].(bool); ok { + return verbose + } + + // default setting + return false +} diff --git a/pkg/util/util.go b/pkg/util/util.go index fd0daa2..fc53b67 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -2,12 +2,14 @@ package util import ( "bytes" + "cmp" "crypto/tls" "fmt" "io" "net/http" "os" "os/exec" + "slices" "strings" ) @@ -28,7 +30,7 @@ func IsDirectory(path string) (bool, error) { // This returns an *os.FileInfo type fileInfo, err := os.Stat(path) if err != nil { - return false, fmt.Errorf("failed to stat path: %v", err) + return false, fmt.Errorf("failed to stat path (%s): %v", path, err) } // IsDir is short for fileInfo.Mode().IsDir() @@ -63,7 +65,7 @@ func MakeRequest(url string, httpMethod string, body []byte, headers map[string] // NOTE: This currently requires git to be installed. // TODO: Change how this is done to not require executing a command. func GitCommit() string { - c := exec.Command("git", "rev-parse", "HEAD") + c := exec.Command("git", "rev-parse", "--short=8", "HEAD") stdout, err := c.Output() if err != nil { return "" @@ -80,6 +82,11 @@ func RemoveIndex[T comparable](s []T, index int) []T { return append(ret, s[index+1:]...) } +func RemoveDuplicates[T cmp.Ordered](s []T) []T { + slices.Sort(s) + return slices.Compact(s) +} + // General function to copy elements from slice if condition is true. func CopyIf[T comparable](s []T, condition func(t T) bool) []T { var f = make([]T, 0) From cb73258a84db24ea8a24801fbae597f8284a815e Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 20 Sep 2024 16:56:46 -0600 Subject: [PATCH 020/102] Minor changes to error format in dhcpd plugin --- pkg/generator/plugins/dhcpd/dhcpd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/generator/plugins/dhcpd/dhcpd.go b/pkg/generator/plugins/dhcpd/dhcpd.go index a736064..1a37b9c 100644 --- a/pkg/generator/plugins/dhcpd/dhcpd.go +++ b/pkg/generator/plugins/dhcpd/dhcpd.go @@ -37,7 +37,7 @@ func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (gene if client != nil { eths, err = client.FetchEthernetInterfaces(opts...) if err != nil { - return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) + return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %w", err) } } From e05bd58ef6281d3678dd4d1e2b66b48e6359efb0 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 23 Sep 2024 11:28:53 -0600 Subject: [PATCH 021/102] Changed binaries rule to include build plugins --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e3bb1a4..6b63ac2 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ all: plugins exe test .PHONY: main driver binaries exe main: exe driver: exe -binaries: exe +binaries: exe plugins exe: $(prog) # build named executable from go sources From bc6e8561790933f248db3eac665ef4fb779698d8 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 23 Sep 2024 11:29:38 -0600 Subject: [PATCH 022/102] Removed plugins rule that have binaries rule --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6b63ac2..2c18932 100644 --- a/Makefile +++ b/Makefile @@ -24,11 +24,11 @@ $(prog): $(sources) go build --tags=all -o $(prog) .PHONY: container -container: binaries plugins +container: binaries $(DOCKER) build . --build-arg --no-cache --pull --tag '$(prog):$(git_tag)-dirty' .PHONY: container-testing -container-testing: binaries plugins +container-testing: binaries $(DOCKER) build . --tag $(prog):testing # build all of the generators into plugins From 84e27d622aff75375cbb8afad30f8ed26802d627 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 24 Sep 2024 15:27:23 -0600 Subject: [PATCH 023/102] Added option to compress and archive multiple generated files --- cmd/generate.go | 21 ++++++++++++++++ pkg/util/util.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/cmd/generate.go b/cmd/generate.go index 390f564..8ce2093 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -21,6 +21,7 @@ var ( templatePaths []string pluginPath string cacertPath string + useCompression bool ) var generateCmd = &cobra.Command{ @@ -121,6 +122,25 @@ func RunTargets(config *configurator.Config, args []string, targets ...string) { } log.Info().Msgf("wrote file to '%s'\n", outputPath) } + } else if outputPath != "" && targetCount > 1 && useCompression { + // write multiple files to archive, compress, then save to output path + out, err := os.Create(fmt.Sprintf("%s.tar.gz", outputPath)) + if err != nil { + log.Error().Err(err).Msg("failed to write archive") + os.Exit(1) + } + files := make([]string, len(outputBytes)) + i := 0 + for path := range outputBytes { + files[i] = path + i++ + } + err = util.CreateArchive(files, out) + if err != nil { + log.Error().Err(err).Msg("failed to create archive") + os.Exit(1) + } + } else if outputPath != "" && targetCount > 1 || templateCount > 1 { // write multiple files in directory using template name err := os.MkdirAll(filepath.Clean(outputPath), 0o755) @@ -159,6 +179,7 @@ func init() { generateCmd.Flags().IntVar(&tokenFetchRetries, "fetch-retries", 5, "set the number of retries to fetch an access token") generateCmd.Flags().StringVar(&remoteHost, "host", "http://localhost", "set the remote host") generateCmd.Flags().IntVar(&remotePort, "port", 80, "set the remote port") + generateCmd.Flags().BoolVar(&useCompression, "compress", false, "set whether to archive and compress multiple file outputs") // requires either 'target' by itself or 'plugin' and 'templates' together // generateCmd.MarkFlagsOneRequired("target", "plugin") diff --git a/pkg/util/util.go b/pkg/util/util.go index fc53b67..6ff13b0 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1,8 +1,10 @@ package util import ( + "archive/tar" "bytes" "cmp" + "compress/gzip" "crypto/tls" "fmt" "io" @@ -97,3 +99,64 @@ func CopyIf[T comparable](s []T, condition func(t T) bool) []T { } return f } + +func CreateArchive(files []string, buf io.Writer) error { + // Create new Writers for gzip and tar + // These writers are chained. Writing to the tar writer will + // write to the gzip writer which in turn will write to + // the "buf" writer + gw := gzip.NewWriter(buf) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + + // Iterate over files and add them to the tar archive + for _, file := range files { + err := addToArchive(tw, file) + if err != nil { + return err + } + } + + return nil +} + +func addToArchive(tw *tar.Writer, filename string) error { + // open file to write to archive + file, err := os.Open(filename) + if err != nil { + return err + } + defer file.Close() + + // get FileInfo for file size, mode, etc. + info, err := file.Stat() + if err != nil { + return err + } + + // create a tar Header from the FileInfo data + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return err + } + + // use full path as name (FileInfoHeader only takes the basename) to + // preserve directory structure + // see for more info: https://golang.org/src/archive/tar/common.go?#L626 + header.Name = filename + + // Write file header to the tar archive + err = tw.WriteHeader(header) + if err != nil { + return err + } + + // copy file content to tar archive + _, err = io.Copy(tw, file) + if err != nil { + return err + } + + return nil +} From e14a8565df6c19af626f6e92eb82276ce3027195 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 25 Sep 2024 18:09:25 -0600 Subject: [PATCH 024/102] Added --always flag to git_tag to prevent erroring out --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2c18932..2279b8a 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ DOCKER ?= docker prog ?= configurator -git_tag := $(shell git describe --abbrev=0 --tags) +git_tag := $(shell git describe --abbrev=0 --tags --always) sources := main.go $(wildcard cmd/*.go) plugin_source_prefix := pkg/generator/plugins plugin_sources := $(filter-out %_test.go,$(wildcard $(plugin_source_prefix)/*/*.go)) From 99eb87d806c951b1b1336bf815bfccb0e0b3f4ef Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 2 Oct 2024 11:50:28 -0600 Subject: [PATCH 025/102] Renamed LICENSE.md to LICENSE --- LICENSE.md => LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename LICENSE.md => LICENSE (97%) diff --git a/LICENSE.md b/LICENSE similarity index 97% rename from LICENSE.md rename to LICENSE index a17415c..aa18635 100644 --- a/LICENSE.md +++ b/LICENSE @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file From 397bfa5b3171f4ad1ea3c43cf951f83177f448cf Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 2 Oct 2024 13:29:56 -0600 Subject: [PATCH 026/102] Fixed typo in conman plugin --- pkg/generator/plugins/conman/conman.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/generator/plugins/conman/conman.go b/pkg/generator/plugins/conman/conman.go index 6a14f89..0a790db 100644 --- a/pkg/generator/plugins/conman/conman.go +++ b/pkg/generator/plugins/conman/conman.go @@ -26,7 +26,7 @@ func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (gen var ( params = generator.GetParams(opts...) client = generator.GetClient(params) - targetKey = params["targets"].(string) // required param + targetKey = params["target"].(string) // required param target = config.Targets[targetKey] eps []configurator.RedfishEndpoint = nil err error = nil From 35ee21b3f7e3e427d0909369a457830ae78c44a3 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 2 Oct 2024 13:46:51 -0600 Subject: [PATCH 027/102] Added consoles substitution to conman plugin --- pkg/generator/plugins/conman/conman.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/generator/plugins/conman/conman.go b/pkg/generator/plugins/conman/conman.go index 0a790db..e514ae9 100644 --- a/pkg/generator/plugins/conman/conman.go +++ b/pkg/generator/plugins/conman/conman.go @@ -62,6 +62,7 @@ func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (gen "plugin_description": g.GetDescription(), "server_opts": "", "global_opts": "", + "consoles": consoles, }, target.TemplatePaths...) } From 8a1fa5211b4e60d10ccec535d97afe710054488d Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 2 Oct 2024 13:58:31 -0600 Subject: [PATCH 028/102] Added CHANGELOG.md --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a6c68b3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +[0.1.0] + +- Initial prerelease of configurator \ No newline at end of file From 8afcf6f005db0537afd37e61ce63a5291c73861c Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 2 Oct 2024 14:31:01 -0600 Subject: [PATCH 029/102] Added dockers to build image and removed unnecessary file --- .goreleaser.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index d5592d1..4a14820 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -24,7 +24,23 @@ archives: - LICENSE - CHANGELOG.md - README.md - - configurator +dockers: + - + image_templates: + - ghcr.io/openchami/{{.ProjectName}}:latest + - ghcr.io/openchami/{{.ProjectName}}:{{ .Tag }} + - ghcr.io/openchami/{{.ProjectName}}:v{{ .Major }} + - ghcr.io/openchami/{{.ProjectName}}:v{{ .Major }}.{{ .Minor }} + build_flag_templates: + - "--pull" + - "--label=org.opencontainers.image.created={{.Date}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}" + - "--label=org.opencontainers.image.revision={{.FullCommit}}" + - "--label=org.opencontainers.image.version={{.Version}}" + extra_files: + - LICENSE + - CHANGELOG.md + - README.md checksum: name_template: 'checksums.txt' snapshot: From 113b6a936809247cec7e8b7cd4e146f37bbb3d1a Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 2 Oct 2024 14:49:35 -0600 Subject: [PATCH 030/102] goreleaser: added lib/ to archive and dockers --- .goreleaser.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 4a14820..5910ef5 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -24,6 +24,7 @@ archives: - LICENSE - CHANGELOG.md - README.md + - lib/ dockers: - image_templates: @@ -41,6 +42,7 @@ dockers: - LICENSE - CHANGELOG.md - README.md + - lib/ checksum: name_template: 'checksums.txt' snapshot: From 34acf3d95b158ef3e1710a70b6e03b33c3f662a4 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 2 Oct 2024 14:55:18 -0600 Subject: [PATCH 031/102] goreleaser: add hook to build plugins --- .goreleaser.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 5910ef5..b0351f8 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -3,6 +3,7 @@ version: 2 before: hooks: - go mod download + - make plugins builds: - env: - CGO_ENABLED=1 From ed0b8f8b592e6f5f8d289e224aadd5d882ceda1d Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 2 Oct 2024 15:26:41 -0600 Subject: [PATCH 032/102] workflow: added docker login to push package --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0b8ae48..4f4adde 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,6 +20,12 @@ jobs: uses: actions/setup-go@v5 with: go-version: 1.21 + - name: Docker Login + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Checkout uses: actions/checkout@v4 with: From 34845f6a5c93c80eb4e75b3541462109d8618260 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 13 Nov 2024 17:40:12 -0700 Subject: [PATCH 033/102] plugin: moved default plugins to compile into executable --- pkg/generator/{plugins/conman => }/conman.go | 13 +++++------- .../{plugins/coredhcp => }/coredhcp.go | 10 ++++----- pkg/generator/{plugins/dhcpd => }/dhcpd.go | 21 ++++++++----------- .../{plugins/dnsmasq => }/dnsmasq.go | 21 ++++++++----------- .../{plugins/example => }/example.go | 5 ++++- .../{plugins/hostfile => }/hostfile.go | 7 ++----- .../{plugins/powerman => }/powerman.go | 7 ++----- pkg/generator/{plugins/syslog => }/syslog.go | 7 ++----- .../{plugins/warewulf => }/warewulf.go | 17 +++++++-------- 9 files changed, 45 insertions(+), 63 deletions(-) rename pkg/generator/{plugins/conman => }/conman.go (84%) rename pkg/generator/{plugins/coredhcp => }/coredhcp.go (81%) rename pkg/generator/{plugins/dhcpd => }/dhcpd.go (76%) rename pkg/generator/{plugins/dnsmasq => }/dnsmasq.go (79%) rename pkg/generator/{plugins/example => }/example.go (91%) rename pkg/generator/{plugins/hostfile => }/hostfile.go (80%) rename pkg/generator/{plugins/powerman => }/powerman.go (80%) rename pkg/generator/{plugins/syslog => }/syslog.go (80%) rename pkg/generator/{plugins/warewulf => }/warewulf.go (84%) diff --git a/pkg/generator/plugins/conman/conman.go b/pkg/generator/conman.go similarity index 84% rename from pkg/generator/plugins/conman/conman.go rename to pkg/generator/conman.go index e514ae9..53358b5 100644 --- a/pkg/generator/plugins/conman/conman.go +++ b/pkg/generator/conman.go @@ -1,10 +1,9 @@ -package main +package generator import ( "fmt" configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -22,10 +21,10 @@ func (g *Conman) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { var ( - params = generator.GetParams(opts...) - client = generator.GetClient(params) + params = GetParams(opts...) + client = GetClient(params) targetKey = params["target"].(string) // required param target = config.Targets[targetKey] eps []configurator.RedfishEndpoint = nil @@ -56,7 +55,7 @@ func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (gen consoles += "# =====================================================================" // apply template substitutions and return output as byte array - return generator.ApplyTemplateFromFiles(generator.Mappings{ + return ApplyTemplateFromFiles(Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), @@ -65,5 +64,3 @@ func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (gen "consoles": consoles, }, target.TemplatePaths...) } - -var Generator Conman diff --git a/pkg/generator/plugins/coredhcp/coredhcp.go b/pkg/generator/coredhcp.go similarity index 81% rename from pkg/generator/plugins/coredhcp/coredhcp.go rename to pkg/generator/coredhcp.go index 4e0729a..817790c 100644 --- a/pkg/generator/plugins/coredhcp/coredhcp.go +++ b/pkg/generator/coredhcp.go @@ -1,10 +1,12 @@ -package main +//go:build coredhcp || plugins +// +build coredhcp plugins + +package generator import ( "fmt" configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -22,8 +24,6 @@ func (g *CoreDhcp) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s' to generate config files. (WIP)", g.GetName()) } -func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } - -var Generator CoreDhcp diff --git a/pkg/generator/plugins/dhcpd/dhcpd.go b/pkg/generator/dhcpd.go similarity index 76% rename from pkg/generator/plugins/dhcpd/dhcpd.go rename to pkg/generator/dhcpd.go index 1a37b9c..f2ce520 100644 --- a/pkg/generator/plugins/dhcpd/dhcpd.go +++ b/pkg/generator/dhcpd.go @@ -1,31 +1,30 @@ -package main +package generator import ( "fmt" configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" ) -type Dhcpd struct{} +type DHCPd struct{} -func (g *Dhcpd) GetName() string { +func (g *DHCPd) GetName() string { return "dhcpd" } -func (g *Dhcpd) GetVersion() string { +func (g *DHCPd) GetVersion() string { return util.GitCommit() } -func (g *Dhcpd) GetDescription() string { +func (g *DHCPd) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *DHCPd) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { var ( - params = generator.GetParams(opts...) - client = generator.GetClient(params) + params = GetParams(opts...) + client = GetClient(params) targetKey = params["target"].(string) target = config.Targets[targetKey] compute_nodes = "" @@ -64,7 +63,7 @@ func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (gene fmt.Printf("") } } - return generator.ApplyTemplateFromFiles(generator.Mappings{ + return ApplyTemplateFromFiles(Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), @@ -72,5 +71,3 @@ func (g *Dhcpd) Generate(config *configurator.Config, opts ...util.Option) (gene "node_entries": "", }, target.TemplatePaths...) } - -var Generator Dhcpd diff --git a/pkg/generator/plugins/dnsmasq/dnsmasq.go b/pkg/generator/dnsmasq.go similarity index 79% rename from pkg/generator/plugins/dnsmasq/dnsmasq.go rename to pkg/generator/dnsmasq.go index 9150009..ab5e648 100644 --- a/pkg/generator/plugins/dnsmasq/dnsmasq.go +++ b/pkg/generator/dnsmasq.go @@ -1,29 +1,28 @@ -package main +package generator import ( "fmt" "strings" configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" ) -type DnsMasq struct{} +type DNSMasq struct{} -func (g *DnsMasq) GetName() string { +func (g *DNSMasq) GetName() string { return "dnsmasq" } -func (g *DnsMasq) GetVersion() string { +func (g *DNSMasq) GetVersion() string { return util.GitCommit() } -func (g *DnsMasq) GetDescription() string { +func (g *DNSMasq) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *DNSMasq) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { // make sure we have a valid config first if config == nil { return nil, fmt.Errorf("invalid config (config is nil)") @@ -31,8 +30,8 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (ge // set all the defaults for variables var ( - params = generator.GetParams(opts...) - client = generator.GetClient(params) + params = GetParams(opts...) + client = GetClient(params) targetKey = params["target"].(string) // required param target = config.Targets[targetKey] eths []configurator.EthernetInterface = nil @@ -74,12 +73,10 @@ func (g *DnsMasq) Generate(config *configurator.Config, opts ...util.Option) (ge output += "# =====================================================================" // apply template substitutions and return output as byte array - return generator.ApplyTemplateFromFiles(generator.Mappings{ + return ApplyTemplateFromFiles(Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), "dhcp-hosts": output, }, target.TemplatePaths...) } - -var Generator DnsMasq diff --git a/pkg/generator/plugins/example/example.go b/pkg/generator/example.go similarity index 91% rename from pkg/generator/plugins/example/example.go rename to pkg/generator/example.go index 64cd3bc..b8b5c1d 100644 --- a/pkg/generator/plugins/example/example.go +++ b/pkg/generator/example.go @@ -1,4 +1,7 @@ -package main +//go:build example || plugins +// +build example plugins + +package generator import ( "fmt" diff --git a/pkg/generator/plugins/hostfile/hostfile.go b/pkg/generator/hostfile.go similarity index 80% rename from pkg/generator/plugins/hostfile/hostfile.go rename to pkg/generator/hostfile.go index 4c611f4..e998714 100644 --- a/pkg/generator/plugins/hostfile/hostfile.go +++ b/pkg/generator/hostfile.go @@ -1,10 +1,9 @@ -package main +package generator import ( "fmt" configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -22,8 +21,6 @@ func (g *Hostfile) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Hostfile) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *Hostfile) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } - -var Generator Hostfile diff --git a/pkg/generator/plugins/powerman/powerman.go b/pkg/generator/powerman.go similarity index 80% rename from pkg/generator/plugins/powerman/powerman.go rename to pkg/generator/powerman.go index 1dca29e..36be6fc 100644 --- a/pkg/generator/plugins/powerman/powerman.go +++ b/pkg/generator/powerman.go @@ -1,10 +1,9 @@ -package main +package generator import ( "fmt" configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -22,8 +21,6 @@ func (g *Powerman) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } - -var Generator Powerman diff --git a/pkg/generator/plugins/syslog/syslog.go b/pkg/generator/syslog.go similarity index 80% rename from pkg/generator/plugins/syslog/syslog.go rename to pkg/generator/syslog.go index 94ea295..463f727 100644 --- a/pkg/generator/plugins/syslog/syslog.go +++ b/pkg/generator/syslog.go @@ -1,10 +1,9 @@ -package main +package generator import ( "fmt" configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -22,8 +21,6 @@ func (g *Syslog) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } - -var Generator Syslog diff --git a/pkg/generator/plugins/warewulf/warewulf.go b/pkg/generator/warewulf.go similarity index 84% rename from pkg/generator/plugins/warewulf/warewulf.go rename to pkg/generator/warewulf.go index 8f40d7a..49b0c2c 100644 --- a/pkg/generator/plugins/warewulf/warewulf.go +++ b/pkg/generator/warewulf.go @@ -1,4 +1,4 @@ -package main +package generator import ( "fmt" @@ -6,7 +6,6 @@ import ( "strings" configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -24,13 +23,13 @@ func (g *Warewulf) GetDescription() string { return "Configurator generator plugin for 'warewulf' config files." } -func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { var ( - params = generator.GetParams(opts...) - client = generator.GetClient(params) + params = GetParams(opts...) + client = GetClient(params) targetKey = params["target"].(string) target = config.Targets[targetKey] - outputs = make(generator.FileMap, len(target.FilePaths)+len(target.TemplatePaths)) + outputs = make(FileMap, len(target.FilePaths)+len(target.TemplatePaths)) ) // check if our client is included and is valid @@ -72,11 +71,11 @@ func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (g nodeEntries := "" // load files and templates and copy to outputs - files, err := generator.LoadFiles(target.FilePaths...) + files, err := LoadFiles(target.FilePaths...) if err != nil { return nil, fmt.Errorf("failed to load files: %v", err) } - templates, err := generator.ApplyTemplateFromFiles(generator.Mappings{ + templates, err := ApplyTemplateFromFiles(Mappings{ "node_entries": nodeEntries, }, target.TemplatePaths...) if err != nil { @@ -98,5 +97,3 @@ func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (g return outputs, err } - -var Generator Warewulf From 0bbd22a5580661c394e68bae315e4c4ee3008297 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 13 Nov 2024 17:41:42 -0700 Subject: [PATCH 034/102] goreleaser: updated build command --- .goreleaser.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b0351f8..fcf715b 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -5,12 +5,14 @@ before: - go mod download - make plugins builds: - - env: - - CGO_ENABLED=1 + - id: "configurator" goos: - - linux + - linux goarch: - amd64 + - arm64 + flags: + - -tags:all archives: - format: tar.gz # this name template makes the OS and Arch compatible with the results of uname. From 9f6a8ac428fa97befb0709ab0390c981cfee440b Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 13 Nov 2024 17:42:48 -0700 Subject: [PATCH 035/102] refactor: added default plugins and check before loading --- pkg/generator/generator.go | 75 +++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 9a00545..7fff8d8 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -14,28 +14,45 @@ import ( "github.com/nikolalohinski/gonja/v2/exec" ) -type Mappings map[string]any -type FileMap map[string][]byte -type FileList [][]byte -type Template []byte +type ( + Mappings map[string]any + FileMap map[string][]byte + FileList [][]byte + Template []byte -// Generator interface used to define how files are created. Plugins can -// be created entirely independent of the main driver program. -type Generator interface { - GetName() string - GetVersion() string - GetDescription() string - Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) -} + // Generator interface used to define how files are created. Plugins can + // be created entirely independent of the main driver program. + Generator interface { + GetName() string + GetVersion() string + GetDescription() string + Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) + } -// Params defined and used by the "generate" subcommand. -type Params struct { - Args []string - Generators map[string]Generator - TemplatePaths []string - PluginPath string - Target string - Verbose bool + // Params defined and used by the "generate" subcommand. + Params struct { + Args []string + TemplatePaths []string + PluginPath string + Target string + Verbose bool + } +) + +var DefaultGenerators = createDefaultGenerators() + +func createDefaultGenerators() map[string]Generator { + var ( + generatorMap = map[string]Generator{} + generators = []Generator{ + &Conman{}, &DHCPd{}, &DNSMasq{}, &Hostfile{}, + &Powerman{}, &Syslog{}, &Warewulf{}, + } + ) + for _, g := range generators { + generatorMap[g.GetName()] = g + } + return generatorMap } // Converts the file outputs from map[string][]byte to map[string]string. @@ -397,6 +414,10 @@ func GenerateWithTarget(config *configurator.Config, params Params) (FileMap, er configurator.WithAccessToken(config.AccessToken), configurator.WithCertPoolFile(config.CertPath), ) + target configurator.Target + generator Generator + err error + ok bool ) // check if a target is supplied @@ -405,7 +426,7 @@ func GenerateWithTarget(config *configurator.Config, params Params) (FileMap, er } // load target information from config - target, ok := config.Targets[params.Target] + target, ok = config.Targets[params.Target] if !ok { return nil, fmt.Errorf("target not found in config") } @@ -415,10 +436,14 @@ func GenerateWithTarget(config *configurator.Config, params Params) (FileMap, er target.PluginPath = params.PluginPath } - // only load the plugin needed for this target - generator, err := LoadPlugin(target.PluginPath) - if err != nil { - return nil, fmt.Errorf("failed to load plugin: %w", err) + // check if generator is built-in first before loading + generator, ok = DefaultGenerators[params.Target] + if !ok { + // only load the plugin needed for this target if we don't find default + generator, err = LoadPlugin(target.PluginPath) + if err != nil { + return nil, fmt.Errorf("failed to load plugin: %w", err) + } } // run the generator plugin from target passed From ca6e4a862506b751b528e018bc36918ff1ccf5a4 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Nov 2024 13:43:28 -0700 Subject: [PATCH 036/102] goreleaser: fix typo with builds.flags --- .goreleaser.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index fcf715b..5bd35f7 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -12,7 +12,7 @@ builds: - amd64 - arm64 flags: - - -tags:all + - -tags=all archives: - format: tar.gz # this name template makes the OS and Arch compatible with the results of uname. From e93bef79f2ff362a8fb1162dbe51a2fc6a8d7c2f Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Nov 2024 13:44:24 -0700 Subject: [PATCH 037/102] cmd: moved --cacert flag to use with 'serve' command --- cmd/generate.go | 2 -- cmd/root.go | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 8ce2093..c373b9b 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -20,7 +20,6 @@ var ( tokenFetchRetries int templatePaths []string pluginPath string - cacertPath string useCompression bool ) @@ -175,7 +174,6 @@ func init() { generateCmd.Flags().StringSliceVar(&templatePaths, "template", []string{}, "set the paths for the Jinja 2 templates to use") generateCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugin path") generateCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") - generateCmd.Flags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") generateCmd.Flags().IntVar(&tokenFetchRetries, "fetch-retries", 5, "set the number of retries to fetch an access token") generateCmd.Flags().StringVar(&remoteHost, "host", "http://localhost", "set the remote host") generateCmd.Flags().IntVar(&remotePort, "port", 80, "set the remote port") diff --git a/cmd/root.go b/cmd/root.go index 26ccba7..6d323aa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,8 +10,9 @@ import ( ) var ( - configPath string config configurator.Config + configPath string + cacertPath string verbose bool targets []string outputPath string @@ -42,6 +43,7 @@ func init() { cobra.OnInitialize(initConfig) rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "./config.yaml", "set the config path") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set to enable verbose output") + rootCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") } func initConfig() { From 2a9e7c72dc7aea8b12ff6289bc7acd8445514dec Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Nov 2024 13:45:19 -0700 Subject: [PATCH 038/102] generator: added warn when default generator not found and fix error messages --- pkg/generator/generator.go | 2 ++ pkg/server/server.go | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 7fff8d8..e46b576 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -12,6 +12,7 @@ import ( "github.com/OpenCHAMI/configurator/pkg/util" "github.com/nikolalohinski/gonja/v2" "github.com/nikolalohinski/gonja/v2/exec" + "github.com/rs/zerolog/log" ) type ( @@ -440,6 +441,7 @@ func GenerateWithTarget(config *configurator.Config, params Params) (FileMap, er generator, ok = DefaultGenerators[params.Target] if !ok { // only load the plugin needed for this target if we don't find default + log.Error().Msg("did not find target in default generators") generator, err = LoadPlugin(target.PluginPath) if err != nil { return nil, fmt.Errorf("failed to load plugin: %w", err) diff --git a/pkg/server/server.go b/pkg/server/server.go index 89e0d71..66fdc58 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -76,7 +76,7 @@ func (s *Server) Serve() error { var err error tokenAuth, err = configurator.FetchPublicKeyFromURL(s.Config.Server.Jwks.Uri) if err != nil { - logrus.Errorf("failed to fetch JWKS: %w", err) + logrus.Errorf("failed to fetch JWKS: %v", err) continue } break @@ -138,7 +138,7 @@ func (s *Server) Generate(w http.ResponseWriter, r *http.Request) { // generate a new config file from supplied params outputs, err := generator.GenerateWithTarget(s.Config, s.GeneratorParams) if err != nil { - writeErrorResponse(w, "failed to generate file: %w", err) + writeErrorResponse(w, "failed to generate file: %v", err) return } @@ -146,12 +146,12 @@ func (s *Server) Generate(w http.ResponseWriter, r *http.Request) { tmp := generator.ConvertContentsToString(outputs) b, err := json.Marshal(tmp) if err != nil { - writeErrorResponse(w, "failed to marshal output: %w", err) + writeErrorResponse(w, "failed to marshal output: %v", err) return } _, err = w.Write(b) if err != nil { - writeErrorResponse(w, "failed to write response: %w", err) + writeErrorResponse(w, "failed to write response: %v", err) return } } @@ -163,7 +163,7 @@ func (s *Server) Generate(w http.ResponseWriter, r *http.Request) { func (s *Server) ManageTemplates(w http.ResponseWriter, r *http.Request) { _, err := w.Write([]byte("this is not implemented yet")) if err != nil { - writeErrorResponse(w, "failed to write response: %w", err) + writeErrorResponse(w, "failed to write response: %v", err) return } } From 4bd4dac1297b9bf4243d438b8b76e19bcec1dcc8 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Nov 2024 16:21:04 -0700 Subject: [PATCH 039/102] cmd: removed flag and added check for cacert --- cmd/serve.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index f7e4401..507d330 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -4,11 +4,15 @@ package cmd import ( + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" + "net" "net/http" "os" + "time" "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/server" @@ -55,13 +59,33 @@ var serveCmd = &cobra.Command{ Retries: config.Server.Jwks.Retries, }, GeneratorParams: generator.Params{ - Args: args, - PluginPath: pluginPath, + Args: args, + // PluginPath: pluginPath, // Target: target, // NOTE: targets are set via HTTP requests (ex: curl http://configurator:3334/generate?target=dnsmasq) Verbose: verbose, }, } + // add cert to client if `--cacert` flag is passed + if cacertPath != "" { + cacert, _ := os.ReadFile(cacertPath) + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + server.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + } + // start listening with the server err := server.Serve() if errors.Is(err, http.ErrServerClosed) { @@ -78,7 +102,7 @@ var serveCmd = &cobra.Command{ func init() { serveCmd.Flags().StringVar(&config.Server.Host, "host", config.Server.Host, "set the server host") serveCmd.Flags().IntVar(&config.Server.Port, "port", config.Server.Port, "set the server port") - serveCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugins directory path") + // serveCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugins directory path") serveCmd.Flags().StringVar(&config.Server.Jwks.Uri, "jwks-uri", config.Server.Jwks.Uri, "set the JWKS url to fetch public key") serveCmd.Flags().IntVar(&config.Server.Jwks.Retries, "jwks-fetch-retries", config.Server.Jwks.Retries, "set the JWKS fetch retry count") rootCmd.AddCommand(serveCmd) From b5d492d6c01d20135412206d62cad86433c52f62 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Nov 2024 16:26:17 -0700 Subject: [PATCH 040/102] cmd: added error when specifying config path but not found --- cmd/root.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 6d323aa..ab190da 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,6 +6,7 @@ import ( configurator "github.com/OpenCHAMI/configurator/pkg" "github.com/OpenCHAMI/configurator/pkg/util" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -41,12 +42,13 @@ func Execute() { func init() { cobra.OnInitialize(initConfig) - rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "./config.yaml", "set the config path") + rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config path") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set to enable verbose output") rootCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") } func initConfig() { + // empty from not being set if configPath != "" { exists, err := util.PathExists(configPath) if err != nil { @@ -55,9 +57,13 @@ func initConfig() { } else if exists { config = configurator.LoadConfig(configPath) } else { - config = configurator.NewConfig() + // show error and exit since a path was specified + log.Error().Str("path", configPath).Msg("config file not found") + os.Exit(1) } } else { + // set to the default value and create a new one + configPath = "./config.yaml" config = configurator.NewConfig() } From 043f8ec1201b078d2a003165b1a8c9a35508cfd2 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Nov 2024 16:30:26 -0700 Subject: [PATCH 041/102] goreleaser/docker: removed lib/ refs and other minor changes --- .goreleaser.yaml | 2 -- Dockerfile | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 5bd35f7..d91d3e3 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -27,7 +27,6 @@ archives: - LICENSE - CHANGELOG.md - README.md - - lib/ dockers: - image_templates: @@ -45,7 +44,6 @@ dockers: - LICENSE - CHANGELOG.md - README.md - - lib/ checksum: name_template: 'checksums.txt' snapshot: diff --git a/Dockerfile b/Dockerfile index b36b37d..3a1e3a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,15 +2,13 @@ FROM cgr.dev/chainguard/wolfi-base RUN apk add --no-cache tini bash RUN mkdir -p /configurator -RUN mkdir -p /configurator/lib # nobody 65534:65534 USER 65534:65534 # copy the binary and all of the default plugins COPY configurator /configurator/configurator -COPY lib/* /configurator/lib/* -CMD ["/configurator"] +CMD ["/configurator/configurator"] ENTRYPOINT [ "/sbin/tini", "--" ] \ No newline at end of file From dac6c2306f7a3c9d5ae3f03c76b6a27f537f7e90 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Nov 2024 16:53:00 -0700 Subject: [PATCH 042/102] client: moved cacert logic from 'serve' cmd to client --- cmd/serve.go | 28 +------------ pkg/generator/generator.go | 18 ++++++--- pkg/server/server.go | 83 +++++++++++++++++++++++++------------- 3 files changed, 70 insertions(+), 59 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index 507d330..647482b 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -4,15 +4,11 @@ package cmd import ( - "crypto/tls" - "crypto/x509" "encoding/json" "errors" "fmt" - "net" "net/http" "os" - "time" "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/server" @@ -48,7 +44,7 @@ var serveCmd = &cobra.Command{ fmt.Printf("%v\n", string(b)) } - // set up the routes and start the server + // set up the routes and start the serve server := server.Server{ Config: &config, Server: &http.Server{ @@ -66,28 +62,8 @@ var serveCmd = &cobra.Command{ }, } - // add cert to client if `--cacert` flag is passed - if cacertPath != "" { - cacert, _ := os.ReadFile(cacertPath) - certPool := x509.NewCertPool() - certPool.AppendCertsFromPEM(cacert) - server.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: certPool, - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - Dial: (&net.Dialer{ - Timeout: 120 * time.Second, - KeepAlive: 120 * time.Second, - }).Dial, - TLSHandshakeTimeout: 120 * time.Second, - ResponseHeaderTimeout: 120 * time.Second, - } - } - // start listening with the server - err := server.Serve() + err := server.Serve(cacertPath) if errors.Is(err, http.ErrServerClosed) { if verbose { fmt.Printf("Server closed.") diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index e46b576..e705e16 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -36,6 +36,7 @@ type ( TemplatePaths []string PluginPath string Target string + Client *configurator.SmdClient Verbose bool } ) @@ -409,17 +410,24 @@ func Generate(config *configurator.Config, params Params) (FileMap, error) { func GenerateWithTarget(config *configurator.Config, params Params) (FileMap, error) { // load generator plugins to generate configs or to print var ( + client configurator.SmdClient + target configurator.Target + generator Generator + err error + ok bool + ) + + // check if we have a client from params first and create new one if not + if params.Client == nil { client = configurator.NewSmdClient( configurator.WithHost(config.SmdClient.Host), configurator.WithPort(config.SmdClient.Port), configurator.WithAccessToken(config.AccessToken), configurator.WithCertPoolFile(config.CertPath), ) - target configurator.Target - generator Generator - err error - ok bool - ) + } else { + client = *params.Client + } // check if a target is supplied if len(params.Args) == 0 && params.Target == "" { diff --git a/pkg/server/server.go b/pkg/server/server.go index 66fdc58..9afc7db 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -4,8 +4,11 @@ package server import ( + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" + "net" "net/http" "os" "time" @@ -60,13 +63,33 @@ func New(config *configurator.Config) *Server { } // Main function to start up configurator as a service. -func (s *Server) Serve() error { +func (s *Server) Serve(cacertPath string) error { // create client just for the server to use to fetch data from SMD - _ = &configurator.SmdClient{ + client := &configurator.SmdClient{ Host: s.Config.SmdClient.Host, Port: s.Config.SmdClient.Port, } + // add cert to client if `--cacert` flag is passed + if cacertPath != "" { + cacert, _ := os.ReadFile(cacertPath) + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + } + // set the server address with config values s.Server.Addr = fmt.Sprintf("%s:%d", s.Config.Server.Host, s.Config.Server.Port) @@ -104,12 +127,12 @@ func (s *Server) Serve() error { ) // protected routes if using auth - r.HandleFunc("/generate", s.Generate) + r.HandleFunc("/generate", s.Generate(client)) r.HandleFunc("/templates", s.ManageTemplates) }) } else { // public routes without auth - router.HandleFunc("/generate", s.Generate) + router.HandleFunc("/generate", s.Generate(client)) router.HandleFunc("/templates", s.ManageTemplates) } @@ -127,32 +150,36 @@ func (s *Server) Close() { // This is the corresponding service function to generate templated files, that // works similarly to the CLI variant. This function takes similiar arguments as // query parameters that are included in the HTTP request URL. -func (s *Server) Generate(w http.ResponseWriter, r *http.Request) { - // get all of the expect query URL params and validate - s.GeneratorParams.Target = r.URL.Query().Get("target") - if s.GeneratorParams.Target == "" { - writeErrorResponse(w, "must specify a target") - return - } +func (s *Server) Generate(client *configurator.SmdClient) func(w http.ResponseWriter, r *http.Request) { - // generate a new config file from supplied params - outputs, err := generator.GenerateWithTarget(s.Config, s.GeneratorParams) - if err != nil { - writeErrorResponse(w, "failed to generate file: %v", err) - return - } + return func(w http.ResponseWriter, r *http.Request) { + // get all of the expect query URL params and validate + s.GeneratorParams.Target = r.URL.Query().Get("target") + s.GeneratorParams.Client = client + if s.GeneratorParams.Target == "" { + writeErrorResponse(w, "must specify a target") + return + } - // marshal output to JSON then send response to client - tmp := generator.ConvertContentsToString(outputs) - b, err := json.Marshal(tmp) - if err != nil { - writeErrorResponse(w, "failed to marshal output: %v", err) - return - } - _, err = w.Write(b) - if err != nil { - writeErrorResponse(w, "failed to write response: %v", err) - return + // generate a new config file from supplied params + outputs, err := generator.GenerateWithTarget(s.Config, s.GeneratorParams) + if err != nil { + writeErrorResponse(w, "failed to generate file: %v", err) + return + } + + // marshal output to JSON then send response to client + tmp := generator.ConvertContentsToString(outputs) + b, err := json.Marshal(tmp) + if err != nil { + writeErrorResponse(w, "failed to marshal output: %v", err) + return + } + _, err = w.Write(b) + if err != nil { + writeErrorResponse(w, "failed to write response: %v", err) + return + } } } From 516f100075da089e3991d1d1a26c578ffe69592a Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Nov 2024 17:06:49 -0700 Subject: [PATCH 043/102] chore: moved contents of dist/ to res/ --- {dist => res}/archlinux/PKGBUILD | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {dist => res}/archlinux/PKGBUILD (100%) diff --git a/dist/archlinux/PKGBUILD b/res/archlinux/PKGBUILD similarity index 100% rename from dist/archlinux/PKGBUILD rename to res/archlinux/PKGBUILD From f0c48d5d77260e8ad03b3380e999fcfefa5eb12c Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 14 Nov 2024 17:07:26 -0700 Subject: [PATCH 044/102] gitignore: added dist/ to file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e848d1d..a51dd2f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ **.conf **.ignore **.tar.gz +dist/ From b31056f2974cf9f300001555d4bd854ee859232a Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 21 Nov 2024 14:10:27 -0700 Subject: [PATCH 045/102] readme: update information --- README.md | 107 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index acfcb36..c7969f7 100644 --- a/README.md +++ b/README.md @@ -1,66 +1,78 @@ # OpenCHAMI Configurator -The `configurator` (portmanteau of config + generator) is an extensible tool that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate commonly used config files based on Jinja 2 template files. The tool and generator plugins are written in Go and plugins can be written by following the ["Creating Generator Plugins"](#creating-generator-plugins) section of this README. +The `configurator` is an extensible tool that is capable of dynamically generating files on the fly. The tool includes a built-in generator that fetchs data from an instance of [SMD](https://github.com/OpenCHAMI/smd) to generate files based on Jinja 2 template files. The tool and generator plugins are written in Go and plugins can be written by following the ["Creating Generator Plugins"](#creating-generator-plugins) section of this README. ## Building and Usage -The `configurator` is built using standard `go` build tools. The project separates the client and server with build tags. To get started, clone the project, download the dependencies, and build the project: +The `configurator` is built using standard `go` build tools. The project separates the client and server components using build tags. To get started, clone the project, download the dependencies, and build the project: ```bash git clone https://github.com/OpenCHAMI/configurator.git go mod tidy go build --tags all # equivalent to `go build --tags client,server`` - -## ...or just run `make` in project directory ``` -This will build the main driver program, but also requires generator plugins to define how new config files are generated. The default plugins can be built using the following build command: +This will build the main driver program with the default generators that are found in the `pkg/generators` directory. + +> [!WARNING] +> Not all of the plugins have completed generation implementations and are a WIP. + +### Running Configurator with CLI + +After you build the program, run the following command to use the tool: ```bash -go build -buildmode=plugin -o lib/conman.so internal/generator/plugins/conman/conman.go -go build -buildmode=plugin -o lib/coredhcp.so internal/generator/plugins/coredhcp/coredhcp.go -go build -buildmode=plugin -o lib/dnsmasq.so internal/generator/plugins/dnsmasq/dnsmasq.go -go build -buildmode=plugin -o lib/powerman.so internal/generator/plugins/powerman/powerman.go -go build -buildmode=plugin -o lib/syslog.so internal/generator/plugins/syslog/syslog.go +export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... +./configurator generate --config config.yaml --target dnsmasq -o dnsmasq.conf --cacert configurator.pem ``` -**NOTE: Not all of the plugins have completed generation implementations and are WIP.** +This will generate a new `dnsmasq` config file based on the Jinja 2 template specified in the config file for "dnsmasq". The files will be written to `dnsmasq.conf` as specified with the `-o/--output` flag. The `--target` flag specifies the type of config file to generate by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). -These commands will build the default plugins and store them in the "lib" directory by default. Alternatively, the plugins can be built using `make plugins` if GNU make is installed and available. After you build the plugins, run the following to use the tool: +In other words, there should be an entry in the config file that looks like this: + +```yaml +... +targets: + dnsmasq: + plugin: "lib/dnsmasq.so" # optional, if we want to use a plugin instead + templates: + - templates/dnsmasq.j2 +... -```bash -./configurator generate --config config.yaml --target dnsmasq -o dnsmasq.conf ``` -This will generate a new `dnsmasq` config file based on the Jinja 2 template specified in the config file for "dnsmasq". The `--target` flag specifies the type of config file to generate by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `configurator` tool requires a valid access token when making requests to an instance of SMD that has protected routes. +> [!NOTE] +> The `configurator` tool requires a valid access token when making requests to an instance of SMD that has protected routes. + +### Running Configurator as a Service The tool can also run as a service to generate files for clients: ```bash +export CONFIGURATOR_JWKS_URL="http://my.openchami.cluster:8443/key" ./configurator serve --config config.yaml ``` Once the server is up and listening for HTTP requests, you can try making a request to it with `curl` or `configurator fetch`. Both commands below are essentially equivalent: ```bash -curl http://127.0.0.1:3334/generate?target=dnsmasq -H "Authorization: Bearer $ACCESS_TOKEN" +export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... +curl http://127.0.0.1:3334/generate?target=dnsmasq -X GET -H "Authorization: Bearer $ACCESS_TOKEN" --cacert configurator.pem # ...or... -./configurator fetch --target dnsmasq --host http://127.0.0.1 --port 3334 +./configurator fetch --target dnsmasq --host http://127.0.0.1:3334 --cacert configurator.pem ``` -This will do the same thing as the `generate` subcommand, but remotely. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set. The `ACCESS_TOKEN` environment variable passed to `curl` and it's corresponding CLI argument both expects a token as a JWT. +This will do the same thing as the `generate` subcommand, but through a GET request where the file contents is returned in the response. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set when starting the server with `serve`. The `ACCESS_TOKEN` environment variable is passed to `curl` using the `Authorization` header and expects a token as a JWT. ### Docker -New images can be built and tested using the `Dockerfile` provided in the project. However, the binary executable and the generator plugins must first be built before building the image since the Docker build copies the binary over. Therefore, build all of the binaries first by following the first section of ["Building and Usage"](#building-and-usage). If you run the `make docker`, this will be done for you. Otherwise, run the `docker build` command after building the executable and libraries. +New images can be built and tested using the `Dockerfile` provided in the project. However, the binary executable and the generator plugins must first be built before building the image since the Docker build copies the binary over. Therefore, build all of the binaries first by following the first section of ["Building and Usage"](#building-and-usage). Running `make docker` from the Makefile will automate this process. Otherwise, run the `docker build` command after building the executable and libraries. ```bash docker build -t configurator:testing path/to/configurator/Dockerfile -# ...or -make docker ``` -Keep in mind that all plugins included in the project are build in the `lib/` directory and copied from there. If you want to easily include your own external generator plugins, you can build it and copy the `lib.so` file to that location. Make sure that the `Generator` interface is implemented correct as described in the ["Creating Generator Plugins"](#creating-generator-plugins) or the plugin will not load. Additionally, the name string returned from the `GetName()` method is used for looking up the plugin after all plugins have been loaded by the main driver. +If you want to easily include your own external generator plugins, you can build it and copy the `lib.so` file to `lib/`. Make sure that the `Generator` interface is implemented correctly as described in the ["Creating Generator Plugins"](#creating-generator-plugins) or the plugin will not load (you should get an error that specifically says this). Additionally, the name string returned from the `GetName()` method is used for looking up the plugin with the `--target` flag by the main driver program. Alternatively, pull the latest existing image/container from the GitHub container repository. @@ -68,23 +80,27 @@ Alternatively, pull the latest existing image/container from the GitHub containe docker pull ghcr.io/openchami/configurator:latest ``` -Then, run the container similarly to the binary. +Then, run the Docker container similarly to running the binary. -``` -docker run ghcr.io/openchami/configurator:latest configurator generate --config config.yaml --target dnsmasq +```bash +export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... +docker run ghcr.io/openchami/configurator:latest configurator generate --config config.yaml --target dnsmasq -o dnsmasq.conf --cacert configurator.pem ``` ### Creating Generator Plugins -The `configurator` uses generator plugins to define how config files are generated using a `Generator` interface. The interface is defined like so: +The `configurator` uses built-in and user-defined generators that implement the `Generator` interface to describe how config files should be generated. The interface is defined like so: ```go -type Files = map[string][]byte +// maps the file path to its contents +type FileMap = map[string][]byte + +// interface for generator plugins type Generator interface { GetName() string GetVersion() string GetDescription() string - Generate(config *configurator.Config, opts ...util.Option) (Files, error) + Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) } ``` @@ -93,23 +109,30 @@ A new plugin can be created by implementing the methods from interface and expor ```go package main -type MyGenerator struct {} +type MyGenerator struct { + PluginInfo map[string]any +} + +var pluginInfo map[string]any + +// this function is not a part of the `Generator` interface +func (g *MyGenerator) LoadFromFile() map[string]any{ /*...*/ } func (g *MyGenerator) GetName() string { // just an example...this can be done however you want - pluginInfo := LoadFromFile("path/to/plugin/info.json") - return pluginInfo["name"] + g.PluginInfo := LoadFromFile("path/to/plugin/info.json") + return g.PluginInfo["name"] } func (g *MyGenerator) GetVersion() string { - return "v1.0.0" + return g.PluginInfo["version"] // "v1.0.0" } func (g *MyGenerator) GetDescription() string { - return "This is an example plugin." + return g.PluginInfo["description"] // "This is an example plugin." } -func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) (generator.Files, error) { +func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { // do config generation stuff here... var ( params = generator.GetParams(opts...) @@ -121,7 +144,7 @@ func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) // ... blah, blah, blah, check error, format output, and so on... } - // apply the substitutions to Jinja template and return output as byte array + // apply the substitutions to Jinja template and return output as FileMap (i.e. path and it's contents) return generator.ApplyTemplate(path, generator.Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), @@ -140,7 +163,7 @@ Finally, build the plugin and put it somewhere specified by `plugins` in your co go build -buildmode=plugin -o lib/mygenerator.so path/to/mygenerator.go ``` -Now your plugin should be available to use with the `configurator` main driver. If you get an error about not loading the correct symbol type, make sure that you generator function definitions match the `Generator` interface exactly. +Now your plugin should be available to use with the `configurator` main driver program. If you get an error about not loading the correct symbol type, make sure that your generator function definitions match the `Generator` interface entirely and that you don't have a partially implemented interface. ## Configuration @@ -150,10 +173,10 @@ Here is an example config file to start using configurator: server: # Server-related parameters when using as service host: 127.0.0.1 port: 3334 - jwks: # Set the JWKS uri to protect /generate route + jwks: # Set the JWKS uri for protected routes uri: "" retries: 5 -smd: . # SMD-related parameters +smd: # SMD-related parameters host: http://127.0.0.1 port: 27779 plugins: # path to plugin directories @@ -173,20 +196,18 @@ targets: # targets to call with --target flag - templates/warewulf/vnfs/* - templates/warewulf/bootstrap.jinja - templates/warewulf/database.jinja - targets: # additional targets to run + targets: # additional targets to run (does not run recursively) - dnsmasq ``` -The `server` section sets the properties for running the `configurator` tool as a service and is not required if you're only using the CLI. Also note that the `jwks-uri` parameter is only needs for protecting endpoints. If it is not set, then the API is entirely public. The `smd` section tells the `configurator` tool where to find SMD to pull state management data used by the internal client. The `templates` section is where the paths are mapped to each generator plugin by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `plugins` is a list of paths to load generator plugins. +The `server` section sets the properties for running the `configurator` tool as a service and is not required if you're only using the CLI. Also note that the `jwks.uri` parameter is only needed for protecting endpoints. If it is not set, then all API routes are entirely public. The `smd` section tells the `configurator` tool where to find the SMD service to pull state management data used internally by the client's generator. The `templates` section is where the paths are mapped to each generator by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). The `plugins` is a list of paths to search for and load external generator plugins. ## Running the Tests -The `configurator` project includes a collection of tests focused on verifying plugin behavior and generating files. The tests do not currently test fetching information from SMD (or whatever remote source). The tests can be ran with either of the following commands: +The `configurator` project includes a collection of tests focused on verifying plugin behavior and generating files. The tests do not include fetching information from any remote sources, can be ran with the following command: ```bash go test ./tests/generate_test.go --tags=all -# ...or alternatively with GNU make... -make test ``` From 32065dc163a662357c656702709792f6fe0e8c6f Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 21 Nov 2024 14:13:31 -0700 Subject: [PATCH 046/102] refactor: name changes and code clean up --- cmd/config.go | 4 +- cmd/fetch.go | 9 +-- cmd/generate.go | 196 +++++++++++++++++++++++++----------------------- cmd/root.go | 13 ++-- cmd/serve.go | 47 +++++------- 5 files changed, 134 insertions(+), 135 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 1de4015..05e183d 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -5,7 +5,7 @@ import ( "github.com/spf13/cobra" - configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -23,7 +23,7 @@ var configCmd = &cobra.Command{ fmt.Printf("file or directory exists\n") continue } - configurator.SaveDefaultConfig(path) + config.SaveDefault(path) } }, } diff --git a/cmd/fetch.go b/cmd/fetch.go index c2ba4e5..0bbf309 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -25,11 +25,11 @@ var fetchCmd = &cobra.Command{ } // check to see if an access token is available from env - if config.AccessToken == "" { + if conf.AccessToken == "" { // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead accessToken := os.Getenv("ACCESS_TOKEN") if accessToken != "" { - config.AccessToken = accessToken + conf.AccessToken = accessToken } else { // TODO: try and fetch token first if it is needed if verbose { @@ -46,7 +46,7 @@ var fetchCmd = &cobra.Command{ for _, target := range targets { // make a request for each target - url := fmt.Sprintf("%s:%d/generate?target=%s", remoteHost, remotePort, target) + url := fmt.Sprintf("%s/generate?target=%s", remoteHost, target) res, body, err := util.MakeRequest(url, http.MethodGet, nil, headers) if err != nil { log.Error().Err(err).Msg("failed to make request") @@ -63,8 +63,7 @@ var fetchCmd = &cobra.Command{ } func init() { - fetchCmd.Flags().StringVar(&remoteHost, "host", "", "set the remote configurator host") - fetchCmd.Flags().IntVar(&remotePort, "port", 3334, "set the remote configurator port") + fetchCmd.Flags().StringVar(&remoteHost, "host", "", "set the remote configurator host and port") fetchCmd.Flags().StringSliceVar(&targets, "target", nil, "set the target configs to make") fetchCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") fetchCmd.Flags().StringVar(&accessToken, "access-token", "o", "set the output path for config targets") diff --git a/cmd/generate.go b/cmd/generate.go index c373b9b..634a775 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -9,7 +9,7 @@ import ( "os" "path/filepath" - configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" "github.com/rs/zerolog/log" @@ -28,155 +28,165 @@ var generateCmd = &cobra.Command{ Short: "Generate a config file from state management", Run: func(cmd *cobra.Command, args []string) { // make sure that we have a token present before trying to make request - if config.AccessToken == "" { + if conf.AccessToken == "" { // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead accessToken := os.Getenv("ACCESS_TOKEN") if accessToken != "" { - config.AccessToken = accessToken + conf.AccessToken = accessToken } else { // TODO: try and fetch token first if it is needed if verbose { - fmt.Printf("No token found. Attempting to generate config without one...\n") + fmt.Printf("No token found. Attempting to generate conf without one...\n") } } } // use cert path from cobra if empty - if config.CertPath == "" { - config.CertPath = cacertPath + if conf.CertPath == "" { + conf.CertPath = cacertPath } - // show config as JSON and generators if verbose + // show conf as JSON and generators if verbose if verbose { - b, err := json.MarshalIndent(config, "", " ") + b, err := json.MarshalIndent(conf, "", " ") if err != nil { - fmt.Printf("failed to marshal config: %v\n", err) + fmt.Printf("failed to marshal conf: %v\n", err) } fmt.Printf("%v\n", string(b)) } // run all of the target recursively until completion if provided if len(targets) > 0 { - RunTargets(&config, args, targets...) + RunTargets(&conf, args, targets...) } else { if pluginPath == "" { fmt.Printf("no plugin path specified") return } - // run generator.Generate() with just plugin path and templates provided - generator.Generate(&config, generator.Params{ - PluginPath: pluginPath, - TemplatePaths: templatePaths, - }) + // load the templates to use + templates := map[string]generator.Template{} + for _, path := range templatePaths { + template := generator.Template{} + template.LoadFromFile(path) + if !template.IsEmpty() { + templates[path] = template + } + } + params := generator.Params{ + Templates: templates, + } + + // run generator.Generate() with just plugin path and templates provided + outputBytes, err := generator.Generate(pluginPath, params) + if err != nil { + log.Error().Err(err).Msg("failed to generate files") + } + + // if we have more than one target and output is set, create configs in directory + writeOutput(outputBytes, len(targets), len(outputMap)) } }, } // Generate files by supplying a list of targets as string values. Currently, -// targets are defined statically in a config file. Targets are ran recursively +// targets are defined statically in a conf file. Targets are ran recursively // if more targets are nested in a defined target, but will not run additional // child targets if it is the same as the parent. // // NOTE: This may be changed in the future how this is done. -func RunTargets(config *configurator.Config, args []string, targets ...string) { - // generate config with each supplied target +func RunTargets(conf *config.Config, args []string, targets ...string) { + // generate conf with each supplied target for _, target := range targets { - outputBytes, err := generator.GenerateWithTarget(config, generator.Params{ - Args: args, - PluginPath: pluginPath, - Target: target, - Verbose: verbose, - }) + outputBytes, err := generator.GenerateWithTarget(conf, target) if err != nil { - log.Error().Err(err).Msg("failed to generate config") + log.Error().Err(err).Msg("failed to generate conf") os.Exit(1) } // if we have more than one target and output is set, create configs in directory - var ( - outputMap = generator.ConvertContentsToString(outputBytes) - targetCount = len(targets) - templateCount = len(outputMap) - ) - if outputPath == "" { - // write only to stdout by default - if len(outputMap) == 1 { - for _, contents := range outputMap { - fmt.Printf("%s\n", string(contents)) - } - } else { - for path, contents := range outputMap { - fmt.Printf("-- file: %s, size: %d B\n%s\n", path, len(contents), string(contents)) - } - } - } else if outputPath != "" && targetCount == 1 && templateCount == 1 { - // write just a single file using provided name - for _, contents := range outputBytes { - err := os.WriteFile(outputPath, contents, 0o644) - if err != nil { - log.Error().Err(err).Msg("failed to write config to file") - os.Exit(1) - } - log.Info().Msgf("wrote file to '%s'\n", outputPath) - } - } else if outputPath != "" && targetCount > 1 && useCompression { - // write multiple files to archive, compress, then save to output path - out, err := os.Create(fmt.Sprintf("%s.tar.gz", outputPath)) - if err != nil { - log.Error().Err(err).Msg("failed to write archive") - os.Exit(1) - } - files := make([]string, len(outputBytes)) - i := 0 - for path := range outputBytes { - files[i] = path - i++ - } - err = util.CreateArchive(files, out) - if err != nil { - log.Error().Err(err).Msg("failed to create archive") - os.Exit(1) - } - - } else if outputPath != "" && targetCount > 1 || templateCount > 1 { - // write multiple files in directory using template name - err := os.MkdirAll(filepath.Clean(outputPath), 0o755) - if err != nil { - log.Error().Err(err).Msg("failed to make output directory") - os.Exit(1) - } - for path, contents := range outputBytes { - filename := filepath.Base(path) - cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) - err := os.WriteFile(cleanPath, contents, 0o755) - if err != nil { - log.Error().Err(err).Msg("failed to write config to file") - os.Exit(1) - } - log.Info().Msgf("wrote file to '%s'\n", cleanPath) - } - } + writeOutput(outputBytes, len(targets), len(outputMap)) // remove any targets that are the same as current to prevent infinite loop - nextTargets := util.CopyIf(config.Targets[target].RunTargets, func(nextTarget string) bool { + nextTargets := util.CopyIf(conf.Targets[target].RunTargets, func(nextTarget string) bool { return nextTarget != target }) // ...then, run any other targets that the current target has - RunTargets(config, args, nextTargets...) + RunTargets(conf, args, nextTargets...) + } +} + +func writeOutput(outputBytes generator.FileMap, targetCount int, templateCount int) { + outputMap := generator.ConvertContentsToString(outputBytes) + if outputPath == "" { + // write only to stdout by default + if len(outputMap) == 1 { + for _, contents := range outputMap { + fmt.Printf("%s\n", string(contents)) + } + } else { + for path, contents := range outputMap { + fmt.Printf("-- file: %s, size: %d B\n%s\n", path, len(contents), string(contents)) + } + } + } else if outputPath != "" && targetCount == 1 && templateCount == 1 { + // write just a single file using provided name + for _, contents := range outputBytes { + err := os.WriteFile(outputPath, contents, 0o644) + if err != nil { + log.Error().Err(err).Msg("failed to write conf to file") + os.Exit(1) + } + log.Info().Msgf("wrote file to '%s'\n", outputPath) + } + } else if outputPath != "" && targetCount > 1 && useCompression { + // write multiple files to archive, compress, then save to output path + out, err := os.Create(fmt.Sprintf("%s.tar.gz", outputPath)) + if err != nil { + log.Error().Err(err).Msg("failed to write archive") + os.Exit(1) + } + files := make([]string, len(outputBytes)) + i := 0 + for path := range outputBytes { + files[i] = path + i++ + } + err = util.CreateArchive(files, out) + if err != nil { + log.Error().Err(err).Msg("failed to create archive") + os.Exit(1) + } + + } else if outputPath != "" && targetCount > 1 || templateCount > 1 { + // write multiple files in directory using template name + err := os.MkdirAll(filepath.Clean(outputPath), 0o755) + if err != nil { + log.Error().Err(err).Msg("failed to make output directory") + os.Exit(1) + } + for path, contents := range outputBytes { + filename := filepath.Base(path) + cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) + err := os.WriteFile(cleanPath, contents, 0o755) + if err != nil { + log.Error().Err(err).Msg("failed to write conf to file") + os.Exit(1) + } + log.Info().Msgf("wrote file to '%s'\n", cleanPath) + } } } func init() { - generateCmd.Flags().StringSliceVar(&targets, "target", []string{}, "set the targets to run pre-defined config") + generateCmd.Flags().StringSliceVar(&targets, "target", []string{}, "set the targets to run pre-defined conf") generateCmd.Flags().StringSliceVar(&templatePaths, "template", []string{}, "set the paths for the Jinja 2 templates to use") generateCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugin path") - generateCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for config targets") + generateCmd.Flags().StringVarP(&outputPath, "output", "o", "", "set the output path for conf targets") generateCmd.Flags().IntVar(&tokenFetchRetries, "fetch-retries", 5, "set the number of retries to fetch an access token") generateCmd.Flags().StringVar(&remoteHost, "host", "http://localhost", "set the remote host") - generateCmd.Flags().IntVar(&remotePort, "port", 80, "set the remote port") generateCmd.Flags().BoolVar(&useCompression, "compress", false, "set whether to archive and compress multiple file outputs") // requires either 'target' by itself or 'plugin' and 'templates' together diff --git a/cmd/root.go b/cmd/root.go index ab190da..be9659c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,14 +4,14 @@ import ( "fmt" "os" - configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) var ( - config configurator.Config + conf config.Config configPath string cacertPath string verbose bool @@ -19,12 +19,11 @@ var ( outputPath string accessToken string remoteHost string - remotePort int ) var rootCmd = &cobra.Command{ Use: "configurator", - Short: "Tool for building common config files", + Short: "Dynamically generate files defined by generators", Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { cmd.Help() @@ -55,7 +54,7 @@ func initConfig() { fmt.Printf("failed to load config") os.Exit(1) } else if exists { - config = configurator.LoadConfig(configPath) + conf = config.Load(configPath) } else { // show error and exit since a path was specified log.Error().Str("path", configPath).Msg("config file not found") @@ -64,7 +63,7 @@ func initConfig() { } else { // set to the default value and create a new one configPath = "./config.yaml" - config = configurator.NewConfig() + conf = config.New() } // @@ -74,6 +73,6 @@ func initConfig() { // set the JWKS url if we find the CONFIGURATOR_JWKS_URL environment variable jwksUrl := os.Getenv("CONFIGURATOR_JWKS_URL") if jwksUrl != "" { - config.Server.Jwks.Uri = jwksUrl + conf.Server.Jwks.Uri = jwksUrl } } diff --git a/cmd/serve.go b/cmd/serve.go index 647482b..61bfeff 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -10,8 +10,8 @@ import ( "net/http" "os" - "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/server" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -20,66 +20,57 @@ var serveCmd = &cobra.Command{ Short: "Start configurator as a server and listen for requests", Run: func(cmd *cobra.Command, args []string) { // make sure that we have a token present before trying to make request - if config.AccessToken == "" { - // TODO: make request to check if request will need token - - // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead + if conf.AccessToken == "" { + // check if ACCESS_TOKEN env var is set if no access token is provided and use that instead accessToken := os.Getenv("ACCESS_TOKEN") if accessToken != "" { - config.AccessToken = accessToken + conf.AccessToken = accessToken } else { - // TODO: try and fetch token first if it is needed if verbose { - fmt.Printf("No token found. Attempting to generate config without one...\n") + fmt.Printf("No token found. Continuing without one...\n") } } } - // show config as JSON and generators if verbose + // show conf as JSON and generators if verbose if verbose { - b, err := json.MarshalIndent(config, "", " ") + b, err := json.MarshalIndent(conf, "", "\t") if err != nil { - fmt.Printf("failed to marshal config: %v\n", err) + log.Error().Err(err).Msg("failed to marshal config") } fmt.Printf("%v\n", string(b)) } // set up the routes and start the serve server := server.Server{ - Config: &config, + Config: &conf, Server: &http.Server{ - Addr: fmt.Sprintf("%s:%d", config.Server.Host, config.Server.Port), + Addr: fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port), }, Jwks: server.Jwks{ - Uri: config.Server.Jwks.Uri, - Retries: config.Server.Jwks.Retries, - }, - GeneratorParams: generator.Params{ - Args: args, - // PluginPath: pluginPath, - // Target: target, // NOTE: targets are set via HTTP requests (ex: curl http://configurator:3334/generate?target=dnsmasq) - Verbose: verbose, + Uri: conf.Server.Jwks.Uri, + Retries: conf.Server.Jwks.Retries, }, } // start listening with the server - err := server.Serve(cacertPath) + err := server.Serve() if errors.Is(err, http.ErrServerClosed) { if verbose { - fmt.Printf("Server closed.") + log.Info().Msg("server closed") } } else if err != nil { - fmt.Errorf("failed to start server: %v", err) + log.Error().Err(err).Msg("failed to start server") os.Exit(1) } }, } func init() { - serveCmd.Flags().StringVar(&config.Server.Host, "host", config.Server.Host, "set the server host") - serveCmd.Flags().IntVar(&config.Server.Port, "port", config.Server.Port, "set the server port") + serveCmd.Flags().StringVar(&conf.Server.Host, "host", conf.Server.Host, "set the server host") + serveCmd.Flags().IntVar(&conf.Server.Port, "port", conf.Server.Port, "set the server port") // serveCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugins directory path") - serveCmd.Flags().StringVar(&config.Server.Jwks.Uri, "jwks-uri", config.Server.Jwks.Uri, "set the JWKS url to fetch public key") - serveCmd.Flags().IntVar(&config.Server.Jwks.Retries, "jwks-fetch-retries", config.Server.Jwks.Retries, "set the JWKS fetch retry count") + serveCmd.Flags().StringVar(&conf.Server.Jwks.Uri, "jwks-uri", conf.Server.Jwks.Uri, "set the JWKS url to fetch public key") + serveCmd.Flags().IntVar(&conf.Server.Jwks.Retries, "jwks-fetch-retries", conf.Server.Jwks.Retries, "set the JWKS fetch retry count") rootCmd.AddCommand(serveCmd) } From a7b8fb0de5dc9e143f609885b2be2c47b8606068 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 21 Nov 2024 14:14:44 -0700 Subject: [PATCH 047/102] refactor: more code cleanup and simplification --- pkg/generator/conman.go | 34 ++-- pkg/generator/dhcpd.go | 41 ++--- pkg/generator/dnsmasq.go | 33 ++-- pkg/generator/example.go | 12 +- pkg/generator/generator.go | 342 +++++++------------------------------ pkg/generator/hostfile.go | 4 +- pkg/generator/params.go | 43 +++++ pkg/generator/powerman.go | 4 +- pkg/generator/syslog.go | 4 +- pkg/generator/templates.go | 95 +++++++++++ pkg/generator/warewulf.go | 54 ++---- 11 files changed, 268 insertions(+), 398 deletions(-) create mode 100644 pkg/generator/params.go create mode 100644 pkg/generator/templates.go diff --git a/pkg/generator/conman.go b/pkg/generator/conman.go index 53358b5..ccadaa6 100644 --- a/pkg/generator/conman.go +++ b/pkg/generator/conman.go @@ -4,6 +4,8 @@ import ( "fmt" configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/client" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -21,32 +23,20 @@ func (g *Conman) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { +func (g *Conman) Generate(config *config.Config, params Params) (FileMap, error) { var ( - params = GetParams(opts...) - client = GetClient(params) - targetKey = params["target"].(string) // required param - target = config.Targets[targetKey] - eps []configurator.RedfishEndpoint = nil - err error = nil - // serverOpts = "" - // globalOpts = "" - consoles = "" + smdClient = client.NewSmdClient(params.ClientOpts...) + eps = []configurator.RedfishEndpoint{} + err error = nil + consoles = "" ) // fetch required data from SMD to create config - if client != nil { - eps, err = client.FetchRedfishEndpoints(opts...) - if err != nil { - return nil, fmt.Errorf("failed to fetch redfish endpoints with client: %v", err) - } + eps, err = smdClient.FetchRedfishEndpoints(params.Verbose) + if err != nil { + return nil, fmt.Errorf("failed to fetch redfish endpoints with client: %v", err) } - // add any additional conman or server opts - // if extraOpts, ok := params["opts"].(map[string]any); ok { - - // } - // format output to write to config file consoles = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" for _, ep := range eps { @@ -55,12 +45,12 @@ func (g *Conman) Generate(config *configurator.Config, opts ...util.Option) (Fil consoles += "# =====================================================================" // apply template substitutions and return output as byte array - return ApplyTemplateFromFiles(Mappings{ + return ApplyTemplates(Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), "server_opts": "", "global_opts": "", "consoles": consoles, - }, target.TemplatePaths...) + }, params.Templates) } diff --git a/pkg/generator/dhcpd.go b/pkg/generator/dhcpd.go index f2ce520..7425a57 100644 --- a/pkg/generator/dhcpd.go +++ b/pkg/generator/dhcpd.go @@ -4,6 +4,8 @@ import ( "fmt" configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/client" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -21,23 +23,18 @@ func (g *DHCPd) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *DHCPd) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { +func (g *DHCPd) Generate(config *config.Config, params Params) (FileMap, error) { var ( - params = GetParams(opts...) - client = GetClient(params) - targetKey = params["target"].(string) - target = config.Targets[targetKey] - compute_nodes = "" - eths []configurator.EthernetInterface = nil - err error = nil + smdClient = client.NewSmdClient(params.ClientOpts...) + eths = []configurator.EthernetInterface{} + computeNodes = "" + err error = nil ) // - if client != nil { - eths, err = client.FetchEthernetInterfaces(opts...) - if err != nil { - return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %w", err) - } + eths, err = smdClient.FetchEthernetInterfaces(params.Verbose) + if err != nil { + return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %w", err) } // check if we have the required params first @@ -49,25 +46,23 @@ func (g *DHCPd) Generate(config *configurator.Config, opts ...util.Option) (File } // format output to write to config file - compute_nodes = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" + computeNodes = "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" for _, eth := range eths { if len(eth.IpAddresses) == 0 { continue } - compute_nodes += fmt.Sprintf("host %s { hardware ethernet %s; fixed-address %s} ", eth.ComponentId, eth.MacAddress, eth.IpAddresses[0]) + computeNodes += fmt.Sprintf("host %s { hardware ethernet %s; fixed-address %s} ", eth.ComponentId, eth.MacAddress, eth.IpAddresses[0]) } - compute_nodes += "# =====================================================================" + computeNodes += "# =====================================================================" - if verbose, ok := params["verbose"].(bool); ok { - if verbose { - fmt.Printf("") - } + if params.Verbose { + fmt.Printf("") } - return ApplyTemplateFromFiles(Mappings{ + return ApplyTemplates(Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), - "compute_nodes": compute_nodes, + "compute_nodes": computeNodes, "node_entries": "", - }, target.TemplatePaths...) + }, params.Templates) } diff --git a/pkg/generator/dnsmasq.go b/pkg/generator/dnsmasq.go index ab5e648..83bf1d6 100644 --- a/pkg/generator/dnsmasq.go +++ b/pkg/generator/dnsmasq.go @@ -2,9 +2,10 @@ package generator import ( "fmt" - "strings" configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/client" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -22,7 +23,7 @@ func (g *DNSMasq) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *DNSMasq) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { +func (g *DNSMasq) Generate(config *config.Config, params Params) (FileMap, error) { // make sure we have a valid config first if config == nil { return nil, fmt.Errorf("invalid config (config is nil)") @@ -30,20 +31,15 @@ func (g *DNSMasq) Generate(config *configurator.Config, opts ...util.Option) (Fi // set all the defaults for variables var ( - params = GetParams(opts...) - client = GetClient(params) - targetKey = params["target"].(string) // required param - target = config.Targets[targetKey] - eths []configurator.EthernetInterface = nil - err error = nil + smdClient = client.NewSmdClient(params.ClientOpts...) + eths = []configurator.EthernetInterface{} + err error = nil ) // if we have a client, try making the request for the ethernet interfaces - if client != nil { - eths, err = client.FetchEthernetInterfaces(opts...) - if err != nil { - return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) - } + eths, err = smdClient.FetchEthernetInterfaces(params.Verbose) + if err != nil { + return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) } // check if we have the required params first @@ -54,13 +50,6 @@ func (g *DNSMasq) Generate(config *configurator.Config, opts ...util.Option) (Fi return nil, fmt.Errorf("no ethernet interfaces found") } - // print message if verbose param found - if verbose, ok := params["verbose"].(bool); ok { - if verbose { - fmt.Printf("template: \n%s\nethernet interfaces found: %v\n", strings.Join(target.TemplatePaths, "\n\t"), len(eths)) - } - } - // format output to write to config file output := "# ========== DYNAMICALLY GENERATED BY OPENCHAMI CONFIGURATOR ==========\n" for _, eth := range eths { @@ -73,10 +62,10 @@ func (g *DNSMasq) Generate(config *configurator.Config, opts ...util.Option) (Fi output += "# =====================================================================" // apply template substitutions and return output as byte array - return ApplyTemplateFromFiles(Mappings{ + return ApplyTemplates(Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), "dhcp-hosts": output, - }, target.TemplatePaths...) + }, params.Templates) } diff --git a/pkg/generator/example.go b/pkg/generator/example.go index b8b5c1d..f18abe4 100644 --- a/pkg/generator/example.go +++ b/pkg/generator/example.go @@ -1,13 +1,9 @@ -//go:build example || plugins -// +build example plugins - package generator import ( "fmt" - configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/OpenCHAMI/configurator/pkg/generator" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -27,11 +23,9 @@ func (g *Example) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Example) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *Example) Generate(config *config.Config, params Params) (FileMap, error) { g.Message = ` This is an example generator plugin. See the file in 'internal/generator/plugins/example/example.go' on information about constructing plugins and plugin requirements.` - return generator.FileMap{"example": []byte(g.Message)}, nil + return FileMap{"example": []byte(g.Message)}, nil } - -var Generator Example diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index e705e16..a56c026 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -1,7 +1,6 @@ package generator import ( - "bytes" "fmt" "io/fs" "os" @@ -9,9 +8,9 @@ import ( "plugin" configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/client" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" - "github.com/nikolalohinski/gonja/v2" - "github.com/nikolalohinski/gonja/v2/exec" "github.com/rs/zerolog/log" ) @@ -19,7 +18,6 @@ type ( Mappings map[string]any FileMap map[string][]byte FileList [][]byte - Template []byte // Generator interface used to define how files are created. Plugins can // be created entirely independent of the main driver program. @@ -27,17 +25,7 @@ type ( GetName() string GetVersion() string GetDescription() string - Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) - } - - // Params defined and used by the "generate" subcommand. - Params struct { - Args []string - TemplatePaths []string - PluginPath string - Target string - Client *configurator.SmdClient - Verbose bool + Generate(config *config.Config, params Params) (FileMap, error) } ) @@ -47,8 +35,7 @@ func createDefaultGenerators() map[string]Generator { var ( generatorMap = map[string]Generator{} generators = []Generator{ - &Conman{}, &DHCPd{}, &DNSMasq{}, &Hostfile{}, - &Powerman{}, &Syslog{}, &Warewulf{}, + &Conman{}, &DHCPd{}, &DNSMasq{}, &Warewulf{}, &Example{}, } ) for _, g := range generators { @@ -129,11 +116,11 @@ func LoadPlugin(path string) (Generator, error) { // // Returns a map of generators. Each generator can be accessed by the name // returned by the generator.GetName() implemented. -func LoadPlugins(dirpath string, opts ...util.Option) (map[string]Generator, error) { +func LoadPlugins(dirpath string, opts ...Option) (map[string]Generator, error) { // check if verbose option is supplied var ( generators = make(map[string]Generator) - params = util.ToDict(opts...) + params = ToParams(opts...) ) // @@ -150,7 +137,7 @@ func LoadPlugins(dirpath string, opts ...util.Option) (map[string]Generator, err } // show the plugins found if verbose flag is set - if params.GetVerbose() { + if params.Verbose { fmt.Printf("-- found plugin '%s'\n", gen.GetName()) } @@ -163,239 +150,33 @@ func LoadPlugins(dirpath string, opts ...util.Option) (map[string]Generator, err return nil, fmt.Errorf("failed to walk directory: %w", err) } - // items, _ := os.ReadDir(dirpath) - // for _, item := range items { - // if item.IsDir() { - // subitems, _ := os.ReadDir(item.Name()) - // for _, subitem := range subitems { - // if !subitem.IsDir() { - // gen, err := LoadPlugin(subitem.Name()) - // if err != nil { - // fmt.Printf("failed to load generator in directory '%s': %v\n", item.Name(), err) - // continue - // } - // if verbose, ok := params["verbose"].(bool); ok { - // if verbose { - // fmt.Printf("-- found plugin '%s'\n", item.Name()) - // } - // } - // gens[gen.GetName()] = gen - // } - // } - // } else { - // gen, err := LoadPlugin(dirpath + item.Name()) - // if err != nil { - // fmt.Printf("failed to load plugin: %v\n", err) - // continue - // } - // if verbose, ok := params["verbose"].(bool); ok { - // if verbose { - // fmt.Printf("-- found plugin '%s'\n", dirpath+item.Name()) - // } - // } - // gens[gen.GetName()] = gen - // } - // } - return generators, nil } -func LoadTemplate(path string) (Template, error) { - // skip loading template if path is a directory with no error - if isDir, err := util.IsDirectory(path); err == nil && isDir { - return nil, nil - } else if err != nil { - return nil, fmt.Errorf("failed to test if template path is directory: %w", err) - } - - // try and read the contents of the file - // NOTE: we don't care if this is actually a Jinja template - // or not...at least for now. - return os.ReadFile(path) -} - -func LoadTemplates(paths []string, opts ...util.Option) (map[string]Template, error) { - var ( - templates = make(map[string]Template) - params = util.ToDict(opts...) - ) - - for _, path := range paths { - err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { - // skip trying to load generator plugin if directory or error - if info.IsDir() || err != nil { - return nil - } - - // load the contents of the template - template, err := LoadTemplate(path) - if err != nil { - return fmt.Errorf("failed to load generator in directory '%s': %w", path, err) - } - - // show the templates loaded if verbose flag is set - if params.GetVerbose() { - fmt.Printf("-- loaded tempalte '%s'\n", path) - } - - // map each template by the path it was loaded from - templates[path] = template - return nil - }) - - if err != nil { - return nil, fmt.Errorf("failed to walk directory: %w", err) - } - } - - return templates, nil -} - -// Option to specify "target" in parameter map. This is used to set which generator -// to use to generate a config file. -func WithTarget(target string) util.Option { - return func(p util.Params) { - if p != nil { - p["target"] = target - } - } -} - -// Option to specify "type" in parameter map. This is not currently used. -func WithType(_type string) util.Option { - return func(p util.Params) { - if p != nil { - p["type"] = _type - } - } -} - -// Option to the plugin to load -func WithPlugin(path string) util.Option { - return func(p util.Params) { - if p != nil { - plugin, err := LoadPlugin(path) - if err != nil { - return - } - p["plugin"] = plugin - } - } -} - -func WithTemplates(paths []string) util.Option { - return func(p util.Params) { - if p != nil { - templates, err := LoadTemplates(paths) - if err != nil { - - } - p["templates"] = templates - } - } -} - -// Option to a specific client to include in implementing plugin generator.Generate(). -// -// NOTE: This may be changed to pass some kind of client interface as an argument in -// the future instead. -func WithClient(client configurator.SmdClient) util.Option { - return func(p util.Params) { - p["client"] = client - } -} - -// Helper function to get client in generator.Generate() plugin implementations. -func GetClient(params util.Params) *configurator.SmdClient { - return util.Get[configurator.SmdClient](params, "client") -} - -// Helper function to get the target in generator.Generate() plugin implementations. -func GetTarget(config *configurator.Config, key string) configurator.Target { - return config.Targets[key] -} - -// Helper function to load all options set with With*() into parameter map. -func GetParams(opts ...util.Option) util.Params { - params := util.Params{} - for _, opt := range opts { - opt(params) - } - return params -} - -// Wrapper function to slightly abstract away some of the nuances with using gonja -// into a single function call. This function is *mostly* for convenience and -// simplication. If no paths are supplied, then no templates will be applied and -// there will be no output. -// -// The "FileList" returns a slice of byte arrays in the same order as the argument -// list supplied, but with the Jinja templating applied. -func ApplyTemplates(mappings Mappings, contents ...[]byte) (FileList, error) { - var ( - data = exec.NewContext(mappings) - outputs = FileList{} - ) - - for _, b := range contents { - // load jinja template from file - t, err := gonja.FromBytes(b) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %w", err) - } - - // execute/render jinja template - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %w", err) - } - outputs = append(outputs, b.Bytes()) - } - - return outputs, nil -} - -// Wrapper function similiar to "ApplyTemplates" but takes file paths as arguments. -// This function will load templates from a file instead of using file contents. -func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) { - var ( - data = exec.NewContext(mappings) - outputs = FileMap{} - ) - - for _, path := range paths { - // load jinja template from file - t, err := gonja.FromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read template from file: %w", err) - } - - // execute/render jinja template - b := bytes.Buffer{} - if err = t.Execute(&b, data); err != nil { - return nil, fmt.Errorf("failed to execute: %w", err) - } - outputs[path] = b.Bytes() - } - - return outputs, nil -} - // Generate() is the main function to generate a collection of files and returns them as a map. // This function only expects a path to a plugin and paths to a collection of templates to // be used. This function will only load the plugin on-demand and fetch resources as needed. -func Generate(config *configurator.Config, params Params) (FileMap, error) { +// +// This function requires that a target and plugin path be set at minimum. +func Generate(plugin string, params Params) (FileMap, error) { var ( - gen Generator - client = configurator.NewSmdClient() + generator Generator + ok bool + err error ) - return gen.Generate( - config, - WithPlugin(params.PluginPath), - WithTemplates(params.TemplatePaths), - WithClient(client), - ) + // check if generator is built-in first before loading external plugin + generator, ok = DefaultGenerators[plugin] + if !ok { + // only load the plugin needed for this target if we don't find default + log.Error().Msg("could not find target in default generators") + generator, err = LoadPlugin(plugin) + if err != nil { + return nil, fmt.Errorf("failed to load plugin from file: %v", err) + } + } + + return generator.Generate(nil, params) } // Main function to generate a collection of files as a map with the path as the key and @@ -407,59 +188,66 @@ func Generate(config *configurator.Config, params Params) (FileMap, error) { // It is also call when running the configurator as a service with the "/generate" route. // // TODO: Separate loading plugins so we can load them once when running as a service. -func GenerateWithTarget(config *configurator.Config, params Params) (FileMap, error) { +func GenerateWithTarget(config *config.Config, target string) (FileMap, error) { // load generator plugins to generate configs or to print var ( - client configurator.SmdClient - target configurator.Target - generator Generator - err error - ok bool + opts []client.Option + targetInfo configurator.Target + generator Generator + params Params + err error + ok bool ) - // check if we have a client from params first and create new one if not - if params.Client == nil { - client = configurator.NewSmdClient( - configurator.WithHost(config.SmdClient.Host), - configurator.WithPort(config.SmdClient.Port), - configurator.WithAccessToken(config.AccessToken), - configurator.WithCertPoolFile(config.CertPath), - ) - } else { - client = *params.Client - } - // check if a target is supplied - if len(params.Args) == 0 && params.Target == "" { + if target == "" { return nil, fmt.Errorf("must specify a target") } // load target information from config - target, ok = config.Targets[params.Target] + targetInfo, ok = config.Targets[target] if !ok { - return nil, fmt.Errorf("target not found in config") + log.Warn().Msg("target not found in config") } - // if plugin path specified from CLI, use that instead - if params.PluginPath != "" { - target.PluginPath = params.PluginPath + // if no plugin supplied in config target, then using the target supplied + if targetInfo.Plugin == "" { + targetInfo.Plugin = target } // check if generator is built-in first before loading - generator, ok = DefaultGenerators[params.Target] + generator, ok = DefaultGenerators[target] if !ok { // only load the plugin needed for this target if we don't find default - log.Error().Msg("did not find target in default generators") - generator, err = LoadPlugin(target.PluginPath) + log.Error().Msg("could not find target in default generators") + generator, err = LoadPlugin(targetInfo.Plugin) if err != nil { - return nil, fmt.Errorf("failed to load plugin: %w", err) + return nil, fmt.Errorf("failed to load plugin: %v", err) } } + // prepare params to pass into generator + params.Templates = map[string]Template{} + for _, templatePath := range targetInfo.TemplatePaths { + template := Template{} + template.LoadFromFile(templatePath) + params.Templates[templatePath] = template + } + + // set the client options + if config.AccessToken != "" { + params.ClientOpts = append(opts, client.WithAccessToken(config.AccessToken)) + } + if config.CertPath != "" { + params.ClientOpts = append(opts, client.WithCertPoolFile(config.CertPath)) + } + + // load files that are not to be copied + params.Files, err = LoadFiles(targetInfo.FilePaths...) + if err != nil { + return nil, fmt.Errorf("failed to load files to copy: %v", err) + } + // run the generator plugin from target passed - return generator.Generate( - config, - WithTarget(generator.GetName()), - WithClient(client), - ) + return generator.Generate(config, params) } diff --git a/pkg/generator/hostfile.go b/pkg/generator/hostfile.go index e998714..7ce26c8 100644 --- a/pkg/generator/hostfile.go +++ b/pkg/generator/hostfile.go @@ -3,7 +3,7 @@ package generator import ( "fmt" - configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -21,6 +21,6 @@ func (g *Hostfile) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Hostfile) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { +func (g *Hostfile) Generate(config *config.Config, opts ...Option) (FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/pkg/generator/params.go b/pkg/generator/params.go new file mode 100644 index 0000000..e54420a --- /dev/null +++ b/pkg/generator/params.go @@ -0,0 +1,43 @@ +package generator + +import ( + configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/client" + "github.com/OpenCHAMI/configurator/pkg/config" +) + +type ( + // Params used by the generator + Params struct { + Templates map[string]Template + Files map[string][]byte + ClientOpts []client.Option + Verbose bool + } + Option func(Params) +) + +func ToParams(opts ...Option) Params { + params := Params{} + for _, opt := range opts { + opt(params) + } + return params +} + +func WithClientOpts(opts ...client.Option) Option { + return func(p Params) { + p.ClientOpts = opts + } +} + +func WithTemplates(templates map[string]Template) Option { + return func(p Params) { + p.Templates = templates + } +} + +// Helper function to get the target in generator.Generate() plugin implementations. +func GetTarget(config *config.Config, key string) configurator.Target { + return config.Targets[key] +} diff --git a/pkg/generator/powerman.go b/pkg/generator/powerman.go index 36be6fc..08745e5 100644 --- a/pkg/generator/powerman.go +++ b/pkg/generator/powerman.go @@ -3,7 +3,7 @@ package generator import ( "fmt" - configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -21,6 +21,6 @@ func (g *Powerman) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Powerman) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { +func (g *Powerman) Generate(config *config.Config, opts ...Option) (FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/pkg/generator/syslog.go b/pkg/generator/syslog.go index 463f727..67b28cf 100644 --- a/pkg/generator/syslog.go +++ b/pkg/generator/syslog.go @@ -3,7 +3,7 @@ package generator import ( "fmt" - configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -21,6 +21,6 @@ func (g *Syslog) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s'.", g.GetName()) } -func (g *Syslog) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { +func (g *Syslog) Generate(config *config.Config, opts ...Option) (FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } diff --git a/pkg/generator/templates.go b/pkg/generator/templates.go new file mode 100644 index 0000000..6d4ae5d --- /dev/null +++ b/pkg/generator/templates.go @@ -0,0 +1,95 @@ +package generator + +import ( + "bytes" + "fmt" + "os" + + "github.com/OpenCHAMI/configurator/pkg/util" + "github.com/nikolalohinski/gonja/v2" + "github.com/nikolalohinski/gonja/v2/exec" +) + +type Template struct { + Contents []byte `json:"contents"` +} + +func (t *Template) LoadFromFile(path string) error { + // skip loading template if path is a directory with no error + if isDir, err := util.IsDirectory(path); err == nil && isDir { + return nil + } else if err != nil { + return fmt.Errorf("failed to test if template path is directory: %w", err) + } + + // try and read the contents of the file + // NOTE: we don't care if this is actually a Jinja template + // or not...at least for now. + contents, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read file: %v", err) + } + t.Contents = contents + return nil +} + +func (t *Template) IsEmpty() bool { + return len(t.Contents) <= 0 +} + +// Wrapper function to slightly abstract away some of the nuances with using gonja +// into a single function call. This function is *mostly* for convenience and +// simplication. If no paths are supplied, then no templates will be applied and +// there will be no output. +// +// The "FileList" returns a slice of byte arrays in the same order as the argument +// list supplied, but with the Jinja templating applied. +func ApplyTemplates(mappings Mappings, templates map[string]Template) (FileMap, error) { + var ( + data = exec.NewContext(mappings) + outputs = FileMap{} + ) + + for path, template := range templates { + // load jinja template from file + t, err := gonja.FromBytes(template.Contents) + if err != nil { + return nil, fmt.Errorf("failed to read template from file: %w", err) + } + + // execute/render jinja template + b := bytes.Buffer{} + if err = t.Execute(&b, data); err != nil { + return nil, fmt.Errorf("failed to execute: %w", err) + } + outputs[path] = b.Bytes() + } + + return outputs, nil +} + +// Wrapper function similiar to "ApplyTemplates" but takes file paths as arguments. +// This function will load templates from a file instead of using file contents. +func ApplyTemplateFromFiles(mappings Mappings, paths ...string) (FileMap, error) { + var ( + data = exec.NewContext(mappings) + outputs = FileMap{} + ) + + for _, path := range paths { + // load jinja template from file + t, err := gonja.FromFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read template from file: %w", err) + } + + // execute/render jinja template + b := bytes.Buffer{} + if err = t.Execute(&b, data); err != nil { + return nil, fmt.Errorf("failed to execute: %w", err) + } + outputs[path] = b.Bytes() + } + + return outputs, nil +} diff --git a/pkg/generator/warewulf.go b/pkg/generator/warewulf.go index 49b0c2c..cbc4129 100644 --- a/pkg/generator/warewulf.go +++ b/pkg/generator/warewulf.go @@ -3,9 +3,9 @@ package generator import ( "fmt" "maps" - "strings" - configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/client" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -23,22 +23,15 @@ func (g *Warewulf) GetDescription() string { return "Configurator generator plugin for 'warewulf' config files." } -func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { +func (g *Warewulf) Generate(config *config.Config, params Params) (FileMap, error) { var ( - params = GetParams(opts...) - client = GetClient(params) - targetKey = params["target"].(string) - target = config.Targets[targetKey] - outputs = make(FileMap, len(target.FilePaths)+len(target.TemplatePaths)) + smdClient = client.NewSmdClient(params.ClientOpts...) + outputs = make(FileMap, len(params.Templates)) + nodeEntries = "" ) - // check if our client is included and is valid - if client == nil { - return nil, fmt.Errorf("invalid client (client is nil)") - } - // if we have a client, try making the request for the ethernet interfaces - eths, err := client.FetchEthernetInterfaces(opts...) + eths, err := smdClient.FetchEthernetInterfaces(params.Verbose) if err != nil { return nil, fmt.Errorf("failed to fetch ethernet interfaces with client: %v", err) } @@ -51,15 +44,8 @@ func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (F return nil, fmt.Errorf("no ethernet interfaces found") } - // print message if verbose param found - if verbose, ok := params["verbose"].(bool); ok { - if verbose { - fmt.Printf("template: \n%s\n ethernet interfaces found: %v\n", strings.Join(target.TemplatePaths, "\n\t"), len(eths)) - } - } - // fetch redfish endpoints and handle errors - eps, err := client.FetchRedfishEndpoints(opts...) + eps, err := smdClient.FetchRedfishEndpoints(params.Verbose) if err != nil { return nil, fmt.Errorf("failed to fetch redfish endpoints: %v", err) } @@ -67,31 +53,21 @@ func (g *Warewulf) Generate(config *configurator.Config, opts ...util.Option) (F return nil, fmt.Errorf("no redfish endpoints found") } - // format output for template substitution - nodeEntries := "" - - // load files and templates and copy to outputs - files, err := LoadFiles(target.FilePaths...) - if err != nil { - return nil, fmt.Errorf("failed to load files: %v", err) - } - templates, err := ApplyTemplateFromFiles(Mappings{ + templates, err := ApplyTemplates(Mappings{ "node_entries": nodeEntries, - }, target.TemplatePaths...) + }, params.Templates) if err != nil { return nil, fmt.Errorf("failed to load templates: %v", err) } - maps.Copy(outputs, files) + maps.Copy(outputs, params.Files) maps.Copy(outputs, templates) // print message if verbose param is found - if verbose, ok := params["verbose"].(bool); ok { - if verbose { - fmt.Printf("templates and files loaded: \n") - for path, _ := range outputs { - fmt.Printf("\t%s", path) - } + if params.Verbose { + fmt.Printf("templates and files loaded: \n") + for path, _ := range outputs { + fmt.Printf("\t%s", path) } } From 72dd8416aaa1606fb7f15ce15a5bc27838207c08 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 21 Nov 2024 14:15:43 -0700 Subject: [PATCH 048/102] feat: add initial implementation of server target API --- pkg/server/server.go | 177 ++++++++++++++++++++++++++++--------------- 1 file changed, 116 insertions(+), 61 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 9afc7db..cd448cc 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -4,16 +4,16 @@ package server import ( - "crypto/tls" - "crypto/x509" "encoding/json" "fmt" - "net" + "io" "net/http" "os" "time" configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/client" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/jwtauth/v5" "github.com/go-chi/chi/v5" @@ -36,62 +36,43 @@ type Jwks struct { } type Server struct { *http.Server - Config *configurator.Config + Config *config.Config Jwks Jwks `yaml:"jwks"` GeneratorParams generator.Params TokenAuth *jwtauth.JWTAuth + Targets map[string]Target +} + +type Target struct { + Name string `json:"name"` + PluginPath string `json:"plugin"` + Templates []generator.Template `json:"templates"` } // Constructor to make a new server instance with an optional config. -func New(config *configurator.Config) *Server { +func New(conf *config.Config) *Server { // create default config if none supplied - if config == nil { - c := configurator.NewConfig() - config = &c + if conf == nil { + c := config.New() + conf = &c } // return based on config values return &Server{ - Config: config, + Config: conf, Server: &http.Server{ - Addr: fmt.Sprintf("%s:%d", config.Server.Host, config.Server.Port), + Addr: fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port), }, Jwks: Jwks{ - Uri: config.Server.Jwks.Uri, - Retries: config.Server.Jwks.Retries, + Uri: conf.Server.Jwks.Uri, + Retries: conf.Server.Jwks.Retries, }, } } // Main function to start up configurator as a service. -func (s *Server) Serve(cacertPath string) error { - // create client just for the server to use to fetch data from SMD - client := &configurator.SmdClient{ - Host: s.Config.SmdClient.Host, - Port: s.Config.SmdClient.Port, - } - - // add cert to client if `--cacert` flag is passed - if cacertPath != "" { - cacert, _ := os.ReadFile(cacertPath) - certPool := x509.NewCertPool() - certPool.AppendCertsFromPEM(cacert) - client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: certPool, - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - Dial: (&net.Dialer{ - Timeout: 120 * time.Second, - KeepAlive: 120 * time.Second, - }).Dial, - TLSHandshakeTimeout: 120 * time.Second, - ResponseHeaderTimeout: 120 * time.Second, - } - } - +func (s *Server) Serve() error { // set the server address with config values - s.Server.Addr = fmt.Sprintf("%s:%d", s.Config.Server.Host, s.Config.Server.Port) + s.Server.Addr = s.Config.Server.Host // fetch JWKS public key from authorization server if s.Config.Server.Jwks.Uri != "" && tokenAuth == nil { @@ -110,6 +91,12 @@ func (s *Server) Serve(cacertPath string) error { zerolog.TimeFieldFormat = zerolog.TimeFormatUnix logger := log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + // create client with opts to use to fetch data from SMD + opts := []client.Option{ + client.WithAccessToken(s.Config.AccessToken), + client.WithCertPoolFile(s.Config.CertPath), + } + // create new go-chi router with its routes router := chi.NewRouter() router.Use(middleware.RequestID) @@ -127,13 +114,13 @@ func (s *Server) Serve(cacertPath string) error { ) // protected routes if using auth - r.HandleFunc("/generate", s.Generate(client)) - r.HandleFunc("/templates", s.ManageTemplates) + r.HandleFunc("/generate", s.Generate(opts...)) + r.Post("/targets", s.createTarget) }) } else { // public routes without auth - router.HandleFunc("/generate", s.Generate(client)) - router.HandleFunc("/templates", s.ManageTemplates) + router.HandleFunc("/generate", s.Generate(opts...)) + router.Post("/targets", s.createTarget) } // always available public routes go here (none at the moment) @@ -150,22 +137,37 @@ func (s *Server) Close() { // This is the corresponding service function to generate templated files, that // works similarly to the CLI variant. This function takes similiar arguments as // query parameters that are included in the HTTP request URL. -func (s *Server) Generate(client *configurator.SmdClient) func(w http.ResponseWriter, r *http.Request) { - +func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { // get all of the expect query URL params and validate - s.GeneratorParams.Target = r.URL.Query().Get("target") - s.GeneratorParams.Client = client - if s.GeneratorParams.Target == "" { + var ( + target string = r.URL.Query().Get("target") + ) + s.GeneratorParams = parseGeneratorParams(r, opts...) + if target == "" { writeErrorResponse(w, "must specify a target") return } - // generate a new config file from supplied params - outputs, err := generator.GenerateWithTarget(s.Config, s.GeneratorParams) - if err != nil { - writeErrorResponse(w, "failed to generate file: %v", err) - return + // try to generate with target supplied by client first + var ( + t *Target = s.getTarget(target) + outputs generator.FileMap + err error + ) + + if t != nil { + outputs, err = generator.Generate(t.PluginPath, s.GeneratorParams) + if err != nil { + + } + } else { + // try and generate a new config file from supplied params + outputs, err = generator.GenerateWithTarget(s.Config, target) + if err != nil { + writeErrorResponse(w, "failed to generate file: %v", err) + return + } } // marshal output to JSON then send response to client @@ -183,22 +185,75 @@ func (s *Server) Generate(client *configurator.SmdClient) func(w http.ResponseWr } } -// Incomplete WIP function for managing templates remotely. There is currently no -// internal API to do this yet. +// Create a new target with name, generator, templates, and files. +// +// Example: +// +// curl -X POST /target?name=test&plugin=dnsmasq // // TODO: need to implement template managing API first in "internal/generator/templates" or something -func (s *Server) ManageTemplates(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte("this is not implemented yet")) - if err != nil { - writeErrorResponse(w, "failed to write response: %v", err) +func (s *Server) createTarget(w http.ResponseWriter, r *http.Request) { + var ( + target = Target{} + bytes []byte + err error + ) + if r == nil { + writeErrorResponse(w, "request is invalid") return } + + bytes, err = io.ReadAll(r.Body) + if err != nil { + writeErrorResponse(w, "failed to read response body: %v", err) + return + } + defer r.Body.Close() + + err = json.Unmarshal(bytes, &target) + if err != nil { + writeErrorResponse(w, "failed to unmarshal target: %v", err) + return + } + + // make sure a plugin and at least one template is supplied + if target.Name == "" { + writeErrorResponse(w, "target name is required") + return + } + if target.PluginPath == "" { + writeErrorResponse(w, "must supply a generator name") + return + } + if len(target.Templates) <= 0 { + writeErrorResponse(w, "must provided at least one template") + return + } + + s.Targets[target.Name] = target + +} + +func (s *Server) getTarget(target string) *Target { + t, ok := s.Targets[target] + if ok { + return &t + } + return nil } // Wrapper function to simplify writting error message responses. This function // is only intended to be used with the service and nothing else. func writeErrorResponse(w http.ResponseWriter, format string, a ...any) error { errmsg := fmt.Sprintf(format, a...) - w.Write([]byte(errmsg)) + log.Error().Msg(errmsg) + http.Error(w, errmsg, http.StatusInternalServerError) return fmt.Errorf(errmsg) } + +func parseGeneratorParams(r *http.Request, opts ...client.Option) generator.Params { + var params = generator.Params{ + ClientOpts: opts, + } + return params +} From 69aac3c92992f990a66f5381db9c2e7bc5a255a3 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 21 Nov 2024 14:16:03 -0700 Subject: [PATCH 049/102] refactor: more code cleanup and reorganization --- pkg/client/client.go | 66 ++++++++++++++ pkg/{client.go => client/smd.go} | 148 +++++++++---------------------- pkg/{ => config}/config.go | 57 +++++------- pkg/configurator.go | 7 ++ 4 files changed, 138 insertions(+), 140 deletions(-) create mode 100644 pkg/client/client.go rename pkg/{client.go => client/smd.go} (54%) rename pkg/{ => config}/config.go (60%) diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 0000000..a4b7454 --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,66 @@ +package client + +import ( + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "os" + "time" +) + +type Option func(*Params) +type Params struct { + Host string `yaml:"host"` + AccessToken string `yaml:"access-token"` + Transport *http.Transport +} + +func ToParams(opts ...Option) *Params { + params := &Params{} + for _, opt := range opts { + opt(params) + } + return params +} + +func WithHost(host string) Option { + return func(c *Params) { + c.Host = host + } +} + +func WithAccessToken(token string) Option { + return func(c *Params) { + c.AccessToken = token + } +} + +func WithCertPool(certPool *x509.CertPool) Option { + return func(c *Params) { + c.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + } +} + +// FIXME: Need to check for errors when reading from a file +func WithCertPoolFile(certPath string) Option { + if certPath == "" { + return func(sc *Params) {} + } + cacert, _ := os.ReadFile(certPath) + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + return WithCertPool(certPool) +} diff --git a/pkg/client.go b/pkg/client/smd.go similarity index 54% rename from pkg/client.go rename to pkg/client/smd.go index ddbfab0..9a01401 100644 --- a/pkg/client.go +++ b/pkg/client/smd.go @@ -1,22 +1,15 @@ -package configurator +package client import ( "bytes" - "crypto/tls" - "crypto/x509" "encoding/json" "fmt" "io" - "net" "net/http" - "os" - "time" - "github.com/OpenCHAMI/configurator/pkg/util" + configurator "github.com/OpenCHAMI/configurator/pkg" ) -type ClientOption func(*SmdClient) - // An struct that's meant to extend functionality of the base HTTP client by // adding commonly made requests to SMD. The implemented functions are can be // used in generator plugins to fetch data when it is needed to substitute @@ -28,101 +21,43 @@ type SmdClient struct { AccessToken string `yaml:"access-token"` } -// Constructor function that allows supplying ClientOption arguments to set +// Constructor function that allows supplying Option arguments to set // things like the host, port, access token, etc. -func NewSmdClient(opts ...ClientOption) SmdClient { - client := SmdClient{} - for _, opt := range opts { - opt(&client) - } - return client -} - -func WithHost(host string) ClientOption { - return func(c *SmdClient) { - c.Host = host - } -} - -func WithPort(port int) ClientOption { - return func(c *SmdClient) { - c.Port = port - } -} - -func WithAccessToken(token string) ClientOption { - return func(c *SmdClient) { - c.AccessToken = token - } -} - -func WithCertPool(certPool *x509.CertPool) ClientOption { - return func(c *SmdClient) { - c.Client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: certPool, - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - Dial: (&net.Dialer{ - Timeout: 120 * time.Second, - KeepAlive: 120 * time.Second, - }).Dial, - TLSHandshakeTimeout: 120 * time.Second, - ResponseHeaderTimeout: 120 * time.Second, +func NewSmdClient(opts ...Option) SmdClient { + var ( + params = ToParams(opts...) + client = SmdClient{ + Host: params.Host, + AccessToken: params.AccessToken, } - } -} + ) -// FIXME: Need to check for errors when reading from a file -func WithCertPoolFile(certPath string) ClientOption { - if certPath == "" { - return func(sc *SmdClient) {} - } - cacert, _ := os.ReadFile(certPath) - certPool := x509.NewCertPool() - certPool.AppendCertsFromPEM(cacert) - return WithCertPool(certPool) -} - -func WithVerbosity() util.Option { - return func(p util.Params) { - p["verbose"] = true - } -} - -// Create a set of params with all default values. -func NewParams() util.Params { - return util.Params{ - "verbose": false, - } + return client } // Fetch the ethernet interfaces from SMD service using its API. An access token may be required if the SMD // service SMD_JWKS_URL envirnoment variable is set. -func (client *SmdClient) FetchEthernetInterfaces(opts ...util.Option) ([]EthernetInterface, error) { +func (client *SmdClient) FetchEthernetInterfaces(verbose bool) ([]configurator.EthernetInterface, error) { var ( - params = util.ToDict(opts...) - verbose = util.Get[bool](params, "verbose") - eths = []EthernetInterface{} + eths = []configurator.EthernetInterface{} + bytes []byte + err error ) // make request to SMD endpoint - b, err := client.makeRequest("/Inventory/EthernetInterfaces") + bytes, err = client.makeRequest("/Inventory/EthernetInterfaces") if err != nil { return nil, fmt.Errorf("failed to read HTTP response: %v", err) } // unmarshal response body JSON and extract in object - err = json.Unmarshal(b, ðs) + err = json.Unmarshal(bytes, ðs) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) } // print what we got if verbose is set - if verbose != nil { - if *verbose { - fmt.Printf("Ethernet Interfaces: %v\n", string(b)) - } + if verbose { + fmt.Printf("Ethernet Interfaces: %v\n", string(bytes)) } return eths, nil @@ -130,68 +65,68 @@ func (client *SmdClient) FetchEthernetInterfaces(opts ...util.Option) ([]Etherne // Fetch the components from SMD using its API. An access token may be required if the SMD // service SMD_JWKS_URL envirnoment variable is set. -func (client *SmdClient) FetchComponents(opts ...util.Option) ([]Component, error) { +func (client *SmdClient) FetchComponents(verbose bool) ([]configurator.Component, error) { var ( - params = util.ToDict(opts...) - verbose = util.Get[bool](params, "verbose") - comps = []Component{} + comps = []configurator.Component{} + bytes []byte + err error ) // make request to SMD endpoint - b, err := client.makeRequest("/State/Components") + bytes, err = client.makeRequest("/State/Components") if err != nil { return nil, fmt.Errorf("failed to make HTTP request: %v", err) } // make sure our response is actually JSON - if !json.Valid(b) { - return nil, fmt.Errorf("expected valid JSON response: %v", string(b)) + if !json.Valid(bytes) { + return nil, fmt.Errorf("expected valid JSON response: %v", string(bytes)) } // unmarshal response body JSON and extract in object var tmp map[string]any - err = json.Unmarshal(b, &tmp) + err = json.Unmarshal(bytes, &tmp) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) } - b, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) + bytes, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) if err != nil { return nil, fmt.Errorf("failed to marshal JSON: %v", err) } - err = json.Unmarshal(b, &comps) + err = json.Unmarshal(bytes, &comps) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) } // print what we got if verbose is set - if verbose != nil { - if *verbose { - fmt.Printf("Components: %v\n", string(b)) - } + if verbose { + fmt.Printf("Components: %v\n", string(bytes)) } return comps, nil } -func (client *SmdClient) FetchRedfishEndpoints(opts ...util.Option) ([]RedfishEndpoint, error) { +// TODO: improve implementation of this function +func (client *SmdClient) FetchRedfishEndpoints(verbose bool) ([]configurator.RedfishEndpoint, error) { var ( - params = util.ToDict(opts...) - verbose = util.Get[bool](params, "verbose") - eps = []RedfishEndpoint{} + eps = []configurator.RedfishEndpoint{} + tmp map[string]any ) + // make initial request to get JSON with 'RedfishEndpoints' as property b, err := client.makeRequest("/Inventory/RedfishEndpoints") if err != nil { return nil, fmt.Errorf("failed to make HTTP resquest: %v", err) } + // make sure response is in JSON if !json.Valid(b) { return nil, fmt.Errorf("expected valid JSON response: %v", string(b)) } - var tmp map[string]any err = json.Unmarshal(b, &tmp) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %v", err) } + // marshal RedfishEndpoint JSON back to configurator.RedfishEndpoint b, err = json.Marshal(tmp["RedfishEndpoints"].([]any)) if err != nil { return nil, fmt.Errorf("failed to marshal JSON: %v", err) @@ -201,10 +136,9 @@ func (client *SmdClient) FetchRedfishEndpoints(opts ...util.Option) ([]RedfishEn return nil, fmt.Errorf("failed to unmarshal response: %v", err) } - if verbose != nil { - if *verbose { - fmt.Printf("Redfish endpoints: %v\n", string(b)) - } + // show the final result + if verbose { + fmt.Printf("Redfish endpoints: %v\n", string(b)) } return eps, nil diff --git a/pkg/config.go b/pkg/config/config.go similarity index 60% rename from pkg/config.go rename to pkg/config/config.go index 89db7ab..2a6ef79 100644 --- a/pkg/config.go +++ b/pkg/config/config.go @@ -1,22 +1,15 @@ -package configurator +package config import ( "log" "os" "path/filepath" + configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/client" "gopkg.in/yaml.v2" ) -type Options struct{} - -type Target struct { - PluginPath string `yaml:"plugin,omitempty"` - TemplatePaths []string `yaml:"templates,omitempty"` - FilePaths []string `yaml:"files,omitempty"` - RunTargets []string `yaml:"targets,omitempty"` -} - type Jwks struct { Uri string `yaml:"uri"` Retries int `yaml:"retries,omitempty"` @@ -29,35 +22,34 @@ type Server struct { } type Config struct { - Version string `yaml:"version,omitempty"` - Server Server `yaml:"server,omitempty"` - SmdClient SmdClient `yaml:"smd,omitempty"` - AccessToken string `yaml:"access-token,omitempty"` - Targets map[string]Target `yaml:"targets,omitempty"` - PluginDirs []string `yaml:"plugins,omitempty"` - CertPath string `yaml:"cacert,omitempty"` - Options Options `yaml:"options,omitempty"` + Version string `yaml:"version,omitempty"` + Server Server `yaml:"server,omitempty"` + SmdClient client.SmdClient `yaml:"smd,omitempty"` + AccessToken string `yaml:"access-token,omitempty"` + Targets map[string]configurator.Target `yaml:"targets,omitempty"` + PluginDirs []string `yaml:"plugins,omitempty"` + CertPath string `yaml:"cacert,omitempty"` } // Creates a new config with default parameters. -func NewConfig() Config { +func New() Config { return Config{ Version: "", - SmdClient: SmdClient{ + SmdClient: client.SmdClient{ Host: "http://127.0.0.1", Port: 27779, }, - Targets: map[string]Target{ - "dnsmasq": Target{ - PluginPath: "", + Targets: map[string]configurator.Target{ + "dnsmasq": configurator.Target{ + Plugin: "", TemplatePaths: []string{}, }, - "conman": Target{ - PluginPath: "", + "conman": configurator.Target{ + Plugin: "", TemplatePaths: []string{}, }, - "warewulf": Target{ - PluginPath: "", + "warewulf": configurator.Target{ + Plugin: "", TemplatePaths: []string{ "templates/warewulf/defaults/node.jinja", "templates/warewulf/defaults/provision.jinja", @@ -74,12 +66,11 @@ func NewConfig() Config { Retries: 5, }, }, - Options: Options{}, } } -func LoadConfig(path string) Config { - var c Config = NewConfig() +func Load(path string) Config { + var c Config = New() file, err := os.ReadFile(path) if err != nil { log.Printf("failed to read config file: %v\n", err) @@ -93,7 +84,7 @@ func LoadConfig(path string) Config { return c } -func (config *Config) SaveConfig(path string) { +func (config *Config) Save(path string) { path = filepath.Clean(path) if path == "" || path == "." { path = "config.yaml" @@ -110,12 +101,12 @@ func (config *Config) SaveConfig(path string) { } } -func SaveDefaultConfig(path string) { +func SaveDefault(path string) { path = filepath.Clean(path) if path == "" || path == "." { path = "config.yaml" } - var c = NewConfig() + var c = New() data, err := yaml.Marshal(c) if err != nil { log.Printf("failed to marshal config: %v\n", err) diff --git a/pkg/configurator.go b/pkg/configurator.go index 68dae99..7a19a36 100644 --- a/pkg/configurator.go +++ b/pkg/configurator.go @@ -2,6 +2,13 @@ package configurator import "encoding/json" +type Target struct { + Plugin string `yaml:"plugin,omitempty"` // Set the plugin or it's path + TemplatePaths []string `yaml:"templates,omitempty"` // Set the template paths + FilePaths []string `yaml:"files,omitempty"` // Set the file paths + RunTargets []string `yaml:"targets,omitempty"` // Set additional targets to run +} + type IPAddr struct { IpAddress string `json:"IPAddress"` Network string `json:"Network"` From 5ca4d17d4288e0878e6b53260f6859e85235112e Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 3 Dec 2024 12:51:13 -0700 Subject: [PATCH 050/102] fix: added missing output maps --- cmd/generate.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/generate.go b/cmd/generate.go index 634a775..c1d02af 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -85,6 +85,7 @@ var generateCmd = &cobra.Command{ } // if we have more than one target and output is set, create configs in directory + outputMap := generator.ConvertContentsToString(outputBytes) writeOutput(outputBytes, len(targets), len(outputMap)) } }, @@ -106,6 +107,7 @@ func RunTargets(conf *config.Config, args []string, targets ...string) { } // if we have more than one target and output is set, create configs in directory + outputMap := generator.ConvertContentsToString(outputBytes) writeOutput(outputBytes, len(targets), len(outputMap)) // remove any targets that are the same as current to prevent infinite loop From e1a9f4ae3643af143817d2efd67512d781c8db9c Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 3 Dec 2024 12:51:48 -0700 Subject: [PATCH 051/102] fix: changed output to use log instead of fmt --- cmd/root.go | 2 +- cmd/serve.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index be9659c..1d1997c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -51,7 +51,7 @@ func initConfig() { if configPath != "" { exists, err := util.PathExists(configPath) if err != nil { - fmt.Printf("failed to load config") + log.Error().Err(err).Str("path", configPath).Msg("failed to load config") os.Exit(1) } else if exists { conf = config.Load(configPath) diff --git a/cmd/serve.go b/cmd/serve.go index 61bfeff..d421538 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -27,7 +27,7 @@ var serveCmd = &cobra.Command{ conf.AccessToken = accessToken } else { if verbose { - fmt.Printf("No token found. Continuing without one...\n") + log.Warn().Msg("No token found. Continuing without one...\n") } } } From 40c85646811cbc0318a6d228b55af8ddbfc700da Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 3 Dec 2024 12:52:06 -0700 Subject: [PATCH 052/102] fix: changed more output to use log instead of fmt --- pkg/server/server.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index cd448cc..689a0fd 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -19,7 +19,6 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/rs/zerolog" - "github.com/sirupsen/logrus" openchami_authenticator "github.com/openchami/chi-middleware/auth" openchami_logger "github.com/openchami/chi-middleware/log" @@ -71,6 +70,10 @@ func New(conf *config.Config) *Server { // Main function to start up configurator as a service. func (s *Server) Serve() error { + // Setup logger + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + logger := log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + // set the server address with config values s.Server.Addr = s.Config.Server.Host @@ -80,17 +83,13 @@ func (s *Server) Serve() error { var err error tokenAuth, err = configurator.FetchPublicKeyFromURL(s.Config.Server.Jwks.Uri) if err != nil { - logrus.Errorf("failed to fetch JWKS: %v", err) + log.Error().Err(err).Msgf("failed to fetch JWKS") continue } break } } - // Setup logger - zerolog.TimeFieldFormat = zerolog.TimeFormatUnix - logger := log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) - // create client with opts to use to fetch data from SMD opts := []client.Option{ client.WithAccessToken(s.Config.AccessToken), From b858ff3fe59900cc597652f85deb2bb4c9aabc24 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 3 Dec 2024 13:30:03 -0700 Subject: [PATCH 053/102] readme: updated example to use coredhcp and other changes --- README.md | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c7969f7..840558e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The `configurator` is an extensible tool that is capable of dynamically generati ## Building and Usage -The `configurator` is built using standard `go` build tools. The project separates the client and server components using build tags. To get started, clone the project, download the dependencies, and build the project: +The `configurator` is built using standard `go` build tools. The project separates the client, server, and generator components using build tags. To get started, clone the project, download the dependencies, and build the project: ```bash git clone https://github.com/OpenCHAMI/configurator.git @@ -23,20 +23,20 @@ After you build the program, run the following command to use the tool: ```bash export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -./configurator generate --config config.yaml --target dnsmasq -o dnsmasq.conf --cacert configurator.pem +./configurator generate --config config.yaml --target coredhcp -o coredhcp.conf --cacert ochami.pem ``` -This will generate a new `dnsmasq` config file based on the Jinja 2 template specified in the config file for "dnsmasq". The files will be written to `dnsmasq.conf` as specified with the `-o/--output` flag. The `--target` flag specifies the type of config file to generate by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). +This will generate a new `coredhcp` config file based on the Jinja 2 template specified in the config file for "coredhcp". The files will be written to `coredhcp.conf` as specified with the `-o/--output` flag. The `--target` flag specifies the type of config file to generate by its name (see the [`Creating Generator Plugins`](#creating-generator-plugins) section for details). In other words, there should be an entry in the config file that looks like this: ```yaml ... targets: - dnsmasq: - plugin: "lib/dnsmasq.so" # optional, if we want to use a plugin instead + coredhcp: + plugin: "lib/coredhcp.so" # optional, if we want to use an external plugin instead templates: - - templates/dnsmasq.j2 + - templates/coredhcp.j2 ... ``` @@ -57,9 +57,9 @@ Once the server is up and listening for HTTP requests, you can try making a requ ```bash export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -curl http://127.0.0.1:3334/generate?target=dnsmasq -X GET -H "Authorization: Bearer $ACCESS_TOKEN" --cacert configurator.pem +curl http://127.0.0.1:3334/generate?target=dnsmasq -X GET -H "Authorization: Bearer $ACCESS_TOKEN" --cacert ochami.pem # ...or... -./configurator fetch --target dnsmasq --host http://127.0.0.1:3334 --cacert configurator.pem +./configurator fetch --target dnsmasq --host http://127.0.0.1:3334 --cacert ochami.pem ``` This will do the same thing as the `generate` subcommand, but through a GET request where the file contents is returned in the response. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set when starting the server with `serve`. The `ACCESS_TOKEN` environment variable is passed to `curl` using the `Authorization` header and expects a token as a JWT. @@ -84,7 +84,7 @@ Then, run the Docker container similarly to running the binary. ```bash export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -docker run ghcr.io/openchami/configurator:latest configurator generate --config config.yaml --target dnsmasq -o dnsmasq.conf --cacert configurator.pem +docker run ghcr.io/openchami/configurator:latest configurator generate --config config.yaml --target coredhcp -o coredhcp.conf --cacert configurator.pem ``` ### Creating Generator Plugins @@ -177,25 +177,15 @@ server: # Server-related parameters when using as service uri: "" retries: 5 smd: # SMD-related parameters - host: http://127.0.0.1 - port: 27779 + host: http://127.0.0.1:27779 plugins: # path to plugin directories - "lib/" targets: # targets to call with --target flag - dnsmasq: + coredhcp: templates: - - templates/dnsmasq.jinja - warewulf: - templates: # files using Jinja templating - - templates/warewulf/vnfs/dhcpd-template.jinja - - templates/warewulf/vnfs/dnsmasq-template.jinja + - templates/coredhcp.j2 files: # files to be copied without templating - - templates/warewulf/defaults/provision.jinja - - templates/warewulf/defaults/node.jinja - - templates/warewulf/filesystem/examples/* - - templates/warewulf/vnfs/* - - templates/warewulf/bootstrap.jinja - - templates/warewulf/database.jinja + - extra/nodes.conf targets: # additional targets to run (does not run recursively) - dnsmasq ``` @@ -210,7 +200,6 @@ The `configurator` project includes a collection of tests focused on verifying p go test ./tests/generate_test.go --tags=all ``` - ## Known Issues - Adds a new `OAuthClient` with every token request From c5ee0552b4497a79164745a84b3805fdc8163037 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 3 Dec 2024 13:37:58 -0700 Subject: [PATCH 054/102] readme: updated example to use coredhcp --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 840558e..d040037 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,9 @@ Once the server is up and listening for HTTP requests, you can try making a requ ```bash export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... -curl http://127.0.0.1:3334/generate?target=dnsmasq -X GET -H "Authorization: Bearer $ACCESS_TOKEN" --cacert ochami.pem +curl http://127.0.0.1:3334/generate?target=coredhcp -X GET -H "Authorization: Bearer $ACCESS_TOKEN" --cacert ochami.pem # ...or... -./configurator fetch --target dnsmasq --host http://127.0.0.1:3334 --cacert ochami.pem +./configurator fetch --target coredhcp --host http://127.0.0.1:3334 --cacert ochami.pem ``` This will do the same thing as the `generate` subcommand, but through a GET request where the file contents is returned in the response. The access token is only required if the `CONFIGURATOR_JWKS_URL` environment variable is set when starting the server with `serve`. The `ACCESS_TOKEN` environment variable is passed to `curl` using the `Authorization` header and expects a token as a JWT. From 8f23422db043a6afb0b2cfd701e3d31f68305933 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 3 Dec 2024 13:45:54 -0700 Subject: [PATCH 055/102] refactor: minor changes --- pkg/generator/warewulf.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/generator/warewulf.go b/pkg/generator/warewulf.go index cbc4129..6e35e55 100644 --- a/pkg/generator/warewulf.go +++ b/pkg/generator/warewulf.go @@ -3,10 +3,12 @@ package generator import ( "fmt" "maps" + "strings" "github.com/OpenCHAMI/configurator/pkg/client" "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" + "github.com/caarlos0/log" ) type Warewulf struct{} @@ -28,6 +30,7 @@ func (g *Warewulf) Generate(config *config.Config, params Params) (FileMap, erro smdClient = client.NewSmdClient(params.ClientOpts...) outputs = make(FileMap, len(params.Templates)) nodeEntries = "" + paths = []string{} ) // if we have a client, try making the request for the ethernet interfaces @@ -65,11 +68,11 @@ func (g *Warewulf) Generate(config *config.Config, params Params) (FileMap, erro // print message if verbose param is found if params.Verbose { - fmt.Printf("templates and files loaded: \n") for path, _ := range outputs { - fmt.Printf("\t%s", path) + paths = append(paths, path) } } + log.Info().Str("paths", strings.Join(paths, ":")).Msg("templates and files loaded: \n") return outputs, err } From e3a846182881111333b401971ce5fcf0ceeebd8a Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 3 Dec 2024 15:54:53 -0700 Subject: [PATCH 056/102] cmd: updated --host flag and removed --port --- cmd/generate.go | 9 +++++---- cmd/serve.go | 9 +++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index c1d02af..0e480d0 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -50,8 +50,9 @@ var generateCmd = &cobra.Command{ if verbose { b, err := json.MarshalIndent(conf, "", " ") if err != nil { - fmt.Printf("failed to marshal conf: %v\n", err) + log.Error().Err(err).Printf("failed to marshal config") } + // print the config file as JSON fmt.Printf("%v\n", string(b)) } @@ -92,17 +93,17 @@ var generateCmd = &cobra.Command{ } // Generate files by supplying a list of targets as string values. Currently, -// targets are defined statically in a conf file. Targets are ran recursively +// targets are defined statically in a config file. Targets are ran recursively // if more targets are nested in a defined target, but will not run additional // child targets if it is the same as the parent. // // NOTE: This may be changed in the future how this is done. func RunTargets(conf *config.Config, args []string, targets ...string) { - // generate conf with each supplied target + // generate config with each supplied target for _, target := range targets { outputBytes, err := generator.GenerateWithTarget(conf, target) if err != nil { - log.Error().Err(err).Msg("failed to generate conf") + log.Error().Err(err).Msg("failed to generate config") os.Exit(1) } diff --git a/cmd/serve.go b/cmd/serve.go index d421538..740bef7 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -32,7 +32,7 @@ var serveCmd = &cobra.Command{ } } - // show conf as JSON and generators if verbose + // show config as JSON and generators if verbose if verbose { b, err := json.MarshalIndent(conf, "", "\t") if err != nil { @@ -44,9 +44,7 @@ var serveCmd = &cobra.Command{ // set up the routes and start the serve server := server.Server{ Config: &conf, - Server: &http.Server{ - Addr: fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port), - }, + Server: &http.Server{Addr: conf.Server.Host}, Jwks: server.Jwks{ Uri: conf.Server.Jwks.Uri, Retries: conf.Server.Jwks.Retries, @@ -67,8 +65,7 @@ var serveCmd = &cobra.Command{ } func init() { - serveCmd.Flags().StringVar(&conf.Server.Host, "host", conf.Server.Host, "set the server host") - serveCmd.Flags().IntVar(&conf.Server.Port, "port", conf.Server.Port, "set the server port") + serveCmd.Flags().StringVar(&conf.Server.Host, "host", conf.Server.Host, "set the server host and port") // serveCmd.Flags().StringVar(&pluginPath, "plugin", "", "set the generator plugins directory path") serveCmd.Flags().StringVar(&conf.Server.Jwks.Uri, "jwks-uri", conf.Server.Jwks.Uri, "set the JWKS url to fetch public key") serveCmd.Flags().IntVar(&conf.Server.Jwks.Retries, "jwks-fetch-retries", conf.Server.Jwks.Retries, "set the JWKS fetch retry count") From 678e6b66bdb0f28fceb6519e4f4eb820801d5343 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 4 Dec 2024 15:44:48 -0700 Subject: [PATCH 057/102] fix: changed logging import to zerolog --- pkg/generator/warewulf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/generator/warewulf.go b/pkg/generator/warewulf.go index 6e35e55..bdfbda1 100644 --- a/pkg/generator/warewulf.go +++ b/pkg/generator/warewulf.go @@ -8,7 +8,7 @@ import ( "github.com/OpenCHAMI/configurator/pkg/client" "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" - "github.com/caarlos0/log" + "github.com/rs/zerolog/log" ) type Warewulf struct{} From 31d14bcc53c1a77a1b3e2cb7c86f5d8da07ef17d Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 4 Dec 2024 15:45:07 -0700 Subject: [PATCH 058/102] tests: updated to use new API --- tests/generate_local.hurl | 22 +++++++++++++++++++ tests/generate_test.go | 46 +++++++++++---------------------------- 2 files changed, 35 insertions(+), 33 deletions(-) create mode 100644 tests/generate_local.hurl diff --git a/tests/generate_local.hurl b/tests/generate_local.hurl new file mode 100644 index 0000000..d198e46 --- /dev/null +++ b/tests/generate_local.hurl @@ -0,0 +1,22 @@ +## +## Run these tests after starting server with `configurator serve...` +## + +# Generate a `example` config with default plugin and template +GET http://127.0.0.1:3334/generate?target=example +HTTP 200 + +# Create a new target using the API +POST http://127.0.0.1:3334/targets +{ + "name": "test", + "plugin": "example", + "templates": [{ + "contents": "This is an example template used with the example plugin." + }] +} +HTTP 200 + +# Test the new target just add from POST above +GET http://127.0.0.1:3334/generate?target=example +HTTP 200 \ No newline at end of file diff --git a/tests/generate_test.go b/tests/generate_test.go index 39ba06e..d5e400a 100644 --- a/tests/generate_test.go +++ b/tests/generate_test.go @@ -9,6 +9,7 @@ import ( "testing" configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/server" "github.com/OpenCHAMI/configurator/pkg/util" @@ -22,7 +23,7 @@ func (g *TestGenerator) GetVersion() string { return "v1.0.0" } func (g *TestGenerator) GetDescription() string { return "This is a plugin created for running tests." } -func (g *TestGenerator) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) { // Jinja 2 template file files := [][]byte{ []byte(` @@ -42,28 +43,16 @@ This is another testing Jinja 2 template file using {{plugin_name}}. "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), - }, files...) + }, params.Templates) if err != nil { return nil, fmt.Errorf("failed to apply templates: %v", err) } - // make sure we're able to receive certain arguments when passed - params := generator.GetParams(opts...) - if len(params) <= 0 { - return nil, fmt.Errorf("expect at least one params, but found none") - } - // make sure we have a valid config we can access if config == nil { return nil, fmt.Errorf("invalid config (config is nil)") } - // make sure we're able to get a valid client as well - client := generator.GetClient(params) - if client == nil { - return nil, fmt.Errorf("invalid client (client is nil)") - } - // TODO: make sure we can get a target // make sure we have the same number of files in file list @@ -100,7 +89,7 @@ type TestGenerator struct{} func (g *TestGenerator) GetName() string { return "test" } func (g *TestGenerator) GetVersion() string { return "v1.0.0" } func (g *TestGenerator) GetDescription() string { return "This is a plugin creating for running tests." } -func (g *TestGenerator) Generate(config *configurator.Config, opts ...util.Option) (generator.FileMap, error) { +func (g *TestGenerator) Generate(config *configurator.Config, opts ...generator.Option) (generator.FileMap, error) { return generator.FileMap{"test": []byte("test")}, nil } var Generator TestGenerator @@ -186,7 +175,7 @@ var Generator TestGenerator GetName() string GetVersion() string GetDescription() string - Generate(*configurator.Config, ...util.Option) (generator.FileMap, error) + Generate(*config.Config, ...util.Option) (generator.FileMap, error) }); !ok { t.Error("plugin does not implement all of the generator interface") } @@ -203,7 +192,7 @@ var Generator TestGenerator GetName() string GetVersion() string GetDescription() string - Generate(*configurator.Config, ...util.Option) (generator.FileMap, error) + Generate(*config.Config, ...util.Option) (generator.FileMap, error) }); !ok { t.Error("plugin does not implement all of the generator interface") } @@ -314,9 +303,8 @@ var Generator InvalidGenerator // we're not doing it here since that's not what is being tested. func TestGenerateExample(t *testing.T) { var ( - config = configurator.NewConfig() - client = configurator.NewSmdClient() - gen = TestGenerator{} + conf = config.New() + gen = TestGenerator{} ) // make sure our generator returns expected strings @@ -333,11 +321,7 @@ func TestGenerateExample(t *testing.T) { }) // try to generate a file with templating applied - fileMap, err := gen.Generate( - &config, - generator.WithTarget("test"), - generator.WithClient(client), - ) + fileMap, err := gen.Generate(&conf, generator.Params{}) if err != nil { t.Fatalf("failed to generate file: %v", err) } @@ -356,8 +340,7 @@ func TestGenerateExample(t *testing.T) { // try and load the plugin from a lib here either. func TestGenerateExampleWithServer(t *testing.T) { var ( - config = configurator.NewConfig() - client = configurator.NewSmdClient() + conf = config.New() gen = TestGenerator{} headers = make(map[string]string, 0) ) @@ -365,13 +348,13 @@ func TestGenerateExampleWithServer(t *testing.T) { // NOTE: Currently, the server needs a config to know where to get load plugins, // and how to handle targets/templates. This will be simplified in the future to // decoupled the server from required a config altogether. - config.Targets["test"] = configurator.Target{ + conf.Targets["test"] = configurator.Target{ TemplatePaths: []string{}, FilePaths: []string{}, } // create new server, add test generator, and start in background - server := server.New(&config) + server := server.New(&conf) server.GeneratorParams.Generators = map[string]generator.Generator{ "test": &gen, } @@ -390,10 +373,7 @@ func TestGenerateExampleWithServer(t *testing.T) { // // NOTE: we don't actually use the config in this plugin implementation, // but we do check that a valid config was passed. - fileMap, err := gen.Generate( - &config, - generator.WithClient(client), - ) + fileMap, err := gen.Generate(&conf, generator.Params{}) if err != nil { t.Fatalf("failed to generate file: %v", err) } From 0569c33633a090423ace0419ce432435bcc22248 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 4 Dec 2024 15:48:45 -0700 Subject: [PATCH 059/102] refactor: converted more fmt.Printf to log.* --- cmd/generate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 0e480d0..283875c 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -36,7 +36,7 @@ var generateCmd = &cobra.Command{ } else { // TODO: try and fetch token first if it is needed if verbose { - fmt.Printf("No token found. Attempting to generate conf without one...\n") + log.Warn().Msg("No token found. Attempting to generate conf without one...\n") } } } @@ -61,7 +61,7 @@ var generateCmd = &cobra.Command{ RunTargets(&conf, args, targets...) } else { if pluginPath == "" { - fmt.Printf("no plugin path specified") + log.Error().Msg("no plugin path specified") return } From ebd5c46092acacdb3fbf9ec22bb4318e83f77868 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 4 Dec 2024 15:52:22 -0700 Subject: [PATCH 060/102] fix: corrected log message --- cmd/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/generate.go b/cmd/generate.go index 283875c..b2ca988 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -50,7 +50,7 @@ var generateCmd = &cobra.Command{ if verbose { b, err := json.MarshalIndent(conf, "", " ") if err != nil { - log.Error().Err(err).Printf("failed to marshal config") + log.Error().Err(err).Msg("failed to marshal config") } // print the config file as JSON fmt.Printf("%v\n", string(b)) From 221bb9a23a454488430443f838888f355dc20d0d Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 4 Dec 2024 15:59:05 -0700 Subject: [PATCH 061/102] tests: changed how the test generator is added --- tests/generate_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/generate_test.go b/tests/generate_test.go index d5e400a..ce42bec 100644 --- a/tests/generate_test.go +++ b/tests/generate_test.go @@ -355,9 +355,7 @@ func TestGenerateExampleWithServer(t *testing.T) { // create new server, add test generator, and start in background server := server.New(&conf) - server.GeneratorParams.Generators = map[string]generator.Generator{ - "test": &gen, - } + generator.DefaultGenerators["test"] = &gen go server.Serve() // make request to server to generate a file From 7cb4404cbb116b5bdf2a5f1c3a871290622ec7da Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 4 Dec 2024 16:06:16 -0700 Subject: [PATCH 062/102] tests: fixed minor issues --- tests/generate_test.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/generate_test.go b/tests/generate_test.go index ce42bec..ce3d8d0 100644 --- a/tests/generate_test.go +++ b/tests/generate_test.go @@ -39,7 +39,7 @@ This is another testing Jinja 2 template file using {{plugin_name}}. } // apply Jinja templates to file - fileList, err := generator.ApplyTemplates(generator.Mappings{ + fileMap, err := generator.ApplyTemplates(generator.Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), @@ -56,16 +56,10 @@ This is another testing Jinja 2 template file using {{plugin_name}}. // TODO: make sure we can get a target // make sure we have the same number of files in file list - if len(files) != len(fileList) { + if len(files) != len(fileMap) { return nil, fmt.Errorf("file list output count is not the same as the input") } - // convert file list to file map - fileMap := make(generator.FileMap, len(fileList)) - for i, contents := range fileList { - fileMap[fmt.Sprintf("t-%d.txt", i)] = contents - } - return fileMap, nil } @@ -175,7 +169,7 @@ var Generator TestGenerator GetName() string GetVersion() string GetDescription() string - Generate(*config.Config, ...util.Option) (generator.FileMap, error) + Generate(*config.Config, generator.Params) (generator.FileMap, error) }); !ok { t.Error("plugin does not implement all of the generator interface") } @@ -192,7 +186,7 @@ var Generator TestGenerator GetName() string GetVersion() string GetDescription() string - Generate(*config.Config, ...util.Option) (generator.FileMap, error) + Generate(*config.Config, generator.Params) (generator.FileMap, error) }); !ok { t.Error("plugin does not implement all of the generator interface") } From 9cef01acf3ad8a814912c0751ce2e21648c9f0dd Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 10 Dec 2024 15:08:38 -0700 Subject: [PATCH 063/102] go.mod: updated deps --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index e9ff5f2..cea28f4 100644 --- a/go.mod +++ b/go.mod @@ -40,3 +40,5 @@ require ( golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect ) + +replace github.com/OpenCHAMI/configurator/pkg => ./pkg From c8aa4aae931ff954d7b4b00bf419965c84cf972b Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 10 Dec 2024 15:09:13 -0700 Subject: [PATCH 064/102] generator: added check for *.so extension --- pkg/generator/generator.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index a56c026..19fe3fe 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -130,6 +130,11 @@ func LoadPlugins(dirpath string, opts ...Option) (map[string]Generator, error) { return nil } + // only try loading if file has .so extension + if filepath.Ext(path) != ".so" { + return nil + } + // load the generator plugin from current path gen, err := LoadPlugin(path) if err != nil { From 0fc81ac67c4f9ab93f41ce31bb6ed38d5fd6bd2a Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 10 Dec 2024 15:09:43 -0700 Subject: [PATCH 065/102] tests: updated tests to use local packages --- tests/generate_test.go | 75 +++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/tests/generate_test.go b/tests/generate_test.go index ce3d8d0..077ad72 100644 --- a/tests/generate_test.go +++ b/tests/generate_test.go @@ -3,9 +3,11 @@ package tests import ( "encoding/json" "fmt" + "log" "net/http" "os" "os/exec" + "path/filepath" "testing" configurator "github.com/OpenCHAMI/configurator/pkg" @@ -15,6 +17,12 @@ import ( "github.com/OpenCHAMI/configurator/pkg/util" ) +var ( + workDir string + replaceDir string + err error +) + // A valid test generator that implements the `Generator` interface. type TestGenerator struct{} @@ -56,47 +64,59 @@ This is another testing Jinja 2 template file using {{plugin_name}}. // TODO: make sure we can get a target // make sure we have the same number of files in file list - if len(files) != len(fileMap) { - return nil, fmt.Errorf("file list output count is not the same as the input") + var ( + fileInputCount = len(files) + fileOutputCount = len(fileMap) + ) + if fileInputCount != fileOutputCount { + return nil, fmt.Errorf("file output count (%d) is not the same as the input (%d)", fileOutputCount, fileInputCount) } return fileMap, nil } +func init() { + workDir, err = os.Getwd() + if err != nil { + log.Fatalf("failed to get working directory: %v", err) + } + replaceDir = fmt.Sprintf("%s", filepath.Dir(workDir)) +} + // Test building and loading plugins func TestPlugin(t *testing.T) { var ( testPluginDir = t.TempDir() testPluginPath = fmt.Sprintf("%s/test-plugin.so", testPluginDir) testPluginSourcePath = fmt.Sprintf("%s/test-plugin.go", testPluginDir) - testPluginSource = []byte(` -package main + testPluginSource = []byte( + `package main import ( - configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/generator" - "github.com/OpenCHAMI/configurator/pkg/util" ) type TestGenerator struct{} -func (g *TestGenerator) GetName() string { return "test" } -func (g *TestGenerator) GetVersion() string { return "v1.0.0" } -func (g *TestGenerator) GetDescription() string { return "This is a plugin creating for running tests." } -func (g *TestGenerator) Generate(config *configurator.Config, opts ...generator.Option) (generator.FileMap, error) { +func (g *TestGenerator) GetName() string { return "test" } +func (g *TestGenerator) GetVersion() string { return "v1.0.0" } +func (g *TestGenerator) GetDescription() string { + return "This is a plugin creating for running tests." +} +func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) { return generator.FileMap{"test": []byte("test")}, nil } -var Generator TestGenerator - `) + +var Generator TestGenerator`) ) - wd, err := os.Getwd() - if err != nil { - t.Errorf("failed to get working directory: %v", err) - } + // get directory to replace remote pkg with local + // _, filename, _, _ := runtime.Caller(0) + // replaceDir := fmt.Sprintf("%s", filepath.Dir(workDir)) // show all paths to make sure we're using the correct ones - fmt.Printf("(TestPlugin) working directory: %v\n", wd) + fmt.Printf("(TestPlugin) working directory: %v\n", workDir) fmt.Printf("(TestPlugin) plugin directory: %v\n", testPluginDir) fmt.Printf("(TestPlugin) plugin path: %v\n", testPluginPath) fmt.Printf("(TestPlugin) plugin source path: %v\n", testPluginSourcePath) @@ -134,6 +154,12 @@ var Generator TestGenerator t.Fatalf("failed to execute command: %v\n%s", err, string(output)) } + // use the local `pkg` instead of the release one + cmd = exec.Command("bash", "-c", fmt.Sprintf("go mod edit -replace=github.com/OpenCHAMI/configurator=%s", replaceDir)) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("failed to execute command: %v\n%s", err, string(output)) + } + // run `go mod tidy` for dependencies cmd = exec.Command("bash", "-c", "go mod tidy") if output, err := cmd.CombinedOutput(); err != nil { @@ -216,16 +242,15 @@ var Generator InvalidGenerator `) ) - wd, err := os.Getwd() - if err != nil { - t.Errorf("failed to get working directory: %v", err) - } // show all paths to make sure we're using the correct ones - fmt.Printf("(TestPluginWithInvalidOrNoSymbol) working directory: %v\n", wd) + fmt.Printf("(TestPluginWithInvalidOrNoSymbol) working directory: %v\n", workDir) fmt.Printf("(TestPluginWithInvalidOrNoSymbol) plugin directory: %v\n", testPluginDir) fmt.Printf("(TestPluginWithInvalidOrNoSymbol) plugin path: %v\n", testPluginPath) fmt.Printf("(TestPluginWithInvalidOrNoSymbol) plugin source path: %v\n", testPluginSourcePath) + // get directory to replace remote pkg with local + // _, filename, _, _ := runtime.Caller(0) + // make temporary directory to test plugin err = os.MkdirAll(testPluginDir, os.ModeDir) if err != nil { @@ -259,6 +284,12 @@ var Generator InvalidGenerator t.Fatalf("failed to execute command: %v\n%s", err, string(output)) } + // use the local `pkg` instead of the release one + cmd = exec.Command("bash", "-c", fmt.Sprintf("go mod edit -replace=github.com/OpenCHAMI/configurator=%s", replaceDir)) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("failed to execute command: %v\n%s", err, string(output)) + } + // run `go mod tidy` for dependencies cmd = exec.Command("bash", "-c", "go mod tidy") if output, err := cmd.CombinedOutput(); err != nil { From 184881924479b06755648eb4fef3049bc0492773 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 10 Dec 2024 15:22:03 -0700 Subject: [PATCH 066/102] config: updated to use only host and not port var --- pkg/config/config.go | 10 +++------- pkg/server/server.go | 4 +--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 2a6ef79..7767e5b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -34,11 +34,8 @@ type Config struct { // Creates a new config with default parameters. func New() Config { return Config{ - Version: "", - SmdClient: client.SmdClient{ - Host: "http://127.0.0.1", - Port: 27779, - }, + Version: "", + SmdClient: client.SmdClient{Host: "http://127.0.0.1:27779"}, Targets: map[string]configurator.Target{ "dnsmasq": configurator.Target{ Plugin: "", @@ -59,8 +56,7 @@ func New() Config { PluginDirs: []string{}, Server: Server{ - Host: "127.0.0.1", - Port: 3334, + Host: "127.0.0.1:3334", Jwks: Jwks{ Uri: "", Retries: 5, diff --git a/pkg/server/server.go b/pkg/server/server.go index 689a0fd..24822ed 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -58,9 +58,7 @@ func New(conf *config.Config) *Server { // return based on config values return &Server{ Config: conf, - Server: &http.Server{ - Addr: fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port), - }, + Server: &http.Server{Addr: fmt.Sprintf("%s", conf.Server.Host)}, Jwks: Jwks{ Uri: conf.Server.Jwks.Uri, Retries: conf.Server.Jwks.Retries, From d2b6178350b2bdafb4e0083fcda5ea55e1b1cf4f Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 10 Dec 2024 15:22:37 -0700 Subject: [PATCH 067/102] tests: fixed issue with server not starting with correct config --- tests/generate_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/generate_test.go b/tests/generate_test.go index 077ad72..f458ebc 100644 --- a/tests/generate_test.go +++ b/tests/generate_test.go @@ -23,6 +23,8 @@ var ( err error ) +const () + // A valid test generator that implements the `Generator` interface. type TestGenerator struct{} @@ -33,17 +35,21 @@ func (g *TestGenerator) GetDescription() string { } func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) { // Jinja 2 template file - files := [][]byte{ - []byte(` + files := map[string]generator.Template{ + "test1": generator.Template{ + Contents: []byte(` Name: {{plugin_name}} Version: {{plugin_version}} Description: {{plugin_description}} This is the first test template file. - `), - []byte(` + `), + }, + "test2": generator.Template{ + Contents: []byte(` This is another testing Jinja 2 template file using {{plugin_name}}. - `), + `), + }, } // apply Jinja templates to file @@ -51,7 +57,7 @@ This is another testing Jinja 2 template file using {{plugin_name}}. "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), - }, params.Templates) + }, files) if err != nil { return nil, fmt.Errorf("failed to apply templates: %v", err) } From ad45a540f0638147b9ec2ee4d5ee6af284c0c35e Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 10 Dec 2024 18:31:03 -0700 Subject: [PATCH 068/102] chore: tidy and code cleanup --- cmd/config.go | 5 ++--- go.mod | 1 - pkg/client/smd.go | 7 ++++--- pkg/generator/dhcpd.go | 4 ---- pkg/generator/generator.go | 2 +- tests/generate_test.go | 2 -- 6 files changed, 7 insertions(+), 14 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 05e183d..f56c34c 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -1,8 +1,7 @@ package cmd import ( - "fmt" - + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/OpenCHAMI/configurator/pkg/config" @@ -20,7 +19,7 @@ var configCmd = &cobra.Command{ for _, path := range args { // check and make sure something doesn't exist first if exists, err := util.PathExists(path); exists || err != nil { - fmt.Printf("file or directory exists\n") + log.Error().Err(err).Msg("file or directory exists") continue } config.SaveDefault(path) diff --git a/go.mod b/go.mod index cea28f4..40878bf 100644 --- a/go.mod +++ b/go.mod @@ -41,4 +41,3 @@ require ( golang.org/x/text v0.16.0 // indirect ) -replace github.com/OpenCHAMI/configurator/pkg => ./pkg diff --git a/pkg/client/smd.go b/pkg/client/smd.go index 9a01401..7b001c2 100644 --- a/pkg/client/smd.go +++ b/pkg/client/smd.go @@ -8,6 +8,7 @@ import ( "net/http" configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/caarlos0/log" ) // An struct that's meant to extend functionality of the base HTTP client by @@ -57,7 +58,7 @@ func (client *SmdClient) FetchEthernetInterfaces(verbose bool) ([]configurator.E // print what we got if verbose is set if verbose { - fmt.Printf("Ethernet Interfaces: %v\n", string(bytes)) + log.Info().Str("ethernet_interfaces", string(bytes)).Msg("found interfaces") } return eths, nil @@ -99,7 +100,7 @@ func (client *SmdClient) FetchComponents(verbose bool) ([]configurator.Component // print what we got if verbose is set if verbose { - fmt.Printf("Components: %v\n", string(bytes)) + log.Info().Str("components", string(bytes)).Msg("found components") } return comps, nil @@ -138,7 +139,7 @@ func (client *SmdClient) FetchRedfishEndpoints(verbose bool) ([]configurator.Red // show the final result if verbose { - fmt.Printf("Redfish endpoints: %v\n", string(b)) + log.Info().Str("redfish_endpoints", string(b)).Msg("found redfish endpoints") } return eps, nil diff --git a/pkg/generator/dhcpd.go b/pkg/generator/dhcpd.go index 7425a57..cc32a48 100644 --- a/pkg/generator/dhcpd.go +++ b/pkg/generator/dhcpd.go @@ -54,10 +54,6 @@ func (g *DHCPd) Generate(config *config.Config, params Params) (FileMap, error) computeNodes += fmt.Sprintf("host %s { hardware ethernet %s; fixed-address %s} ", eth.ComponentId, eth.MacAddress, eth.IpAddresses[0]) } computeNodes += "# =====================================================================" - - if params.Verbose { - fmt.Printf("") - } return ApplyTemplates(Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 19fe3fe..b7d5fa8 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -143,7 +143,7 @@ func LoadPlugins(dirpath string, opts ...Option) (map[string]Generator, error) { // show the plugins found if verbose flag is set if params.Verbose { - fmt.Printf("-- found plugin '%s'\n", gen.GetName()) + log.Info().Str("plugin_name", gen.GetName()).Msg("found plugin") } // map each generator plugin by name for lookup diff --git a/tests/generate_test.go b/tests/generate_test.go index f458ebc..21ff978 100644 --- a/tests/generate_test.go +++ b/tests/generate_test.go @@ -23,8 +23,6 @@ var ( err error ) -const () - // A valid test generator that implements the `Generator` interface. type TestGenerator struct{} From d02a49fe805443057caefa7ed8b49e142810d576 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 10 Dec 2024 18:35:55 -0700 Subject: [PATCH 069/102] refactor: changed fmt.Print to log.Info in cmd/fetch --- cmd/fetch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/fetch.go b/cmd/fetch.go index 0bbf309..c61bffb 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -33,7 +33,7 @@ var fetchCmd = &cobra.Command{ } else { // TODO: try and fetch token first if it is needed if verbose { - fmt.Printf("No token found. Attempting to generate config without one...\n") + log.Warn().Msg("No token found. Attempting to generate config without one...") } } } From 66bb1e6c1ae90b054965cc8f1ccb9b7cadde7427 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 10 Dec 2024 19:09:49 -0700 Subject: [PATCH 070/102] fix: changed to use correct logging library --- pkg/client/smd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/smd.go b/pkg/client/smd.go index 7b001c2..e42c0fa 100644 --- a/pkg/client/smd.go +++ b/pkg/client/smd.go @@ -8,7 +8,7 @@ import ( "net/http" configurator "github.com/OpenCHAMI/configurator/pkg" - "github.com/caarlos0/log" + "github.com/rs/zerolog/log" ) // An struct that's meant to extend functionality of the base HTTP client by From b6c3533327b964a4f8b5b8d3f47b78c45ac76349 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 10 Dec 2024 19:10:07 -0700 Subject: [PATCH 071/102] chore: tidy and code cleanup --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 40878bf..e9ff5f2 100644 --- a/go.mod +++ b/go.mod @@ -40,4 +40,3 @@ require ( golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect ) - From 10ed21c5c1cbb88bf752afec4ae1e84d0d6853e7 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 11 Dec 2024 08:57:24 -0700 Subject: [PATCH 072/102] fix: added yaml tag to prevent marshaler from crashing --- pkg/client/smd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/smd.go b/pkg/client/smd.go index e42c0fa..1494d7e 100644 --- a/pkg/client/smd.go +++ b/pkg/client/smd.go @@ -16,7 +16,7 @@ import ( // used in generator plugins to fetch data when it is needed to substitute // values for the Jinja templates used. type SmdClient struct { - http.Client `json:"-"` + http.Client `json:"-" yaml:"-"` Host string `yaml:"host"` Port int `yaml:"port"` AccessToken string `yaml:"access-token"` From eda3cce177c65efedcffec290d1a0dec03df0290 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 11 Dec 2024 08:58:05 -0700 Subject: [PATCH 073/102] refactor: changed log.Printf to log.Error in config --- pkg/config/config.go | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7767e5b..b6a138a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,10 +1,11 @@ package config import ( - "log" "os" "path/filepath" + "github.com/rs/zerolog/log" + configurator "github.com/OpenCHAMI/configurator/pkg" "github.com/OpenCHAMI/configurator/pkg/client" "gopkg.in/yaml.v2" @@ -34,26 +35,9 @@ type Config struct { // Creates a new config with default parameters. func New() Config { return Config{ - Version: "", - SmdClient: client.SmdClient{Host: "http://127.0.0.1:27779"}, - Targets: map[string]configurator.Target{ - "dnsmasq": configurator.Target{ - Plugin: "", - TemplatePaths: []string{}, - }, - "conman": configurator.Target{ - Plugin: "", - TemplatePaths: []string{}, - }, - "warewulf": configurator.Target{ - Plugin: "", - TemplatePaths: []string{ - "templates/warewulf/defaults/node.jinja", - "templates/warewulf/defaults/provision.jinja", - }, - }, - }, - + Version: "", + SmdClient: client.SmdClient{Host: "http://127.0.0.1:27779"}, + Targets: map[string]configurator.Target{}, PluginDirs: []string{}, Server: Server{ Host: "127.0.0.1:3334", @@ -69,12 +53,12 @@ func Load(path string) Config { var c Config = New() file, err := os.ReadFile(path) if err != nil { - log.Printf("failed to read config file: %v\n", err) + log.Error().Err(err).Msg("failed to read config file") return c } err = yaml.Unmarshal(file, &c) if err != nil { - log.Fatalf("failed to unmarshal config: %v\n", err) + log.Error().Err(err).Msg("failed to unmarshal config") return c } return c @@ -87,12 +71,12 @@ func (config *Config) Save(path string) { } data, err := yaml.Marshal(config) if err != nil { - log.Printf("failed to marshal config: %v\n", err) + log.Error().Err(err).Msg("failed to marshal config") return } err = os.WriteFile(path, data, os.ModePerm) if err != nil { - log.Printf("failed to write default config file: %v\n", err) + log.Error().Err(err).Msg("failed to write default config file") return } } @@ -105,12 +89,12 @@ func SaveDefault(path string) { var c = New() data, err := yaml.Marshal(c) if err != nil { - log.Printf("failed to marshal config: %v\n", err) + log.Error().Err(err).Msg("failed to marshal config") return } err = os.WriteFile(path, data, os.ModePerm) if err != nil { - log.Printf("failed to write default config file: %v\n", err) + log.Error().Err(err).Msg("failed to write default config file") return } } From ebe4e02cf0f670ebc1b6cadc5bf606950719cb7f Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 11 Dec 2024 10:53:10 -0700 Subject: [PATCH 074/102] fix: minor changes --- pkg/client/smd.go | 2 +- pkg/config/config.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/client/smd.go b/pkg/client/smd.go index 1494d7e..eb80ce8 100644 --- a/pkg/client/smd.go +++ b/pkg/client/smd.go @@ -151,7 +151,7 @@ func (client *SmdClient) makeRequest(endpoint string) ([]byte, error) { } // fetch DHCP related information from SMD's endpoint: - url := fmt.Sprintf("%s:%d/hsm/v2%s", client.Host, client.Port, endpoint) + url := fmt.Sprintf("%s/hsm/v2%s", client.Host, endpoint) req, err := http.NewRequest(http.MethodGet, url, bytes.NewBuffer([]byte{})) if err != nil { return nil, fmt.Errorf("failed to create new HTTP request: %v", err) diff --git a/pkg/config/config.go b/pkg/config/config.go index b6a138a..c20797b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -74,7 +74,7 @@ func (config *Config) Save(path string) { log.Error().Err(err).Msg("failed to marshal config") return } - err = os.WriteFile(path, data, os.ModePerm) + err = os.WriteFile(path, data, 0o644) if err != nil { log.Error().Err(err).Msg("failed to write default config file") return @@ -92,7 +92,7 @@ func SaveDefault(path string) { log.Error().Err(err).Msg("failed to marshal config") return } - err = os.WriteFile(path, data, os.ModePerm) + err = os.WriteFile(path, data, 0o644) if err != nil { log.Error().Err(err).Msg("failed to write default config file") return From e1ab1e710292c195fbc26efc0e8648ced682ad61 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 12 Dec 2024 14:28:48 -0700 Subject: [PATCH 075/102] refactor: added client opts to serve.cmd and more logging info --- cmd/generate.go | 22 ++++++++++++++++------ cmd/serve.go | 9 +-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index b2ca988..98b22a2 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" + "github.com/OpenCHAMI/configurator/pkg/client" "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/generator" "github.com/OpenCHAMI/configurator/pkg/util" @@ -79,6 +80,15 @@ var generateCmd = &cobra.Command{ Templates: templates, } + // set the client options + opts := []client.Option{} + if conf.AccessToken != "" { + params.ClientOpts = append(opts, client.WithAccessToken(conf.AccessToken)) + } + if conf.CertPath != "" { + params.ClientOpts = append(opts, client.WithCertPoolFile(conf.CertPath)) + } + // run generator.Generate() with just plugin path and templates provided outputBytes, err := generator.Generate(pluginPath, params) if err != nil { @@ -103,7 +113,7 @@ func RunTargets(conf *config.Config, args []string, targets ...string) { for _, target := range targets { outputBytes, err := generator.GenerateWithTarget(conf, target) if err != nil { - log.Error().Err(err).Msg("failed to generate config") + log.Error().Err(err).Str("target", target).Msg("failed to generate config") os.Exit(1) } @@ -139,7 +149,7 @@ func writeOutput(outputBytes generator.FileMap, targetCount int, templateCount i for _, contents := range outputBytes { err := os.WriteFile(outputPath, contents, 0o644) if err != nil { - log.Error().Err(err).Msg("failed to write conf to file") + log.Error().Err(err).Str("path", outputPath).Msg("failed to write config file") os.Exit(1) } log.Info().Msgf("wrote file to '%s'\n", outputPath) @@ -148,7 +158,7 @@ func writeOutput(outputBytes generator.FileMap, targetCount int, templateCount i // write multiple files to archive, compress, then save to output path out, err := os.Create(fmt.Sprintf("%s.tar.gz", outputPath)) if err != nil { - log.Error().Err(err).Msg("failed to write archive") + log.Error().Err(err).Str("path", outputPath).Msg("failed to write archive") os.Exit(1) } files := make([]string, len(outputBytes)) @@ -159,7 +169,7 @@ func writeOutput(outputBytes generator.FileMap, targetCount int, templateCount i } err = util.CreateArchive(files, out) if err != nil { - log.Error().Err(err).Msg("failed to create archive") + log.Error().Err(err).Str("path", outputPath).Msg("failed to create archive") os.Exit(1) } @@ -167,7 +177,7 @@ func writeOutput(outputBytes generator.FileMap, targetCount int, templateCount i // write multiple files in directory using template name err := os.MkdirAll(filepath.Clean(outputPath), 0o755) if err != nil { - log.Error().Err(err).Msg("failed to make output directory") + log.Error().Err(err).Str("path", filepath.Clean(outputPath)).Msg("failed to make output directory") os.Exit(1) } for path, contents := range outputBytes { @@ -175,7 +185,7 @@ func writeOutput(outputBytes generator.FileMap, targetCount int, templateCount i cleanPath := fmt.Sprintf("%s/%s", filepath.Clean(outputPath), filename) err := os.WriteFile(cleanPath, contents, 0o755) if err != nil { - log.Error().Err(err).Msg("failed to write conf to file") + log.Error().Err(err).Str("path", path).Msg("failed to write config to file") os.Exit(1) } log.Info().Msgf("wrote file to '%s'\n", cleanPath) diff --git a/cmd/serve.go b/cmd/serve.go index 740bef7..fd76f02 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -42,14 +42,7 @@ var serveCmd = &cobra.Command{ } // set up the routes and start the serve - server := server.Server{ - Config: &conf, - Server: &http.Server{Addr: conf.Server.Host}, - Jwks: server.Jwks{ - Uri: conf.Server.Jwks.Uri, - Retries: conf.Server.Jwks.Retries, - }, - } + server := server.New(&conf) // start listening with the server err := server.Serve() From 9951b9a1f35ecfd712c0fa0ead144f6323895335 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 12 Dec 2024 14:29:41 -0700 Subject: [PATCH 076/102] refactor: added more logging info --- pkg/generator/generator.go | 4 ++-- pkg/server/server.go | 21 +++++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index b7d5fa8..34333b0 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -212,7 +212,7 @@ func GenerateWithTarget(config *config.Config, target string) (FileMap, error) { // load target information from config targetInfo, ok = config.Targets[target] if !ok { - log.Warn().Msg("target not found in config") + log.Warn().Str("target", target).Msg("target not found in config") } // if no plugin supplied in config target, then using the target supplied @@ -224,7 +224,7 @@ func GenerateWithTarget(config *config.Config, target string) (FileMap, error) { generator, ok = DefaultGenerators[target] if !ok { // only load the plugin needed for this target if we don't find default - log.Error().Msg("could not find target in default generators") + log.Warn().Str("target", target).Msg("could not find target in default generators") generator, err = LoadPlugin(targetInfo.Plugin) if err != nil { return nil, fmt.Errorf("failed to load plugin: %v", err) diff --git a/pkg/server/server.go b/pkg/server/server.go index 24822ed..f8b57c5 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -58,7 +58,7 @@ func New(conf *config.Config) *Server { // return based on config values return &Server{ Config: conf, - Server: &http.Server{Addr: fmt.Sprintf("%s", conf.Server.Host)}, + Server: &http.Server{Addr: conf.Server.Host}, Jwks: Jwks{ Uri: conf.Server.Jwks.Uri, Retries: conf.Server.Jwks.Retries, @@ -138,29 +138,26 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * return func(w http.ResponseWriter, r *http.Request) { // get all of the expect query URL params and validate var ( - target string = r.URL.Query().Get("target") + targetParam string = r.URL.Query().Get("target") + target *Target = s.getTarget(targetParam) + outputs generator.FileMap + err error ) s.GeneratorParams = parseGeneratorParams(r, opts...) - if target == "" { + if targetParam == "" { writeErrorResponse(w, "must specify a target") return } // try to generate with target supplied by client first - var ( - t *Target = s.getTarget(target) - outputs generator.FileMap - err error - ) - - if t != nil { - outputs, err = generator.Generate(t.PluginPath, s.GeneratorParams) + if target != nil { + outputs, err = generator.Generate(target.PluginPath, s.GeneratorParams) if err != nil { } } else { // try and generate a new config file from supplied params - outputs, err = generator.GenerateWithTarget(s.Config, target) + outputs, err = generator.GenerateWithTarget(s.Config, targetParam) if err != nil { writeErrorResponse(w, "failed to generate file: %v", err) return From cccf6321cc9f37de8609d5bc9632f1a171d84e7b Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 13 Dec 2024 12:23:17 -0700 Subject: [PATCH 077/102] fix: set clientopts correctly in generate.go --- cmd/generate.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 98b22a2..44e4e5d 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -81,12 +81,11 @@ var generateCmd = &cobra.Command{ } // set the client options - opts := []client.Option{} if conf.AccessToken != "" { - params.ClientOpts = append(opts, client.WithAccessToken(conf.AccessToken)) + params.ClientOpts = append(params.ClientOpts, client.WithAccessToken(conf.AccessToken)) } if conf.CertPath != "" { - params.ClientOpts = append(opts, client.WithCertPoolFile(conf.CertPath)) + params.ClientOpts = append(params.ClientOpts, client.WithCertPoolFile(conf.CertPath)) } // run generator.Generate() with just plugin path and templates provided From 34bbd1ce85c1ebe06612fa48d7e5710653c980c1 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 16 Dec 2024 15:24:52 -0700 Subject: [PATCH 078/102] server: fixed error message handling --- cmd/fetch.go | 12 ++++++++---- pkg/server/server.go | 35 ++++++++++++++++++++++------------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/cmd/fetch.go b/cmd/fetch.go index c61bffb..1c35a73 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -24,6 +24,11 @@ var fetchCmd = &cobra.Command{ return } + // check if we actually have any targets to run + if len(targets) <= 0 { + log.Error().Msg("must specify a target") + } + // check to see if an access token is available from env if conf.AccessToken == "" { // check if OCHAMI_ACCESS_TOKEN env var is set if no access token is provided and use that instead @@ -49,14 +54,13 @@ var fetchCmd = &cobra.Command{ url := fmt.Sprintf("%s/generate?target=%s", remoteHost, target) res, body, err := util.MakeRequest(url, http.MethodGet, nil, headers) if err != nil { - log.Error().Err(err).Msg("failed to make request") + log.Error().Err(err).Msg("failed to fetch files") return } // handle getting other error codes other than a 200 if res != nil { - if res.StatusCode == http.StatusOK { - log.Info().Msgf("%s\n", string(body)) - } + // NOTE: the server responses are already marshaled to JSON + fmt.Print(string(body)) } } }, diff --git a/pkg/server/server.go b/pkg/server/server.go index f8b57c5..c015c55 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -145,7 +145,7 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * ) s.GeneratorParams = parseGeneratorParams(r, opts...) if targetParam == "" { - writeErrorResponse(w, "must specify a target") + writeErrorResponse(w, true, "must specify a target") return } @@ -159,7 +159,8 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * // try and generate a new config file from supplied params outputs, err = generator.GenerateWithTarget(s.Config, targetParam) if err != nil { - writeErrorResponse(w, "failed to generate file: %v", err) + writeErrorResponse(w, false, "failed to generate file") + log.Error().Err(err).Msg("failed to generate file") return } } @@ -168,12 +169,12 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * tmp := generator.ConvertContentsToString(outputs) b, err := json.Marshal(tmp) if err != nil { - writeErrorResponse(w, "failed to marshal output: %v", err) + writeErrorResponse(w, true, "failed to marshal output: %v", err) return } _, err = w.Write(b) if err != nil { - writeErrorResponse(w, "failed to write response: %v", err) + writeErrorResponse(w, true, "failed to write response: %v", err) return } } @@ -193,34 +194,34 @@ func (s *Server) createTarget(w http.ResponseWriter, r *http.Request) { err error ) if r == nil { - writeErrorResponse(w, "request is invalid") + writeErrorResponse(w, true, "request is invalid") return } bytes, err = io.ReadAll(r.Body) if err != nil { - writeErrorResponse(w, "failed to read response body: %v", err) + writeErrorResponse(w, true, "failed to read response body: %v", err) return } defer r.Body.Close() err = json.Unmarshal(bytes, &target) if err != nil { - writeErrorResponse(w, "failed to unmarshal target: %v", err) + writeErrorResponse(w, true, "failed to unmarshal target: %v", err) return } // make sure a plugin and at least one template is supplied if target.Name == "" { - writeErrorResponse(w, "target name is required") + writeErrorResponse(w, true, "target name is required") return } if target.PluginPath == "" { - writeErrorResponse(w, "must supply a generator name") + writeErrorResponse(w, true, "must supply a generator name") return } if len(target.Templates) <= 0 { - writeErrorResponse(w, "must provided at least one template") + writeErrorResponse(w, true, "must provided at least one template") return } @@ -238,10 +239,18 @@ func (s *Server) getTarget(target string) *Target { // Wrapper function to simplify writting error message responses. This function // is only intended to be used with the service and nothing else. -func writeErrorResponse(w http.ResponseWriter, format string, a ...any) error { +func writeErrorResponse(w http.ResponseWriter, logServer bool, format string, a ...any) error { errmsg := fmt.Sprintf(format, a...) - log.Error().Msg(errmsg) - http.Error(w, errmsg, http.StatusInternalServerError) + if logServer { + + log.Error().Msg(errmsg) + } + bytes, _ := json.Marshal(map[string]any{ + "level": "error", + "time": time.Now().Unix(), + "message": errmsg, + }) + http.Error(w, string(bytes), http.StatusInternalServerError) return fmt.Errorf(errmsg) } From 5c9e9f0540a6ba05a1a0a33f7d81263e861e1f8a Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 16 Dec 2024 15:40:20 -0700 Subject: [PATCH 079/102] cmd: changed 'server' variable name to not collide with package name --- cmd/serve.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index fd76f02..8123936 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -41,11 +41,11 @@ var serveCmd = &cobra.Command{ fmt.Printf("%v\n", string(b)) } - // set up the routes and start the serve - server := server.New(&conf) - // start listening with the server - err := server.Serve() + var ( + s *server.Server = server.New(&conf) + err error = s.Serve() + ) if errors.Is(err, http.ErrServerClosed) { if verbose { log.Info().Msg("server closed") From 2b9e3d66d2c4b114c2aee2fafcad26095a056761 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Mon, 16 Dec 2024 16:03:41 -0700 Subject: [PATCH 080/102] fix: changed how writeErrorResponse works --- pkg/server/server.go | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index c015c55..7ea585a 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -145,7 +145,8 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * ) s.GeneratorParams = parseGeneratorParams(r, opts...) if targetParam == "" { - writeErrorResponse(w, true, "must specify a target") + err = writeErrorResponse(w, "must specify a target") + log.Error().Err(err).Msg("failed to parse generator params") return } @@ -159,7 +160,7 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * // try and generate a new config file from supplied params outputs, err = generator.GenerateWithTarget(s.Config, targetParam) if err != nil { - writeErrorResponse(w, false, "failed to generate file") + writeErrorResponse(w, "failed to generate file") log.Error().Err(err).Msg("failed to generate file") return } @@ -169,12 +170,14 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * tmp := generator.ConvertContentsToString(outputs) b, err := json.Marshal(tmp) if err != nil { - writeErrorResponse(w, true, "failed to marshal output: %v", err) + writeErrorResponse(w, "failed to marshal output: %v", err) + log.Error().Err(err).Msg("failed to marshal output") return } _, err = w.Write(b) if err != nil { - writeErrorResponse(w, true, "failed to write response: %v", err) + writeErrorResponse(w, "failed to write response: %v", err) + log.Error().Err(err).Msg("failed to write response") return } } @@ -194,34 +197,40 @@ func (s *Server) createTarget(w http.ResponseWriter, r *http.Request) { err error ) if r == nil { - writeErrorResponse(w, true, "request is invalid") + err = writeErrorResponse(w, "request is invalid") + log.Error().Err(err).Msg("request == nil") return } bytes, err = io.ReadAll(r.Body) if err != nil { - writeErrorResponse(w, true, "failed to read response body: %v", err) + writeErrorResponse(w, "failed to read response body: %v", err) + log.Error().Err(err).Msg("failed to read response body") return } defer r.Body.Close() err = json.Unmarshal(bytes, &target) if err != nil { - writeErrorResponse(w, true, "failed to unmarshal target: %v", err) + writeErrorResponse(w, "failed to unmarshal target: %v", err) + log.Error().Err(err).Msg("failed to unmarshal target") return } // make sure a plugin and at least one template is supplied if target.Name == "" { - writeErrorResponse(w, true, "target name is required") + err = writeErrorResponse(w, "target name is required") + log.Error().Err(err).Msg("set target as a URL query parameter") return } if target.PluginPath == "" { - writeErrorResponse(w, true, "must supply a generator name") + err = writeErrorResponse(w, "generator name is required") + log.Error().Err(err).Msg("must supply a generator name") return } if len(target.Templates) <= 0 { - writeErrorResponse(w, true, "must provided at least one template") + writeErrorResponse(w, "requires at least one template") + log.Error().Err(err).Msg("must supply at least one template") return } @@ -239,12 +248,8 @@ func (s *Server) getTarget(target string) *Target { // Wrapper function to simplify writting error message responses. This function // is only intended to be used with the service and nothing else. -func writeErrorResponse(w http.ResponseWriter, logServer bool, format string, a ...any) error { +func writeErrorResponse(w http.ResponseWriter, format string, a ...any) error { errmsg := fmt.Sprintf(format, a...) - if logServer { - - log.Error().Msg(errmsg) - } bytes, _ := json.Marshal(map[string]any{ "level": "error", "time": time.Now().Unix(), From 4ff8094988b7c1a6692adc0a66b7698233d6ab66 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 17 Dec 2024 16:28:13 -0700 Subject: [PATCH 081/102] fix: added os.Exit in commands with error --- cmd/fetch.go | 1 + cmd/serve.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/fetch.go b/cmd/fetch.go index 1c35a73..0b105a7 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -27,6 +27,7 @@ var fetchCmd = &cobra.Command{ // check if we actually have any targets to run if len(targets) <= 0 { log.Error().Msg("must specify a target") + os.Exit(1) } // check to see if an access token is available from env diff --git a/cmd/serve.go b/cmd/serve.go index 8123936..48e3629 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -37,6 +37,7 @@ var serveCmd = &cobra.Command{ b, err := json.MarshalIndent(conf, "", "\t") if err != nil { log.Error().Err(err).Msg("failed to marshal config") + os.Exit(1) } fmt.Printf("%v\n", string(b)) } From 235efcdd07df843c619f9e969f64bb0283f853c2 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 17 Dec 2024 16:29:02 -0700 Subject: [PATCH 082/102] fix: added function to load server targets from config --- pkg/server/server.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 7ea585a..92b31f5 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -55,8 +55,7 @@ func New(conf *config.Config) *Server { c := config.New() conf = &c } - // return based on config values - return &Server{ + newServer := &Server{ Config: conf, Server: &http.Server{Addr: conf.Server.Host}, Jwks: Jwks{ @@ -64,6 +63,9 @@ func New(conf *config.Config) *Server { Retries: conf.Server.Jwks.Retries, }, } + // load templates for server from config + newServer.loadTargets() + return newServer } // Main function to start up configurator as a service. @@ -183,6 +185,21 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * } } +func (s *Server) loadTargets() { + for name, target := range s.Config.Targets { + serverTarget := Target{ + Name: name, + PluginPath: target.Plugin, + } + for _, templatePath := range target.TemplatePaths { + template := generator.Template{} + template.LoadFromFile(templatePath) + serverTarget.Templates = append(serverTarget.Templates, template) + } + s.Targets[name] = serverTarget + } +} + // Create a new target with name, generator, templates, and files. // // Example: From 2edd8367bc437cc83366372936d7ff4676833728 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 18 Dec 2024 11:33:48 -0700 Subject: [PATCH 083/102] fix: added check to loadTargets() to prevent panic --- pkg/server/server.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/server/server.go b/pkg/server/server.go index 92b31f5..d8e82d4 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -186,11 +186,17 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * } func (s *Server) loadTargets() { + // make sure the map is initialized first + if s.Targets == nil { + s.Targets = make(map[string]Target) + } + // add targets from config to server for name, target := range s.Config.Targets { serverTarget := Target{ Name: name, PluginPath: target.Plugin, } + // add templates using template paths from config for _, templatePath := range target.TemplatePaths { template := generator.Template{} template.LoadFromFile(templatePath) From 895b539c669c495266758870e5bdd7f6c4d7f363 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 18 Dec 2024 12:16:53 -0700 Subject: [PATCH 084/102] fix: added default generator server targets --- pkg/server/server.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/server/server.go b/pkg/server/server.go index d8e82d4..efe82c5 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -190,6 +190,14 @@ func (s *Server) loadTargets() { if s.Targets == nil { s.Targets = make(map[string]Target) } + // add default generator targets + for name, _ := range generator.DefaultGenerators { + serverTarget := Target{ + Name: name, + PluginPath: name, + } + s.Targets[name] = serverTarget + } // add targets from config to server for name, target := range s.Config.Targets { serverTarget := Target{ From 1f87b180b963c687f21805065be1a8048d49ba4a Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 18 Dec 2024 12:47:29 -0700 Subject: [PATCH 085/102] fix: changed variable name to avoid collision with package --- pkg/generator/generator.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 34333b0..b41e52d 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -165,23 +165,23 @@ func LoadPlugins(dirpath string, opts ...Option) (map[string]Generator, error) { // This function requires that a target and plugin path be set at minimum. func Generate(plugin string, params Params) (FileMap, error) { var ( - generator Generator - ok bool - err error + gen Generator + ok bool + err error ) // check if generator is built-in first before loading external plugin - generator, ok = DefaultGenerators[plugin] + gen, ok = DefaultGenerators[plugin] if !ok { // only load the plugin needed for this target if we don't find default log.Error().Msg("could not find target in default generators") - generator, err = LoadPlugin(plugin) + gen, err = LoadPlugin(plugin) if err != nil { return nil, fmt.Errorf("failed to load plugin from file: %v", err) } } - return generator.Generate(nil, params) + return gen.Generate(nil, params) } // Main function to generate a collection of files as a map with the path as the key and From 4572418dfa1038d37bdb6b008e5a9c7dd7e91529 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 18 Dec 2024 14:01:07 -0700 Subject: [PATCH 086/102] server: added error message when generate fails --- pkg/server/server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index efe82c5..a7f9f3f 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -156,14 +156,15 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * if target != nil { outputs, err = generator.Generate(target.PluginPath, s.GeneratorParams) if err != nil { - + log.Error().Err(err).Msg("failed to generate file") + return } } else { // try and generate a new config file from supplied params outputs, err = generator.GenerateWithTarget(s.Config, targetParam) if err != nil { writeErrorResponse(w, "failed to generate file") - log.Error().Err(err).Msg("failed to generate file") + log.Error().Err(err).Msgf("failed to generate file with target '%s'", target) return } } From 41aa0c7a42678ca1a6a23f4eb7cdfee9dddef97b Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 18 Dec 2024 16:22:46 -0700 Subject: [PATCH 087/102] plugin: updated coredhcp to match interface --- pkg/generator/coredhcp.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/generator/coredhcp.go b/pkg/generator/coredhcp.go index 817790c..0a256e9 100644 --- a/pkg/generator/coredhcp.go +++ b/pkg/generator/coredhcp.go @@ -1,12 +1,9 @@ -//go:build coredhcp || plugins -// +build coredhcp plugins - package generator import ( "fmt" - configurator "github.com/OpenCHAMI/configurator/pkg" + "github.com/OpenCHAMI/configurator/pkg/config" "github.com/OpenCHAMI/configurator/pkg/util" ) @@ -24,6 +21,6 @@ func (g *CoreDhcp) GetDescription() string { return fmt.Sprintf("Configurator generator plugin for '%s' to generate config files. (WIP)", g.GetName()) } -func (g *CoreDhcp) Generate(config *configurator.Config, opts ...util.Option) (FileMap, error) { +func (g *CoreDhcp) Generate(config *config.Config, params Params) (FileMap, error) { return nil, fmt.Errorf("plugin does not implement generation function") } From 93cf6bba128eafac2226c503651cff0d79e98257 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 18 Dec 2024 16:23:47 -0700 Subject: [PATCH 088/102] fix: changed loadTargets to only overwrite plugin path if set --- pkg/server/server.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index a7f9f3f..7da26d7 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -65,6 +65,7 @@ func New(conf *config.Config) *Server { } // load templates for server from config newServer.loadTargets() + log.Debug().Any("targets", newServer.Targets).Msg("new server targets") return newServer } @@ -154,6 +155,7 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * // try to generate with target supplied by client first if target != nil { + log.Debug().Any("target", target).Msg("target for Generate()") outputs, err = generator.Generate(target.PluginPath, s.GeneratorParams) if err != nil { log.Error().Err(err).Msg("failed to generate file") @@ -161,6 +163,7 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * } } else { // try and generate a new config file from supplied params + log.Debug().Str("target", targetParam).Msg("target for GenerateWithTarget()") outputs, err = generator.GenerateWithTarget(s.Config, targetParam) if err != nil { writeErrorResponse(w, "failed to generate file") @@ -199,11 +202,16 @@ func (s *Server) loadTargets() { } s.Targets[name] = serverTarget } - // add targets from config to server + // add targets from config to server (overwrites default targets) for name, target := range s.Config.Targets { serverTarget := Target{ - Name: name, - PluginPath: target.Plugin, + Name: name, + } + // only overwrite plugin path if it's set + if target.Plugin != "" { + serverTarget.PluginPath = target.Plugin + } else { + serverTarget.PluginPath = name } // add templates using template paths from config for _, templatePath := range target.TemplatePaths { @@ -270,8 +278,8 @@ func (s *Server) createTarget(w http.ResponseWriter, r *http.Request) { } -func (s *Server) getTarget(target string) *Target { - t, ok := s.Targets[target] +func (s *Server) getTarget(name string) *Target { + t, ok := s.Targets[name] if ok { return &t } From 6bf75e27cee445ebfd0c9037800bac5bf03ccf7e Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 18 Dec 2024 16:24:15 -0700 Subject: [PATCH 089/102] refactor: added coredhcp default generator --- pkg/generator/generator.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index b41e52d..dfc39d7 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -35,7 +35,7 @@ func createDefaultGenerators() map[string]Generator { var ( generatorMap = map[string]Generator{} generators = []Generator{ - &Conman{}, &DHCPd{}, &DNSMasq{}, &Warewulf{}, &Example{}, + &Conman{}, &DHCPd{}, &DNSMasq{}, &Warewulf{}, &Example{}, &CoreDhcp{}, } ) for _, g := range generators { @@ -171,10 +171,11 @@ func Generate(plugin string, params Params) (FileMap, error) { ) // check if generator is built-in first before loading external plugin + log.Debug().Any("generators", DefaultGenerators).Msg("available generators") gen, ok = DefaultGenerators[plugin] if !ok { // only load the plugin needed for this target if we don't find default - log.Error().Msg("could not find target in default generators") + log.Error().Str("plugin", plugin).Msg("could not find target in default generators") gen, err = LoadPlugin(plugin) if err != nil { return nil, fmt.Errorf("failed to load plugin from file: %v", err) From 4f836630b0de0faaa2c9fca06f8ed4a079bf969e Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 12:14:53 -0700 Subject: [PATCH 090/102] cmd: minor changes --- cmd/generate.go | 3 ++- cmd/root.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 44e4e5d..578c9f0 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -81,6 +81,7 @@ var generateCmd = &cobra.Command{ } // set the client options + // params.ClientOpts = append(params.ClientOpts, client.WithHost(remoteHost)) if conf.AccessToken != "" { params.ClientOpts = append(params.ClientOpts, client.WithAccessToken(conf.AccessToken)) } @@ -89,7 +90,7 @@ var generateCmd = &cobra.Command{ } // run generator.Generate() with just plugin path and templates provided - outputBytes, err := generator.Generate(pluginPath, params) + outputBytes, err := generator.Generate(&conf, pluginPath, params) if err != nil { log.Error().Err(err).Msg("failed to generate files") } diff --git a/cmd/root.go b/cmd/root.go index 1d1997c..583ee6c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,13 +40,13 @@ func Execute() { } func init() { - cobra.OnInitialize(initConfig) + cobra.OnInitialize(InitConfig) rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config path") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set to enable verbose output") rootCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") } -func initConfig() { +func InitConfig() { // empty from not being set if configPath != "" { exists, err := util.PathExists(configPath) From 88d365155f9582d8e85e47e071ed30ec962969b9 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 12:16:08 -0700 Subject: [PATCH 091/102] generator: fixed issue with templates not being generated --- pkg/generator/generator.go | 9 +++++++-- pkg/generator/templates.go | 3 +++ pkg/server/server.go | 12 +++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index dfc39d7..3397e36 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -163,7 +163,7 @@ func LoadPlugins(dirpath string, opts ...Option) (map[string]Generator, error) { // be used. This function will only load the plugin on-demand and fetch resources as needed. // // This function requires that a target and plugin path be set at minimum. -func Generate(plugin string, params Params) (FileMap, error) { +func Generate(config *config.Config, plugin string, params Params) (FileMap, error) { var ( gen Generator ok bool @@ -182,7 +182,7 @@ func Generate(plugin string, params Params) (FileMap, error) { } } - return gen.Generate(nil, params) + return gen.Generate(config, params) } // Main function to generate a collection of files as a map with the path as the key and @@ -232,6 +232,11 @@ func GenerateWithTarget(config *config.Config, target string) (FileMap, error) { } } + // check if there's at least one template available + if len(targetInfo.TemplatePaths) <= 0 { + return nil, fmt.Errorf("expects at least one template to be available") + } + // prepare params to pass into generator params.Templates = map[string]Template{} for _, templatePath := range targetInfo.TemplatePaths { diff --git a/pkg/generator/templates.go b/pkg/generator/templates.go index 6d4ae5d..321076c 100644 --- a/pkg/generator/templates.go +++ b/pkg/generator/templates.go @@ -8,6 +8,7 @@ import ( "github.com/OpenCHAMI/configurator/pkg/util" "github.com/nikolalohinski/gonja/v2" "github.com/nikolalohinski/gonja/v2/exec" + "github.com/rs/zerolog/log" ) type Template struct { @@ -65,6 +66,8 @@ func ApplyTemplates(mappings Mappings, templates map[string]Template) (FileMap, outputs[path] = b.Bytes() } + log.Debug().Any("templates", templates).Any("outputs", outputs).Any("mappings", mappings).Msg("apply templates") + return outputs, nil } diff --git a/pkg/server/server.go b/pkg/server/server.go index 7da26d7..a89e04a 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -93,6 +93,7 @@ func (s *Server) Serve() error { // create client with opts to use to fetch data from SMD opts := []client.Option{ + client.WithHost(s.Config.SmdClient.Host), client.WithAccessToken(s.Config.AccessToken), client.WithCertPoolFile(s.Config.CertPath), } @@ -146,7 +147,7 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * outputs generator.FileMap err error ) - s.GeneratorParams = parseGeneratorParams(r, opts...) + s.GeneratorParams = parseGeneratorParams(r, target, opts...) if targetParam == "" { err = writeErrorResponse(w, "must specify a target") log.Error().Err(err).Msg("failed to parse generator params") @@ -156,7 +157,8 @@ func (s *Server) Generate(opts ...client.Option) func(w http.ResponseWriter, r * // try to generate with target supplied by client first if target != nil { log.Debug().Any("target", target).Msg("target for Generate()") - outputs, err = generator.Generate(target.PluginPath, s.GeneratorParams) + outputs, err = generator.Generate(s.Config, target.PluginPath, s.GeneratorParams) + log.Debug().Any("outputs map", outputs).Msgf("after generate") if err != nil { log.Error().Err(err).Msg("failed to generate file") return @@ -299,9 +301,13 @@ func writeErrorResponse(w http.ResponseWriter, format string, a ...any) error { return fmt.Errorf(errmsg) } -func parseGeneratorParams(r *http.Request, opts ...client.Option) generator.Params { +func parseGeneratorParams(r *http.Request, target *Target, opts ...client.Option) generator.Params { var params = generator.Params{ ClientOpts: opts, + Templates: make(map[string]generator.Template, len(target.Templates)), + } + for i, template := range target.Templates { + params.Templates[fmt.Sprintf("%s_%d", target.Name, i)] = template } return params } From a651d70dfa5a31d8247030ccb11be78c633d0c97 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 12:38:27 -0700 Subject: [PATCH 092/102] generator: fixed issue where dnsmasq plugin did not output correctly --- pkg/generator/dnsmasq.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/generator/dnsmasq.go b/pkg/generator/dnsmasq.go index 83bf1d6..4ce5b8f 100644 --- a/pkg/generator/dnsmasq.go +++ b/pkg/generator/dnsmasq.go @@ -66,6 +66,6 @@ func (g *DNSMasq) Generate(config *config.Config, params Params) (FileMap, error "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), - "dhcp-hosts": output, + "output": output, }, params.Templates) } From 531ad8881d57d1025c0825893c134274527bfbf6 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 12:41:35 -0700 Subject: [PATCH 093/102] readme: added note about illegal template keys --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index d040037..4403f8c 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,10 @@ func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) }) } +> [!NOTE] +> The keys in `generator.ApplyTemplate` must not contain illegal characters such as a `-` or else the templates will not apply correctly. + + // this MUST be named "Generator" for symbol lookup in main driver var Generator MyGenerator ``` From 8915a28258739bcea4c387f57d5c53dde5bbccf7 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 12:50:52 -0700 Subject: [PATCH 094/102] changed 'output' to 'dhcp_hosts' in dhcp plugin and README --- README.md | 4 ++-- pkg/generator/dnsmasq.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4403f8c..02c7be8 100644 --- a/README.md +++ b/README.md @@ -142,14 +142,14 @@ func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) if client { eths, err := client.FetchEthernetInterfaces(opts...) // ... blah, blah, blah, check error, format output, and so on... - } + // apply the substitutions to Jinja template and return output as FileMap (i.e. path and it's contents) return generator.ApplyTemplate(path, generator.Mappings{ "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), - "output": output, + "dhcp_hosts": output, }) } diff --git a/pkg/generator/dnsmasq.go b/pkg/generator/dnsmasq.go index 4ce5b8f..8db68ba 100644 --- a/pkg/generator/dnsmasq.go +++ b/pkg/generator/dnsmasq.go @@ -66,6 +66,6 @@ func (g *DNSMasq) Generate(config *config.Config, params Params) (FileMap, error "plugin_name": g.GetName(), "plugin_version": g.GetVersion(), "plugin_description": g.GetDescription(), - "output": output, + "dhcp_hosts": output, }, params.Templates) } From 3a3e00ce1208645889f9c3763fa14d3b761a4f07 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 12:53:43 -0700 Subject: [PATCH 095/102] readme: fixed formatting issues --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 02c7be8..76521a5 100644 --- a/README.md +++ b/README.md @@ -144,13 +144,14 @@ func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) // ... blah, blah, blah, check error, format output, and so on... - // apply the substitutions to Jinja template and return output as FileMap (i.e. path and it's contents) - return generator.ApplyTemplate(path, generator.Mappings{ - "plugin_name": g.GetName(), - "plugin_version": g.GetVersion(), - "plugin_description": g.GetDescription(), - "dhcp_hosts": output, - }) + // apply the substitutions to Jinja template and return output as FileMap (i.e. path and it's contents) + return generator.ApplyTemplate(path, generator.Mappings{ + "plugin_name": g.GetName(), + "plugin_version": g.GetVersion(), + "plugin_description": g.GetDescription(), + "dhcp_hosts": output, + }) + } } > [!NOTE] From cd40fd194fe53aa9e7b0c83cbd88f78de5595764 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 12:54:45 -0700 Subject: [PATCH 096/102] examples: fixed illegal character in dnsmasq template --- examples/plugin/test.go | 19 +++++++++++++++++++ examples/templates/dnsmasq.jinja | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 examples/plugin/test.go diff --git a/examples/plugin/test.go b/examples/plugin/test.go new file mode 100644 index 0000000..f1b4e9b --- /dev/null +++ b/examples/plugin/test.go @@ -0,0 +1,19 @@ +package main + +import ( + "github.com/OpenCHAMI/configurator/pkg/config" + "github.com/OpenCHAMI/configurator/pkg/generator" +) + +type TestGenerator struct{} + +func (g *TestGenerator) GetName() string { return "test" } +func (g *TestGenerator) GetVersion() string { return "v1.0.0" } +func (g *TestGenerator) GetDescription() string { + return "This is a plugin creating for running tests." +} +func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) { + return generator.FileMap{"test": []byte("test")}, nil +} + +var Generator TestGenerator diff --git a/examples/templates/dnsmasq.jinja b/examples/templates/dnsmasq.jinja index c522039..d187a9a 100644 --- a/examples/templates/dnsmasq.jinja +++ b/examples/templates/dnsmasq.jinja @@ -7,4 +7,4 @@ # Source code: https://github.com/OpenCHAMI/configurator # Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins # -{{ dhcp-hosts }} +{{ dhcp_hosts }} From 733c486cb228c4474c0ff91be2dc7cf5abbe2f1c Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 12:57:29 -0700 Subject: [PATCH 097/102] examples: updated test plugin --- examples/plugin/test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/plugin/test.go b/examples/plugin/test.go index f1b4e9b..58aa310 100644 --- a/examples/plugin/test.go +++ b/examples/plugin/test.go @@ -13,7 +13,11 @@ func (g *TestGenerator) GetDescription() string { return "This is a plugin creating for running tests." } func (g *TestGenerator) Generate(config *config.Config, params generator.Params) (generator.FileMap, error) { - return generator.FileMap{"test": []byte("test")}, nil + return generator.ApplyTemplates(generator.Mappings{ + "plugin_name": g.GetName(), + "plugin_version": g.GetVersion(), + "plugin_description": g.GetDescription(), + }, params.Templates) } var Generator TestGenerator From 3fa7b808022459950e33eae8274d2a794992a614 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 13:11:25 -0700 Subject: [PATCH 098/102] readme: fixed formatting issue and added tip --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 76521a5..ca94345 100644 --- a/README.md +++ b/README.md @@ -154,14 +154,13 @@ func (g *MyGenerator) Generate(config *configurator.Config, opts ...util.Option) } } -> [!NOTE] -> The keys in `generator.ApplyTemplate` must not contain illegal characters such as a `-` or else the templates will not apply correctly. - - // this MUST be named "Generator" for symbol lookup in main driver var Generator MyGenerator ``` +> [!NOTE] +> The keys in `generator.ApplyTemplate` must not contain illegal characters such as a `-` or else the templates will not apply correctly. + Finally, build the plugin and put it somewhere specified by `plugins` in your config. Make sure that the package is `main` before building. ```bash @@ -170,6 +169,9 @@ go build -buildmode=plugin -o lib/mygenerator.so path/to/mygenerator.go Now your plugin should be available to use with the `configurator` main driver program. If you get an error about not loading the correct symbol type, make sure that your generator function definitions match the `Generator` interface entirely and that you don't have a partially implemented interface. +> [!TIP] +> See the `examples/test.go` file for a plugin and template example. + ## Configuration Here is an example config file to start using configurator: From 5b08da019ceb8ff2af0cdbc0b0374d6ec0ae5648 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 13:11:48 -0700 Subject: [PATCH 099/102] examples: added test template --- examples/templates/test.j2 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 examples/templates/test.j2 diff --git a/examples/templates/test.j2 b/examples/templates/test.j2 new file mode 100644 index 0000000..df2bb4e --- /dev/null +++ b/examples/templates/test.j2 @@ -0,0 +1,16 @@ +# +# This file was auto-generated by the OpenCHAMI "configurator" tool using the following plugin: +# Name: {{ plugin_name }} +# Version: {{ plugin_version }} +# Description: {{ plugin_description }} +# +# Source code: https://github.com/OpenCHAMI/configurator +# Creating plugins: https://github.com/OpenCHAMI/configurator/blob/main/README.md#creating-generator-plugins +# + +# TODO: test variables + +# TODO: test if/else statements + +# TODO: test for loops + From b0b52dc4327b7e8978d0fe84ac5b58488a957180 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Thu, 21 Nov 2024 14:27:22 -0700 Subject: [PATCH 100/102] feat: added healthcheck to server --- pkg/server/server.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/server/server.go b/pkg/server/server.go index a89e04a..915dc00 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -116,6 +116,7 @@ func (s *Server) Serve() error { // protected routes if using auth r.HandleFunc("/generate", s.Generate(opts...)) + r.HandleFunc("/status", s.GetStatus) r.Post("/targets", s.createTarget) }) } else { @@ -225,6 +226,19 @@ func (s *Server) loadTargets() { } } +func (s *Server) GetStatus(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + data := map[string]any{ + "code": 200, + "message": "Configurator is healthy", + } + err := json.NewEncoder(w).Encode(data) + if err != nil { + fmt.Printf("failed to encode JSON: %v\n", err) + return + } +} + // Create a new target with name, generator, templates, and files. // // Example: From ba3690cb5d35a93be96c5e5fac6f2fa2735f29a9 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 3 Dec 2024 11:55:53 -0700 Subject: [PATCH 101/102] refactor: moved /status endpoint to always be public route --- pkg/server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 915dc00..bb1f736 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -116,7 +116,6 @@ func (s *Server) Serve() error { // protected routes if using auth r.HandleFunc("/generate", s.Generate(opts...)) - r.HandleFunc("/status", s.GetStatus) r.Post("/targets", s.createTarget) }) } else { @@ -126,6 +125,7 @@ func (s *Server) Serve() error { } // always available public routes go here (none at the moment) + router.HandleFunc("/status", s.GetStatus) s.Handler = router return s.ListenAndServe() From 6b0ed4c03d86577526414e04782d5918005571af Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Tue, 7 Jan 2025 13:27:17 -0700 Subject: [PATCH 102/102] server: changed endpoint from /status to /configurator/status --- pkg/server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index bb1f736..d10ccd7 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -125,7 +125,7 @@ func (s *Server) Serve() error { } // always available public routes go here (none at the moment) - router.HandleFunc("/status", s.GetStatus) + router.HandleFunc("/configurator/status", s.GetStatus) s.Handler = router return s.ListenAndServe()