More refactoring and bug fixes

- Added `libhttp` and `librest_api` targets to CMakeLists.txt
- Added examples for `libhttp` and `librest_api` in examples directory
- Added `cache` and `add` commands
- Added documentation for `libhttp` and `librest_api`
- Added all available HTTP response codes to REST API
- Changed how `bin/compile.sh` works (must supply args to build)
This commit is contained in:
David Allen 2023-05-27 11:28:58 -06:00
parent 5a73651ad1
commit 8b1f1374d8
23 changed files with 575 additions and 164 deletions

View file

@ -130,12 +130,12 @@ namespace gdpm::config{
/* Must check if keys exists first, then populate `_config_params`. */
if(doc.HasMember("remote_sources")){
if(doc["remote_sources"].IsArray()){
if(doc["remote_sources"].IsObject()){
const Value& srcs = doc["remote_sources"];
for(auto& src : srcs.GetObject()){
// config.remote_sources.push_back(src.GetString());
config.remote_sources.insert(
std::pair(src.name.GetString(), src.value.GetString())
string_pair(src.name.GetString(), src.value.GetString())
);
}
} else {

View file

@ -20,7 +20,7 @@ namespace gdpm::http{
response request_get(
const string& url,
const http::params& params
const http::request_params& params
){
CURL *curl = nullptr;
CURLcode res;
@ -58,8 +58,7 @@ namespace gdpm::http{
response request_post(
const string& url,
const char *post_fields,
const http::params& params
const http::request_params& params
){
CURL *curl = nullptr;
CURLcode res;
@ -70,13 +69,25 @@ namespace gdpm::http{
using namespace std::chrono_literals;
utils::delay();
#endif
string h;
std::for_each(
params.headers.begin(),
params.headers.end(),
[&h](const string_pair& kv){
h += kv.first + "=" + kv.second + "&";
}
);
h.pop_back();
h = url_escape(h);
// const char *post_fields = "";
curl_global_init(CURL_GLOBAL_ALL);
curl = curl_easy_init();
if(curl){
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
// curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_fields);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, h.size());
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, h.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl_write_to_buffer);
curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
@ -98,7 +109,7 @@ namespace gdpm::http{
response download_file(
const string& url,
const string& storage_path,
const http::params& params
const http::request_params& params
){
CURL *curl = nullptr;
CURLcode res;

View file

@ -9,9 +9,11 @@
int main(int argc, char **argv){
using namespace gdpm;
result_t <package_manager::exec_args> r_input = package_manager::initialize(argc, argv);
package_manager::exec_args input = r_input.unwrap_unsafe();
package_manager::execute(input.args, input.opts);
package_manager::finalize();
using namespace gdpm::package_manager;
result_t <exec_args> r_input = initialize(argc, argv);
exec_args input = r_input.unwrap_unsafe();
execute(input);
finalize();
return 0;
}

View file

@ -1,11 +1,13 @@
#include "package.hpp"
#include "error.hpp"
#include "log.hpp"
#include "rest_api.hpp"
#include "config.hpp"
#include "cache.hpp"
#include "http.hpp"
#include "remote.hpp"
#include "types.hpp"
#include <future>
#include <rapidjson/ostreamwrapper.h>
#include <rapidjson/prettywriter.h>
@ -21,9 +23,16 @@ namespace gdpm::package{
/* TODO: Need a way to use remote sources from config until none left */
/* Check if the package data is already stored in cache. If it is, there
is no need to do a lookup to synchronize the local database since we
have all the information we need to fetch the asset data. */
/*
Implementation steps:
1. Synchronize the cache information by making a request using remote API.
(can skip with `--no-sync` options)
2. Check if the package is installed. If it is, make sure it is latest version.
If not, download and update to the latest version.
3. Extract package contents and copy/move to the correct install location.
*/
result_t result = cache::get_package_info_by_title(package_titles);
package::info_list p_found = {};
package::info_list p_cache = result.unwrap_unsafe();
@ -37,6 +46,8 @@ namespace gdpm::package{
}
}
/* Match queried package titles with those found in cache. */
log::debug("Searching for packages in cache...");
for(const auto& p_title : package_titles){
auto found = std::find_if(
p_cache.begin(),
@ -50,16 +61,26 @@ namespace gdpm::package{
}
}
/* If size of package_titles == p_found, then all packages can be installed
from cache and there's no need to query remote API. However, this will
only installed the latest *local* version and will not sync with remote. */
if(p_found.size() == package_titles.size()){
log::info("Found all packages stored in local cache.");
}
/* Found nothing to install so there's nothing to do at this point. */
if(p_found.empty()){
constexpr const char *message = "No packages found to install.";
log::error(message);
return error(constants::error::NOT_FOUND, message);
error error(
constants::error::NOT_FOUND,
"No packages found to install."
);
log::error(error);
return error;
}
log::println("Packages to install: ");
for(const auto& p : p_found){
std::string output((p.is_installed) ? p.title + " (reinstall)" : p.title);
string output((p.is_installed) ? p.title + " (reinstall)" : p.title);
log::print(" {} ", (p.is_installed) ? p.title + " (reinstall)" : p.title);
}
log::println("");
@ -70,9 +91,8 @@ namespace gdpm::package{
}
/* Try and obtain all requested packages. */
using ss_pair = std::pair<std::string, std::string>;
std::vector<ss_pair> dir_pairs;
std::vector<std::future<error>> tasks;
std::vector<string_pair> dir_pairs;
task_list tasks;
rest_api::context rest_api_params = rest_api::make_from_config(config);
for(auto& p : p_found){ // TODO: Execute each in parallel using coroutines??
@ -80,8 +100,8 @@ namespace gdpm::package{
in global storage location only. */
log::info("Fetching asset data for \"{}\"...", p.title);
std::string url{config.remote_sources.at(params.remote_source) + rest_api::endpoints::GET_AssetId};
std::string package_dir, tmp_dir, tmp_zip;
string url{config.remote_sources.at(params.remote_source) + rest_api::endpoints::GET_AssetId};
string package_dir, tmp_dir, tmp_zip;
/* Retrieve necessary asset data if it was found already in cache */
Document doc;
@ -161,7 +181,7 @@ namespace gdpm::package{
}
}
dir_pairs.emplace_back(ss_pair(tmp_zip, package_dir + "/"));
dir_pairs.emplace_back(string_pair(tmp_zip, package_dir + "/"));
p.is_installed = true;
p.install_path = package_dir;
@ -182,6 +202,16 @@ namespace gdpm::package{
}
error add(
const config::context& config,
const title_list& package_titles,
const params& params
){
return error();
}
error remove(
const config::context& config,
const string_list& package_titles,
@ -346,11 +376,11 @@ namespace gdpm::package{
error search(
const config::context& config,
const package::title_list &package_titles,
const package::title_list &package_titles,
const package::params& params
){
result_t r_cache = cache::get_package_info_by_title(package_titles);
std::vector<package::info> p_cache = r_cache.unwrap_unsafe();
info_list p_cache = r_cache.unwrap_unsafe();
if(!p_cache.empty() && !config.enable_sync){
print_list(p_cache);
@ -393,22 +423,24 @@ namespace gdpm::package{
using namespace rapidjson;
using namespace std::filesystem;
if(opts.empty() || opts.contains("packages")){
string show((!args.empty()) ? args[0] : "");
if(show.empty() || show == "packages"){
result_t r_installed = cache::get_installed_packages();
info_list p_installed = r_installed.unwrap_unsafe();
if(!p_installed.empty()){
log::println("Installed packages: ");
print_list(p_installed);
}
else{
log::println("empty");
}
}
else if(opts.contains("remote")){
else if(show == "remote"){
remote::print_repositories(config);
}
else{
error error(
constants::error::UNKNOWN_COMMAND,
"Unrecognized subcommand. Try either 'packages' or 'remote' instead."
);
log::error(error);
}
@ -654,6 +686,28 @@ namespace gdpm::package{
}
template <typename T>
auto set_if_key_exists(
const var_opts& o,
const string& k,
T& p
){
if(o.count(k)){ p = std::get<T>(o.at(k)); }
}
params make_params(const var_args& args, const var_opts& opts){
params p;
set_if_key_exists<int>(opts, "jobs", p.parallel_jobs);
set_if_key_exists(opts, "cache", p.enable_cache);
set_if_key_exists(opts, "sync", p.enable_sync);
set_if_key_exists(opts, "skip-prompt", p.skip_prompt);
set_if_key_exists(opts, "remote-source", p.remote_source);
// set_if_key_exists(opts, "install-method", p.install_method);
return p;
}
result_t<info_list> synchronize_database(
const config::context& config,
const title_list& package_titles

View file

@ -35,7 +35,7 @@
*/
namespace gdpm::package_manager{
remote::repository_map repo_sources;
remote::repository_map remote_sources;
CURL *curl;
CURLcode res;
config::context config;
@ -76,8 +76,8 @@ namespace gdpm::package_manager{
}
int execute(const args_t& args, const opts_t& opts){
run_command(action, packages, opts);
int execute(const exec_args& in){
run_command(action, in.args, in.opts);
if(clean_tmp_dir)
package::clean_temporary(config, packages);
return 0;
@ -102,7 +102,8 @@ namespace gdpm::package_manager{
options.allow_unrecognised_options();
options.custom_help("[COMMAND] [OPTIONS...]");
options.add_options("Command")
("command", "Specify the input parameters", cxxopts::value<string_list>())
("command", "Specify the input parameters", cxxopts::value<string>())
("positional", "", cxxopts::value<string_list>())
("install", "Install package or packages.", cxxopts::value<string_list>()->implicit_value(""), "<packages...>")
("remove", "Remove a package or packages.", cxxopts::value<string_list>()->implicit_value(""), "<packages...>")
("update", "Update a package or packages. This will update all packages if no argument is provided.", cxxopts::value<string_list>()->implicit_value(""), "<packages...>")
@ -111,13 +112,14 @@ namespace gdpm::package_manager{
("list", "Show list of installed packages.")
("link", "Create a symlink (or shortcut) to target directory. Must be used with the `--path` argument.", cxxopts::value<string_list>(), "<packages...>")
("clone", "Clone packages into target directory. Must be used with the `--path` argument.", cxxopts::value<string_list>(), "<packages...>")
("cache", "Caching operations", cxxopts::value<string_list>())
("clean", "Clean temporary downloaded files.")
("fetch", "Fetch asset data from remote sources.")
("remote", "Set a source repository.", cxxopts::value<string>()->default_value(constants::AssetRepo), "<url>")
("h,help", "Print this message and exit.")
("version", "Show the current version and exit.")
;
options.parse_positional({"command"});
options.parse_positional({"command", "positional"});
options.positional_help("");
options.add_options("Other")
("c,config", "Set the config file path.", cxxopts::value<string>())
@ -128,9 +130,8 @@ namespace gdpm::package_manager{
("support", "Set the support level for API (all|official|community|testing).")
("max-results", "Set the max results to return from search.", cxxopts::value<int>()->default_value("500"), "<int>")
("godot-version", "Set the Godot version to include in request.", cxxopts::value<string>())
("set-priority", "Set the priority for remote source. Lower values are used first (0...100).", cxxopts::value<int>())
("set-packages-directory", "Set the local package storage location.", cxxopts::value<string>())
("set-temporary-directory", "Set the local temporary storage location.", cxxopts::value<string>())
("package-dir", "Set the local package storage location.", cxxopts::value<string>())
("tmp-dir", "Set the local temporary storage location.", cxxopts::value<string>())
("timeout", "Set the amount of time to wait for a response.", cxxopts::value<size_t>())
("no-sync", "Disable synchronizing with remote.", cxxopts::value<bool>()->implicit_value("true")->default_value("false"))
("y,no-prompt", "Bypass yes/no prompt for installing or removing packages.")
@ -142,17 +143,38 @@ namespace gdpm::package_manager{
}
template <typename T>
void insert(
var_opts& opts,
const cxxopts::ParseResult& result,
const string& key
){
opts.insert(var_opt(key, result[key].as<T>()));
};
result_t<exec_args> _handle_arguments(const cxxargs& args){
const auto& result = args.result;
const auto& options = args.options;
exec_args in;
// auto get_opt = [](const string& key){
// result[key].as<T>();
// }
// auto get_opt = [&result]<typename V>(const string& key){
// return opt(key, result[key].as<V>());
// };
/* Set option variables first to be used in functions below. */
if(result.count("search")){
in.args = result["search"].as<var_args>();
}
if(result.count("help")){
log::println("{}", options.help());
}
if(result.count("config")){
config.path = result["config"].as<string>();
insert<string>(in.opts, result, "config");
config::load(config.path, config);
log::info("Config: {}", config.path);
}
@ -163,9 +185,8 @@ namespace gdpm::package_manager{
string argv = result.arguments_string();
log::println("argv: {}", argv);
remote::add_repositories(config, {});
}
else if(sub_command == "delete"){
else if(sub_command == "remove"){
remote::remove_respositories(config, {});
log::println("argv: {}");
}
@ -174,9 +195,10 @@ namespace gdpm::package_manager{
string path = result["file"].as<string>();
string contents = utils::readfile(path);
packages = utils::parse_lines(contents);
insert<string>(in.opts, result, "file");
}
if(result.count("path")){
in.opts.insert({"path", result["path"].as<string_list>()});
insert<string_list>(in.opts, result, "path");
}
if(result.count("sort")){
string r = result["sort"].as<string>();
@ -187,14 +209,16 @@ namespace gdpm::package_manager{
else if(r == "name") sort = rest_api::sort_e::name;
else if(r == "updated") sort = rest_api::sort_e::updated;
api_params.sort = sort;
in.opts.insert(var_opt("sort", r));
}
if(result.count("type")){
string r = result["type"].as<std::string>();
string r = result["type"].as<string>();
rest_api::type_e type = rest_api::type_e::any;
if(r == "any") type = rest_api::type_e::any;
else if(r == "addon") type = rest_api::type_e::addon;
else if(r == "project") type = rest_api::type_e::project;
api_params.type = type;
in.opts.insert(var_opt("type", r));
}
if(result.count("support")){
string r = result["support"].as<string>();
@ -204,42 +228,52 @@ namespace gdpm::package_manager{
else if(r == "community") support = rest_api::support_e::community;
else if(r == "testing") support = rest_api::support_e::testing;
api_params.support = support;
in.opts.insert(var_opt("support", r));
}
if(result.count("max-results")){
api_params.max_results = result["max-results"].as<int>();
insert<int>(in.opts, result, "max-results");
}
if(result.count("godot-version")){
config.godot_version = result["godot-version"].as<string>();
insert<string>(in.opts, result, "godot-version");
}
if(result.count("timeout")){
config.timeout = result["timeout"].as<size_t>();
insert<size_t>(in.opts, result, "timeout");
}
if(result.count("no-sync")){
config.enable_sync = false;
in.opts.insert(var_opt("sync", "disabled"));
}
if(result.count("set-priority")){
priority = result["set-priority"].as<int>();
if(result.count("package-dir")){
config.packages_dir = result["package-dir"].as<string>();
insert<string>(in.opts, result, "package-dir");
}
if(result.count("set-packages-directory")){
config.packages_dir = result["set-packages-directory"].as<string>();
}
if(result.count("set-temporary-directory")){
config.tmp_dir = result["set-temporary-directory"].as<string>();
if(result.count("tmp-dir")){
config.tmp_dir = result["tmp-dir"].as<string>();
insert<string>(in.opts, result, "tmp-dir");
}
if(result.count("yes")){
skip_prompt = true;
in.opts.insert(opt("skip-prompt", true));
}
if(result.count("link")){
packages = result["link"].as<string_list>();
insert<string_list>(in.opts, result, "link");
}
if(result.count("clone")){
packages = result["clone"].as<string_list>();
insert<string_list>(in.opts, result, "clone");
}
if(result.count("clean")){
in.opts.insert(opt("clean", true));
clean_tmp_dir = true;
}
config.verbose = 0;
config.verbose += result["verbose"].as<int>();
insert<int>(in.opts, result, "verbose");
string json = to_json(config);
if(config.verbose > 0){
log::println("Verbose set to level {}", config.verbose);
@ -251,17 +285,20 @@ namespace gdpm::package_manager{
return result_t(in, error());
}
args_t _argv = result["command"].as<args_t>();
string sub_command = _argv[0];
args_t argv{_argv.begin()+1, _argv.end()};
if(packages.empty() && in.opts.size() > 0){
for(const auto& arg : argv){
packages.emplace_back(arg);
string sub_command = result["command"].as<string>();
if(result.count("positional")){
string_list _argv = result["positional"].as<string_list>();
args_t argv{_argv.begin(), _argv.end()};
if(!argv.empty()){
for(const auto& arg : argv){
in.args.emplace_back(arg);
}
}
}
/* Catch arguments passed with dashes */
if(sub_command == "install") action = action_e::install;
else if(sub_command == "add") action = action_e::add;
else if(sub_command == "remove") action = action_e::remove;
else if(sub_command == "update") action = action_e::update;
else if(sub_command == "search") action = action_e::search;
@ -272,7 +309,7 @@ namespace gdpm::package_manager{
else if(sub_command == "clean") action = action_e::clean;
else if(sub_command == "sync") action = action_e::sync;
else if(sub_command == "remote") action = action_e::remote;
else if(sub_command == "help" || argv[0] == "-h" || argv[0] == "--help" || argv[0].empty()){
else if(sub_command == "help"){ action = action_e::help;
log::println("{}", options.help());
}
else{
@ -283,22 +320,24 @@ namespace gdpm::package_manager{
/* Used to run the command AFTER parsing and setting all command line args. */
void run_command(action_e c, const args_t& args, const opts_t& opts){
package::params params;
void run_command(action_e c, const var_args& args, const var_opts& opts){
package::params params = package::make_params(args, opts);
string_list args_list = unwrap(args);
opts_t opts_list = unwrap(opts);
params.skip_prompt = skip_prompt;
switch(c){
case action_e::install: package::install(config, args, params); break;
case action_e::remove: package::remove(config, args, params); break;
case action_e::update: package::update(config, args, params); break;
case action_e::search: package::search(config, args, params); break;
case action_e::p_export: package::export_to(args); break;
case action_e::list: package::list(config, args, opts); break;
/* ...opts are the paths here */
case action_e::link: package::link(config, args, opts); break;
case action_e::clone: package::clone(config, args, opts); break;
case action_e::clean: package::clean_temporary(config, args); break;
case action_e::sync: package::synchronize_database(config, args); break;
case action_e::remote: remote::_handle_remote(config, args, opts); break;
case action_e::install: package::install(config, args_list, params); break;
case action_e::remove: package::remove(config, args_list, params); break;
case action_e::update: package::update(config, args_list, params); break;
case action_e::search: package::search(config, args_list, params); break;
case action_e::p_export: package::export_to(args_list); break;
case action_e::list: package::list(config, args_list, opts_list); break;
/* ...opts are the paths here */
case action_e::link: package::link(config, args_list, opts_list); break;
case action_e::clone: package::clone(config, args_list, opts_list); break;
case action_e::clean: package::clean_temporary(config, args_list); break;
case action_e::sync: package::synchronize_database(config, args_list); break;
case action_e::remote: remote::_handle_remote(config, args_list, opts_list); break;
case action_e::help: /* ...runs in handle_arguments() */ break;
case action_e::none: /* ...here to run with no command */ break;
}

View file

@ -77,7 +77,7 @@ namespace gdpm::rest_api{
}
string to_string(type_e type){
std::string _s{"type="};
string _s{"type="};
switch(type){
case any: _s += "any"; break;
case addon: _s += "addon"; break;