diff --git a/CMakeLists.txt b/CMakeLists.txt index d5a7d72..d784e7c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ find_package(SQLiteCpp CONFIG REQUIRED) set(CMAKE_CXX_COMPILER "clang++") set(CMAKE_BUILD_RPATH "build/cmake") set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -std=c++20 -Ofast -fPIC -fPIE -fpermissive -Wall -Wno-switch -Wno-unused-variable -Wno-unused-function -Wno-sign-conversion -Wno-deprecated-declarations -pedantic-errors" + "${CMAKE_CXX_FLAGS} -std=c++20 -g -Ofast -fPIC -fPIE -fpermissive -Wall -Wno-switch -Wno-unused-variable -Wno-unused-function -Wno-sign-conversion -Wno-deprecated-declarations -pedantic-errors" ) set(INCLUDE_DIRS "include" diff --git a/README.md b/README.md index 49fcbab..93682b9 100644 --- a/README.md +++ b/README.md @@ -324,11 +324,11 @@ gdpm config set username towk ## Planned Features -- [x] Godot 4 Asset Library compatibility. +- [x] Compatible with Godot 4 -- [ ] PKGBUILD for ArchLinux/Manjaro +- [x] Parallel downloading. -- [ ] Multithreaded downloading. +- [ ] PKGBUILD for ArchLinux/Manjaro. - [ ] Proper updating. @@ -340,6 +340,8 @@ gdpm config set username towk - [ ] Adapted to new Asset API. +- [ ] User interface. + - [ ] Experimental dependency management. There is no way of handling dependencies using the Godot Asset API. This is a [hot topic](https://github.com/godotengine/godot-proposals/issues/142) and might change in the near future. ## Known Issues diff --git a/examples/http/README.md b/examples/http/README.md index 7f96320..5dfa1ce 100644 --- a/examples/http/README.md +++ b/examples/http/README.md @@ -1,15 +1,71 @@ -# GDPM HTTP Example +# GDPM Rest API Example -This is an example showing how to use the GDPM HTTP library to download files. The library uses RapidJSON to get results. +This is an example showing how to use the REST API designed to query the Godot Asset library in C++. It is built using the `libcurl` library. +Here is a snippet making a HTTP get and post request. -Here is a quick snippet to get started: ```c++ -// Get a full list of assets -rest_api::context context = rest_api::make_context(); -rapidjson::Document doc = http::get_asset_list( - "https://godotengine.org/asset-library/api", - context -) -// ... parse the rapidjson::Document +#include "http.hpp" + +#include +#include + +using string = std::string; +using headers_t = std::unordered_map; + +string url = "www.example.com"; +http::context http; +http::request params = {{"user-agent", "firefox"}}; +http::response r = http::request(url, params); +if(r.code == http::OK){ /* ...do something... */ } + +r = http::request(url, params, http::method::POST); +if(r.code == http::OK){ /* ...do something.. */ } +``` + +Here's an example using the `multi` interface for parallel requests and downloads: + +```c++ +#include "http.hpp" + +#include + +template +using ptr = std::unique_ptr; +using strings = std::vector; +using transfers = std::vector; +using responses = std::vector; + +http::multi http; +http::request params = {{"user-agent", "firefox"}} +params.headers.insert(http::header("Accept", "*/*")); +params.headers.insert(http::header("Accept-Encoding", "application/gzip")); +params.headers.insert(http::header("Content-Encoding", "application/gzip")); +params.headers.insert(http::header("Connection", "keep-alive")); + +strings request_urls = { + "www.example.com", + "www.google.com" +} +ptr requests = http::make_requests(request_url, params); +ptr responses = http::execute(std::move(requests)); +for(const auto& r : *responses){ + if(r.code == http::OK){ + // ...do something + } +} + +strings download_urls = { + "" +} +strings storage_paths = { + "./download1" +} +ptr downloads = http::make_downloads(download_url, storage_paths, params); +ptr responses = http::execute(std::move(downloads)); +for(const auto& r : *responses){ + if(r.code == http::OK){ + // ...do something + } +} ``` \ No newline at end of file diff --git a/examples/rest_api/README.md b/examples/rest_api/README.md index d3911d5..7f96320 100644 --- a/examples/rest_api/README.md +++ b/examples/rest_api/README.md @@ -1,26 +1,15 @@ -# GDPM Rest API Example +# GDPM HTTP Example -This is an example showing how to use the REST API designed to query the Godot Asset library in C++. It is built using the `libcurl` library. +This is an example showing how to use the GDPM HTTP library to download files. The library uses RapidJSON to get results. -Here is a snippet making a HTTP get and post request. +Here is a quick snippet to get started: ```c++ -using string = std::string; -using headers_t = std::unordered_map; - -std::string url = "www.example.com"; -http::response r_get = http::request_get(url) -if(r_get.response_code == http::response_code::OK){ - // ...do something... -} - -http::request_params params; -params.headers = { - {"user-agent", "firefox"}, - {"content-type", "application/json"} -} -http::response r_post = http::request_post(url, params); -if(r_post.response_code == http::response_code::OK){ - // ...do something... -} +// Get a full list of assets +rest_api::context context = rest_api::make_context(); +rapidjson::Document doc = http::get_asset_list( + "https://godotengine.org/asset-library/api", + context +) +// ... parse the rapidjson::Document ``` \ No newline at end of file diff --git a/include/concepts.hpp b/include/concepts.hpp deleted file mode 100644 index 86f7e91..0000000 --- a/include/concepts.hpp +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include - -namespace gdpm::concepts{ - template concept RequireMinArgs = requires (std::size_t min){ sizeof...(Args) > min; }; -} \ No newline at end of file diff --git a/include/constants.hpp b/include/constants.hpp index a1f7dec..9e178fa 100644 --- a/include/constants.hpp +++ b/include/constants.hpp @@ -36,8 +36,8 @@ namespace gdpm::print{ #define GDPM_CONFIG_REMOTE_SOURCES std::pair(constants::RemoteName, constants::HostUrl) #define GDPM_CONFIG_THREADS 1 #define GDPM_CONFIG_TIMEOUT_MS 30000 -#define GDPM_CONFIG_ENABLE_SYNC 1 -#define GDPM_CONFIG_ENABLE_FILE_LOGGING 0 +#define GDPM_CONFIG_ENABLE_SYNC true +#define GDPM_CONFIG_ENABLE_FILE_LOGGING true #define GDPM_CONFIG_VERBOSE 0 /* Defines the default package cache for local storage */ diff --git a/include/error.hpp b/include/error.hpp index fa50c58..ed75e56 100644 --- a/include/error.hpp +++ b/include/error.hpp @@ -2,6 +2,7 @@ #include "log.hpp" #include "types.hpp" +#include "utils.hpp" #include #include #include @@ -15,11 +16,14 @@ namespace gdpm::constants::error{ UNKNOWN_COMMAND, UNKNOWN_ARGUMENT, ARGPARSE_ERROR, + ASSERTION_FAILED, + PRECONDITION_FAILED, + POSTCONDITION_FAILED, NOT_FOUND, NOT_DEFINED, NOT_IMPLEMENTED, NO_PACKAGE_FOUND, - PATH_NOT_DEFINED, + MALFORMED_PATH, FILE_EXISTS, FILE_NOT_FOUND, DIRECTORY_EXISTS, @@ -43,12 +47,17 @@ namespace gdpm::constants::error{ "", "An unknown error has occurred.", "Unknown command.", + "Unknown argument.", + "Could not parse argument.", + "Assertion condition failed.", + "Pre-condition failed.", + "Post-condition failed.", "Resource not found.", - "Function not defined.", - "Function not implemented.", + "Resource not defined.", + "Resource not implemented.", "No package found.", - "Path is not well-defined", - "File found.", + "Path is malformed.", + "File already exists", "File does not exist.", "Directory exists.", "Directory not found.", @@ -58,6 +67,10 @@ namespace gdpm::constants::error{ "Invalid configuration.", "Invalid key.", "An HTTP response error has occurred.", + "A SQLite error has occurred.", + "A libzip error has occurred.", + "A libcurl error has occurred.", + "A JSON error has occurred.", "An error has occurred." }; @@ -73,10 +86,12 @@ namespace gdpm::constants::error{ }; namespace gdpm{ + namespace ec = constants::error; class error { public: - constexpr explicit error(int code = 0, const string& message = "{code}"): - m_code(code), m_message(message == "{code}" ? constants::error::get_message(code): message) + constexpr explicit error(int code = 0, const string& message = "{default}"): + m_code(code), + m_message(utils::replace_all(message, "{default}", ec::get_message(code))) {} void set_code(int code) { m_code = code; } @@ -106,13 +121,21 @@ namespace gdpm{ #endif } + // static constexpr void error(int code, const string& message = "{default}"){ + // log::error(gdpm::error(code, message)); + // } + static constexpr gdpm::error error_rc(const gdpm::error& e){ - error(e); + log::error(e); return e; } - static void error(const char *p, const gdpm::error& e){ - println("{}{}{}", p, prefix.contents, e.get_message()); + static constexpr gdpm::error error_rc(int code, const string& message = "{default}"){ + return error_rc(gdpm::error(code, message)); } } + + namespace concepts{ + template concept error_t = requires{ std::is_same::value; }; + } } diff --git a/include/http.hpp b/include/http.hpp index 04b2c94..6ce7df4 100644 --- a/include/http.hpp +++ b/include/http.hpp @@ -2,6 +2,12 @@ #include "constants.hpp" #include "types.hpp" +#include "indicators/indeterminate_progress_bar.hpp" +#include "indicators/dynamic_progress.hpp" +#include "indicators/progress_bar.hpp" +#include "indicators/block_progress_bar.hpp" +#include "utils.hpp" +#include #include #include #include @@ -10,6 +16,13 @@ namespace gdpm::http{ using headers_t = std::unordered_map; using header = std::pair; + enum method{ + GET, + POST, + PUT, + DELETE + }; + // REF: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status enum response_code{ CONTINUE = 100, @@ -77,6 +90,11 @@ namespace gdpm::http{ NETWORK_AUTHENTICATION_REQUIRED = 511 }; + enum transfer_type{ + REQUEST, + DOWNLOAD + }; + struct response{ long code = 0; string body{}; @@ -85,31 +103,91 @@ namespace gdpm::http{ }; - struct request_params { + struct request { headers_t headers = {}; + method method = method::GET; size_t timeout = GDPM_CONFIG_TIMEOUT_MS; int verbose = 0; }; + using namespace indicators; + // BlockProgressBar bar { + // option::BarWidth{50}, + // // option::Start{"["}, + // // option::Fill{"="}, + // // option::Lead{">"}, + // // option::Remainder{" "}, + // // option::End{"]"}, + // option::PrefixText{"Downloading file "}, + // option::PostfixText{""}, + // option::ForegroundColor{Color::green}, + // option::FontStyles{std::vector{FontStyle::bold}}, + // }; + // // option::ShowElapsedTime{true}, + // // option::ShowRemainingTime{true}, + // IndeterminateProgressBar bar_unknown { + // option::BarWidth{50}, + // option::Start{"["}, + // option::Fill{"."}, + // option::Lead{"<==>"}, + // option::PrefixText{"Downloading file "}, + // option::End{"]"}, + // option::PostfixText{""}, + // option::ForegroundColor{Color::green}, + // option::FontStyles{std::vector{FontStyle::bold}}, + // }; + + struct transfer : public non_copyable{ + transfer(){ curl = curl_easy_init(); } + transfer(transfer&&){} + ~transfer(){ } + + CURLcode res; + int id; + CURL *curl = nullptr; + FILE *fp = nullptr; + utils::memory_buffer data = {0}; + + }; + using transfers = std::vector; + using responses = std::vector; + class context : public non_copyable{ public: context(); ~context(); - inline CURL* const get_curl() const; string url_escape(const string& url); - response request_get(const string& url, const http::request_params& params = http::request_params()); - response request_post(const string& url, const http::request_params& params = http::request_params()); - response download_file(const string& url, const string& storage_path, const http::request_params& params = http::request_params()); + response request(const string& url, const http::request& params = http::request()); + response download_file(const string& url, const string& storage_path, const http::request& params = http::request()); long get_download_size(const string& url); long get_bytes_downloaded(const string& url); - + private: - CURL *curl; - curl_slist* _add_headers(CURL *curl, const headers_t& headers); + CURL *curl = nullptr; }; - - extern context http; + class multi{ + public: + multi(long max_allowed_transfers = 2); + ~multi(); + string url_escape(const string& url); + ptr make_requests(const string_list& urls, const http::request& params = http::request()); + ptr make_downloads(const string_list& url, const string_list& storage_path, const http::request& params = http::request()); + ptr execute(ptr transfers, size_t timeout = 1000); + + private: + DynamicProgress progress_bars; + CURLM *cm = nullptr; + CURLMsg *cmessage = nullptr; + CURLMcode cres; + int messages_left = -1; + }; + + curl_slist* add_headers(CURL *curl, const headers_t& headers); + static size_t write_to_buffer(char *contents, size_t size, size_t nmemb, void *userdata); + static size_t write_to_stream(char *ptr, size_t size, size_t nmemb, void *userdata); + static int show_download_progress(void *ptr, curl_off_t total_download, curl_off_t current_downloaded, curl_off_t total_upload, curl_off_t current_upload); + } \ No newline at end of file diff --git a/include/package.hpp b/include/package.hpp index 353137a..4c18534 100644 --- a/include/package.hpp +++ b/include/package.hpp @@ -110,8 +110,11 @@ namespace gdpm::package { GDPM_DLL_EXPORT error search(const config::context& config, const title_list& package_titles, const params& params = package::params()); GDPM_DLL_EXPORT error list(const config::context& config, const params& params = package::params()); GDPM_DLL_EXPORT error export_to(const path_list& paths); + GDPM_DLL_EXPORT error clean(const config::context& config, const title_list& package_titles); + GDPM_DLL_EXPORT error purge(const config::context& config); GDPM_DLL_EXPORT error link(const config::context& config, const title_list& package_titles, const params& params = package::params()); GDPM_DLL_EXPORT error clone(const config::context& config, const title_list& package_titles, const params& params = package::params()); + GDPM_DLL_EXPORT result_t fetch(const config::context& config, const title_list& package_titles); GDPM_DLL_EXPORT void print_list(const rapidjson::Document& json); @@ -120,12 +123,10 @@ namespace gdpm::package { GDPM_DLL_EXPORT void print_table(const rapidjson::Document& json); GDPM_DLL_EXPORT result_t get_package_info(const opts_t& opts); GDPM_DLL_EXPORT result_t get_package_titles(const info_list& packages); - GDPM_DLL_EXPORT void clean_temporary(const config::context& config, const title_list& package_titles); GDPM_DLL_EXPORT void read_file_inputs(title_list& package_titles, const path_list& paths); GDPM_DLL_EXPORT info_list find_cached_packages(const title_list& package_titles); GDPM_DLL_EXPORT info_list find_installed_packages(const title_list& package_titles); /* Dependency Management API */ - GDPM_DLL_EXPORT result_t synchronize_database(const config::context& config, const title_list& package_titles); GDPM_DLL_EXPORT result_t resolve_dependencies(const config::context& config, const title_list& package_titles); GDPM_DLL_EXPORT string to_json(const info& info, bool pretty_print = false); diff --git a/include/package_manager.hpp b/include/package_manager.hpp index 5a60086..fe63961 100644 --- a/include/package_manager.hpp +++ b/include/package_manager.hpp @@ -27,6 +27,7 @@ namespace gdpm::package_manager { update, search, p_export, /* reserved keyword */ + purge, list, link, clone, diff --git a/include/rest_api.hpp b/include/rest_api.hpp index 0847069..efcc351 100644 --- a/include/rest_api.hpp +++ b/include/rest_api.hpp @@ -63,25 +63,29 @@ namespace gdpm::rest_api{ }; request_params make_from_config(const config::context& config); - string to_json(const rapidjson::Document& doc); + string to_json(const json::document& doc); string to_string(type_e type); string to_string(support_e support); string to_string(sort_e sort); error print_params(const request_params& params, const string& filter = ""); error print_asset(const request_params& params, const string& filter = "", const print::style& style = print::style::list); - rapidjson::Document _parse_json(const string& r, int verbose = 0); + json::document _parse_json(const string& r, int verbose = 0); string _prepare_request(const string& url, const request_params& context, const string& filter); bool register_account(const string& username, const string& password, const string& email); bool login(const string& username, const string& password); bool logout(); - rapidjson::Document configure(const string& url = constants::HostUrl, type_e type = any, int verbose = 0); - rapidjson::Document get_assets_list(const string& url, const request_params& params = {}, const string& filter = ""); - rapidjson::Document get_asset(const string& url, int asset_id, const request_params& params = {}, const string& filter = ""); + json::document configure(const string& url = constants::HostUrl, type_e type = any, int verbose = 0); + json::document get_assets_list(const string& url, const request_params& params = {}, const string& filter = ""); + json::document get_asset(const string& url, int asset_id, const request_params& params = {}, const string& filter = ""); bool delete_asset(int asset_id); // ...for moderators bool undelete_asset(int asset_id); // ...for moderators bool set_support_level(int asset_id); // ...for moderators + + namespace multi{ + json::documents get_assets(const string_list& urls, id_list aset_ids, const request_params& api_params, const string_list& filters); + } /* POST /asset diff --git a/include/result.hpp b/include/result.hpp index ac9229a..7553609 100644 --- a/include/result.hpp +++ b/include/result.hpp @@ -2,13 +2,12 @@ #include "log.hpp" #include "error.hpp" -#include "types.hpp" #include #include namespace gdpm{ - template + template class result_t { public: result_t() = delete; @@ -35,13 +34,14 @@ namespace gdpm{ fn_error = error; } + constexpr U get_error() const{ + return std::get(data); + } + constexpr std::unique_ptr unwrap() const { /* First, check if ok() and error() are defined. */ if(!fn_error || !fn_ok){ - error error( - constants::error::NOT_DEFINED - ); - log::error(error); + log::error(error(ec::NOT_DEFINED)); return nullptr; } /* Then, attempt unwrap the data. */ diff --git a/include/types.hpp b/include/types.hpp index 7345a57..6896f0e 100644 --- a/include/types.hpp +++ b/include/types.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -41,13 +42,11 @@ namespace gdpm{ SIZE_T = 6, }; - template - concept error_t = requires{ std::is_same::value; }; - using string = std::string; using string_list = std::vector; using string_map = std::unordered_map; using string_pair = std::pair; + using id_list = std::vector; using any = std::any; using var = std::variant; template @@ -65,6 +64,15 @@ namespace gdpm{ template using _task_list = std::vector>; using task_list = _task_list; + template + using ptr = std::unique_ptr; + namespace json{ + using document = rapidjson::Document; + using documents = std::vector; + } + namespace concepts{ + template concept require_min_args = requires (std::size_t min){ sizeof...(Args) > min; }; + } inline string_list unwrap(const var_args& args){ string_list sl; diff --git a/include/utils.hpp b/include/utils.hpp index aa46221..3a75be3 100644 --- a/include/utils.hpp +++ b/include/utils.hpp @@ -1,6 +1,7 @@ #pragma once #include "constants.hpp" +#include "types.hpp" #include #include @@ -71,6 +72,11 @@ namespace gdpm::utils { std::move(part, from.end(), std::back_inserter(from)); from.erase(part); } + template + std::vector append(const std::vector& a, const std::vector& b){ + a.insert(std::end(a), std::begin(b), std::end(b)); + return a; + } bool to_bool(const std::string& s); std::vector split_lines(const std::string& contents); @@ -84,7 +90,7 @@ namespace gdpm::utils { std::vector parse_lines(const std::string& s); std::string replace_first(const std::string& s, const std::string& from, const std::string& to); std::string replace_all(const std::string& s, const std::string& from, const std::string& to); - int extract_zip(const char *archive, const char *dest, int verbose = 0); + error extract_zip(const char *archive, const char *dest, int verbose = 0); std::string prompt_user(const char *message); bool prompt_user_yn(const char *message); void delay(std::chrono::milliseconds milliseconds = GDPM_REQUEST_DELAY); @@ -93,11 +99,6 @@ namespace gdpm::utils { std::string convert_size(long size); // TODO: Add function to get size of decompressed zip - namespace curl { - extern size_t write_to_buffer(char *contents, size_t size, size_t nmemb, void *userdata); - extern size_t write_to_stream(char *ptr, size_t size, size_t nmemb, void *userdata); - extern int show_progress(void *ptr, curl_off_t total_download, curl_off_t current_downloaded, curl_off_t total_upload, curl_off_t current_upload); - } namespace json { std::string from_array(const std::set& a, const std::string& prefix); std::string from_object(const std::unordered_map& m, const std::string& prefix, const std::string& spaces); diff --git a/src/cache.cpp b/src/cache.cpp index 2a40f6d..aa8a593 100644 --- a/src/cache.cpp +++ b/src/cache.cpp @@ -29,15 +29,15 @@ namespace gdpm::cache{ namespace fs = std::filesystem; fs::path dir_path = fs::path(params.cache_path).parent_path(); if(!fs::exists(dir_path)){ - log::info("Creating cache directories...{}", params.cache_path); + log::debug("Creating cache directories...{}", params.cache_path); fs::create_directories(dir_path); } int rc = sqlite3_open(params.cache_path.c_str(), &db); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, + error error(ec::SQLITE_ERR, std::format( - "create_package_database.sqlite3_open(): {}", + "cache::create_package_database::sqlite3_open(): {}", sqlite3_errmsg(db) ) ); @@ -70,8 +70,8 @@ namespace gdpm::cache{ rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); if(rc != SQLITE_OK){ // log::error("Failed to fetch data: {}\n", sqlite3_errmsg(db)); - error error(constants::error::SQLITE_ERR, std::format( - "create_package_database.sqlite3_exec(): {}", + error error(ec::SQLITE_ERR, std::format( + "cache::create_package_database::sqlite3_exec(): {}", errmsg )); sqlite3_free(errmsg); @@ -101,8 +101,8 @@ namespace gdpm::cache{ // log::println("{}", sql); int rc = sqlite3_open(params.cache_path.c_str(), &db); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "insert_package_info.sqlite3_open(): {}", + error error(ec::SQLITE_ERR, std::format( + "cache::insert_package_info::sqlite3_open(): {}", sqlite3_errmsg(db) )); sqlite3_close(db); @@ -110,10 +110,9 @@ namespace gdpm::cache{ } rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); if(rc != SQLITE_OK){ - error error(rc, std::format( - "insert_package_info.sqlite3_exec(): {}", errmsg - )); - log::error(error); + error error = log::error_rc(ec::SQLITE_ERR, + std::format("cache::insert_package_info::sqlite3_exec(): {}", errmsg) + ); sqlite3_free(errmsg); sqlite3_close(db); return error; @@ -135,8 +134,6 @@ namespace gdpm::cache{ string sql{"BEGIN TRANSACTION;\n"}; auto callback = [](void *data, int argc, char **argv, char **colnames){ - // log::error("{}", (const char*)data); - // p_data *_data = (p_data*)data; package::info_list *_p_vector = (package::info_list*) data; package::info p{ .asset_id = std::stoul(argv[1]), @@ -163,8 +160,8 @@ namespace gdpm::cache{ int rc = sqlite3_open(params.cache_path.c_str(), &db); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "get_package_info_by_id.sqlite3_open(): {}", sqlite3_errmsg(db) + error error(ec::SQLITE_ERR, std::format( + "cache::get_package_info_by_id::sqlite3_open(): {}", sqlite3_errmsg(db) )); sqlite3_close(db); return result_t(package::info_list(), error); @@ -177,7 +174,7 @@ namespace gdpm::cache{ rc = sqlite3_exec(db, sql.c_str(), callback, (void*)&p_vector, &errmsg); if(rc != SQLITE_OK){ error error(constants::error::SQLITE_ERR, std::format( - "get_package_info_by_id.sqlite3_exec(): {}", errmsg + "cache::get_package_info_by_id::sqlite3_exec(): {}", errmsg )); sqlite3_free(errmsg); sqlite3_close(db); @@ -231,8 +228,8 @@ namespace gdpm::cache{ int rc = sqlite3_open(params.cache_path.c_str(), &db); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "get_package_info_by_title.sqlite3_open(): {}", sqlite3_errmsg(db) + error error(ec::SQLITE_ERR, std::format( + "cache::get_package_info_by_title::sqlite3_open(): {}", sqlite3_errmsg(db) )); sqlite3_close(db); return result_t(package::info_list(), error); @@ -246,8 +243,8 @@ namespace gdpm::cache{ // log::println(sql); rc = sqlite3_exec(db, sql.c_str(), callback, (void*)&p_vector, &errmsg); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "get_package_info_by_title.sqlite3_exec(): {}", errmsg + error error(ec::SQLITE_ERR, std::format( + "cache::get_package_info_by_title::sqlite3_exec(): {}", errmsg )); sqlite3_free(errmsg); sqlite3_close(db); @@ -292,8 +289,8 @@ namespace gdpm::cache{ int rc = sqlite3_open(params.cache_path.c_str(), &db); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "get_installed_packages.sqlite3_open(): {}", sqlite3_errmsg(db) + error error(ec::SQLITE_ERR, std::format( + "cache::get_installed_packages::sqlite3_open(): {}", sqlite3_errmsg(db) )); sqlite3_close(db); return result_t(package::info_list(), error); @@ -302,8 +299,8 @@ namespace gdpm::cache{ sql += "SELECT * FROM " + params.table_name + " WHERE is_installed=1; COMMIT;"; rc = sqlite3_exec(db, sql.c_str(), callback, (void*)&p_vector, &errmsg); if(rc != SQLITE_OK){ - error error(rc, std::format( - "get_installed_packages.sqlite3_exec(): {}", errmsg + error error(ec::SQLITE_ERR, std::format( + "cache::get_installed_packages::sqlite3_exec(): {}", errmsg )); log::error(error); sqlite3_free(errmsg); @@ -326,8 +323,8 @@ namespace gdpm::cache{ int rc = sqlite3_open(params.cache_path.c_str(), &db); if(rc != SQLITE_OK){ error error( - constants::error::SQLITE_ERR, std::format( - "update_package_info.sqlite3_open(): {}", sqlite3_errmsg(db) + ec::SQLITE_ERR, std::format( + "cache::update_package_info::sqlite3_open(): {}", sqlite3_errmsg(db) )); sqlite3_close(db); return error; @@ -359,8 +356,8 @@ namespace gdpm::cache{ } rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "update_package_info.sqlite3_exec(): {}\n\t{}", errmsg, sql + error error(ec::SQLITE_ERR, std::format( + "cache::update_package_info::sqlite3_exec(): {}\n\t{}", errmsg, sql )); sqlite3_free(errmsg); sqlite3_close(db); @@ -382,8 +379,8 @@ namespace gdpm::cache{ int rc = sqlite3_open(params.cache_path.c_str(), &db); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "delete_packages.sqlite3_open(): {}", sqlite3_errmsg(db) + error error(ec::SQLITE_ERR, std::format( + "cache::delete_packages::sqlite3_open(): {}", sqlite3_errmsg(db) )); sqlite3_close(db); return error; @@ -395,8 +392,8 @@ namespace gdpm::cache{ } rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "delete_packages.sqlite3_exec(): {}", errmsg + error error(ec::SQLITE_ERR, std::format( + "cache::delete_packages::sqlite3_exec(): {}", errmsg )); sqlite3_free(errmsg); sqlite3_close(db); @@ -418,8 +415,8 @@ namespace gdpm::cache{ int rc = sqlite3_open(params.cache_path.c_str(), &db); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "delete_packages.sqlite3_open(): {}", errmsg + error error(ec::SQLITE_ERR, std::format( + "cache::delete_packages::sqlite3_open(): {}", errmsg )); sqlite3_close(db); return error; @@ -431,8 +428,8 @@ namespace gdpm::cache{ } rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "delete_packages.sqlite3_exec(): {}", errmsg + error error(ec::SQLITE_ERR, std::format( + "cache::delete_packages::sqlite3_exec(): {}", errmsg )); sqlite3_free(errmsg); sqlite3_close(db); @@ -451,8 +448,8 @@ namespace gdpm::cache{ int rc = sqlite3_open(params.cache_path.c_str(), &db); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "drop_package_database.sqlite3_open(): {}", sqlite3_errmsg(db) + error error(ec::SQLITE_ERR, std::format( + "cache::drop_package_database::sqlite3_open(): {}", sqlite3_errmsg(db) )); sqlite3_close(db); return error; @@ -460,8 +457,8 @@ namespace gdpm::cache{ rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); if(rc != SQLITE_OK){ - error error(constants::error::SQLITE_ERR, std::format( - "drop_package_database.sqlite3_exec(): {}", errmsg + error error(ec::SQLITE_ERR, std::format( + "cache::drop_package_database::sqlite3_exec(): {}", errmsg )); sqlite3_free(errmsg); sqlite3_close(db); diff --git a/src/config.cpp b/src/config.cpp index 665a02c..9e5ddc8 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -106,18 +106,15 @@ namespace gdpm::config{ ParseErrorCode status = doc.Parse(contents.c_str()).GetParseError(); if(!doc.IsObject()){ - error error( - constants::error::FILE_NOT_FOUND, + return log::error_rc( + ec::FILE_NOT_FOUND, "Could not load config file." ); - log::error(error); - return error; } error error = validate(doc); - if(error()){ - log::error(error); - return error; + if(error.has_occurred()){ + return log::error_rc(error); } /* Make sure contents were read correctly. */ @@ -138,12 +135,10 @@ namespace gdpm::config{ ); } } else { - gdpm::error error( - constants::error::INVALID_KEY, + return log::error_rc( + ec::INVALID_KEY, "Could not read key `remote_sources`." ); - log::error(error); - return error; } } auto _get_value_string = [](Document& doc, const char *property){ diff --git a/src/http.cpp b/src/http.cpp index 94f90d7..83a1032 100644 --- a/src/http.cpp +++ b/src/http.cpp @@ -2,10 +2,14 @@ #include "http.hpp" #include "utils.hpp" #include "log.hpp" +#include "error.hpp" #include #include +#include +#include #include #include +#include namespace gdpm::http{ @@ -15,114 +19,70 @@ namespace gdpm::http{ curl = curl_easy_init(); } + context::~context(){ - curl_global_cleanup(); curl_easy_cleanup(curl); + curl_global_cleanup(); } - CURL* const context::get_curl() const{ - return curl; - } string context::url_escape(const string &url){ return curl_easy_escape(curl, url.c_str(), url.size());; } - response context::request_get( - const string& url, - const http::request_params& params + + response context::request( + const string& url, + const http::request& params ){ CURLcode res; utils::memory_buffer buf = utils::make_buffer(); + utils::memory_buffer data = utils::make_buffer(); response r; #if (GDPM_DELAY_HTTP_REQUESTS == 1) using namespace std::chrono_literals; utils::delay(); #endif - - // curl_global_init(CURL_GLOBAL_ALL); - // curl = curl_easy_init(); if(curl){ - utils::memory_buffer *data; - curl_slist *list = _add_headers(curl, params.headers); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - // curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl"); - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET"); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_buffer); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); - curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data); - curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress); - curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str()); - curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout); - res = curl_easy_perform(curl); - curl_slist_free_all(list); - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code); - if(res != CURLE_OK && params.verbose > 0) - log::error("_make_request.curl_easy_perform(): {}", curl_easy_strerror(res)); - curl_easy_cleanup(curl); - } - - r.body = buf.addr; - utils::free_buffer(buf); - // curl_global_cleanup(); - return r; - } - - - response context::request_post( - const string& url, - const http::request_params& params - ){ - // CURL *curl = nullptr; - CURLcode res; - utils::memory_buffer buf = utils::make_buffer(); - response r; - -#if (GDPM_DELAY_HTTP_REQUESTS == 1) - using namespace std::chrono_literals; - utils::delay(); -#endif - string h; - std::for_each( - params.headers.begin(), - params.headers.end(), - [&h](const string_pair& kv){ - h += kv.first + "=" + kv.second + "&"; + curl_slist *list = add_headers(curl, params.headers); + if(params.method == method::POST){ + string h; + std::for_each( + params.headers.begin(), + params.headers.end(), + [&h](const string_pair& kv){ + h += kv.first + "=" + kv.second + "&"; + } + ); + h.pop_back(); + h = url_escape(h); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, h.size()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, h.c_str()); + } + else if(params.method == method::GET){ + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET"); } - ); - h.pop_back(); - h = url_escape(h); - - // const char *post_fields = ""; - // curl_global_init(CURL_GLOBAL_ALL); - // curl = curl_easy_init(); - if(curl){ - utils::memory_buffer *data; - curl_slist *list = _add_headers(curl, params.headers); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - // curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl"); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, h.size()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, h.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_buffer); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_to_buffer); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); - curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data); - curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress); + curl_easy_setopt(curl, CURLOPT_XFERINFODATA, (void*)&data); + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, show_download_progress); curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str()); curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout); res = curl_easy_perform(curl); curl_slist_free_all(list); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code); if(res != CURLE_OK && params.verbose > 0) - log::error("_make_request.curl_easy_perform(): {}", curl_easy_strerror(res)); + log::error(ec::LIBCURL_ERR, + std::format("http::context::request::curl_easy_perform(): {}", curl_easy_strerror(res)) + ); curl_easy_cleanup(curl); } r.body = buf.addr; utils::free_buffer(buf); - // curl_global_cleanup(); return r; } @@ -130,7 +90,7 @@ namespace gdpm::http{ response context::download_file( const string& url, const string& storage_path, - const http::request_params& params + const http::request& params ){ // CURL *curl = nullptr; CURLcode res; @@ -141,34 +101,19 @@ namespace gdpm::http{ using namespace std::chrono_literals; utils::delay(); #endif - - // curl_global_init(CURL_GLOBAL_ALL); - // curl = curl_easy_init(); if(curl){ fp = fopen(storage_path.c_str(), "wb"); - // if(!config.username.empty() && !config.password.empty()){ - // std::string curlopt_userpwd{config.username + ":" + config.password}; - // curl_easy_setopt(curl, CURLOPT_USERPWD, curlopt_userpwd.c_str()); - // } - - // /* Switch on full protocol/debug output while testing and disable - // * progress meter by setting to 0L */ - // if(config.verbose){ - // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); - // curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); - // } utils::memory_buffer *data; - curl_slist *list = _add_headers(curl, params.headers); + curl_slist *list = add_headers(curl, params.headers); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - // curl_easy_setopt(curl, CURLOPT_USERPWD, "user:pass"); curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); curl_easy_setopt(curl, CURLOPT_HEADER, 0); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_stream); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_to_stream); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data); - curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress); + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, show_download_progress); curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str()); curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout); res = curl_easy_perform(curl); @@ -177,16 +122,16 @@ namespace gdpm::http{ /* Get response code, process error, save data, and close file. */ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code); if(res != CURLE_OK && params.verbose > 0){ - log::error("download_file.curl_easy_perform() failed: {}", curl_easy_strerror(res)); + log::error(ec::LIBCURL_ERR, + std::format("http::context::download_file::curl_easy_perform() failed: {}", curl_easy_strerror(res)) + ); } fclose(fp); } - // curl_global_cleanup(); return r; } long context::get_download_size(const string& url){ - // CURL *curl = curl_easy_init(); CURLcode res; if(curl){ curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); @@ -221,7 +166,185 @@ namespace gdpm::http{ } - curl_slist* context::_add_headers( + multi::multi(long max_allowed_transfers){ + curl_global_init(CURL_GLOBAL_ALL); + if(max_allowed_transfers > 1) + cm = curl_multi_init(); + curl_multi_setopt(cm, CURLMOPT_MAXCONNECTS, (long)max_allowed_transfers); + } + + multi::~multi(){ + if(cm != nullptr) + curl_multi_cleanup(cm); + curl_global_cleanup(); + } + + + string multi::url_escape(const string &url){ + return curl_easy_escape(cm, url.c_str(), url.size());; + } + + + ptr multi::make_requests( + const string_list& urls, + const http::request& params + ){ + if(cm == nullptr){ + log::error(error(PRECONDITION_FAILED, + "http::multi::make_downloads(): multi client not initialized.") + ); + return std::make_unique(); + } + if(urls.size() <= 0){ + log::warn("No requests to make."); + return std::make_unique(); + } + ptr ts = std::make_unique(); + for(const auto& url : urls){ + transfer t; + if(t.curl){ + curl_slist *list = add_headers(t.curl, params.headers); + if(params.method == method::POST){ + string h; + std::for_each( + params.headers.begin(), + params.headers.end(), + [&h](const string_pair& kv){ + h += kv.first + "=" + kv.second + "&"; + } + + ); + h.pop_back(); + h = url_escape(h); + curl_easy_setopt(t.curl, CURLOPT_POSTFIELDSIZE, h.size()); + curl_easy_setopt(t.curl, CURLOPT_POSTFIELDS, h.c_str()); + } + else if(params.method == method::GET){ + curl_easy_setopt(t.curl, CURLOPT_CUSTOMREQUEST, "GET"); + } + curl_easy_setopt(t.curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(t.curl, CURLOPT_WRITEDATA, (void*)&t.data); + curl_easy_setopt(t.curl, CURLOPT_WRITEFUNCTION, write_to_buffer); + curl_easy_setopt(t.curl, CURLOPT_NOPROGRESS, false); + curl_easy_setopt(t.curl, CURLOPT_XFERINFODATA, &t.data); + curl_easy_setopt(t.curl, CURLOPT_XFERINFOFUNCTION, show_download_progress); + curl_easy_setopt(t.curl, CURLOPT_USERAGENT, constants::UserAgent.c_str()); + curl_easy_setopt(t.curl, CURLOPT_TIMEOUT_MS, params.timeout); + } + } + + return ts; + } + + + ptr multi::make_downloads( + const string_list& urls, + const string_list& storage_paths, + const http::request& params + ){ + if(cm == nullptr){ + log::error(error(ec::PRECONDITION_FAILED, + "http::multi::make_downloads(): multi client not initialized.") + ); + return std::make_unique(); + } + if(urls.size() != storage_paths.size()){ + log::error(error(ec::ASSERTION_FAILED, + "http::context::make_downloads(): urls.size() != storage_paths.size()" + )); + } + ptr ts = std::make_unique(); + for(size_t i = 0; i < urls.size(); i++){ + const string& url = urls.at(i); + const string& storage_path = storage_paths.at(i); + response r; + transfer t; + t.id = i; + + if(t.curl){ + t.fp = fopen(storage_path.c_str(), "wb"); + curl_slist *list = add_headers(t.curl, params.headers); + curl_easy_setopt(t.curl, CURLOPT_URL, url.c_str()); + // curl_easy_setopt(t.curl, CURLOPT_PRIVATE, url.c_str()); + curl_easy_setopt(t.curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(t.curl, CURLOPT_HEADER, 0); + curl_easy_setopt(t.curl, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(t.curl, CURLOPT_WRITEDATA, t.fp); + curl_easy_setopt(t.curl, CURLOPT_WRITEFUNCTION, write_to_stream); + curl_easy_setopt(t.curl, CURLOPT_NOPROGRESS, false); + curl_easy_setopt(t.curl, CURLOPT_XFERINFODATA, &t.data); + curl_easy_setopt(t.curl, CURLOPT_XFERINFOFUNCTION, show_download_progress); + curl_easy_setopt(t.curl, CURLOPT_USERAGENT, constants::UserAgent.c_str()); + curl_easy_setopt(t.curl, CURLOPT_TIMEOUT_MS, params.timeout); + cres = curl_multi_add_handle(cm, t.curl); + curl_slist_free_all(list); + if(cres != CURLM_OK){ + log::error(ec::LIBCURL_ERR, + std::format("http::context::make_downloads(): {}", curl_multi_strerror(cres)) + ); + } + ts->emplace_back(std::move(t)); + /* NOTE: Should the file pointer be closed here? */ + } + } + + return ts; + } + + + ptr multi::execute( + ptr transfers, + size_t timeout + ){ + if(cm == nullptr){ + log::error(error(PRECONDITION_FAILED, + "http::multi::execute(): multi client not initialized") + ); + return std::make_unique(); + } + if(transfers->empty()){ + log::debug("http::multi::execute(): no transfers found"); + return std::make_unique(); + } + size_t transfers_left = transfers->size(); + ptr responses = std::make_unique(transfers->size()); + do{ + int still_alive = 1; + cres = curl_multi_perform(cm, &still_alive); + + while((cmessage = curl_multi_info_read(cm, &messages_left))){ + if(cmessage->msg == CURLMSG_DONE){ + char *url = nullptr; + transfer& t = transfers->at(transfers_left-1); + response& r = responses->at(transfers_left-1); + t.curl = cmessage->easy_handle; + curl_easy_getinfo(cmessage->easy_handle, CURLINFO_EFFECTIVE_URL, &url); + curl_easy_getinfo(cmessage->easy_handle, CURLINFO_RESPONSE_CODE, &r.code); + if((int)cmessage->data.result != CURLM_OK){ + log::error(error(ec::LIBCURL_ERR, + std::format("http::context::execute({}): {} ", (int)cmessage->data.result, curl_easy_strerror(cmessage->data.result), url)) + ); + } + curl_multi_remove_handle(cm, t.curl); + curl_easy_cleanup(t.curl); + if(t.fp) fclose(t.fp); + transfers->pop_back(); + transfers_left -= 1; + } + else{ + log::error(error(ec::LIBCURL_ERR, + std::format("http::context::execute(): {}", (int)cmessage->msg)) + ); + } + } + if(transfers_left) + curl_multi_wait(cm, NULL, 0, timeout, NULL); + }while(transfers_left); + return responses; + } + + + curl_slist* add_headers( CURL *curl, const headers_t& headers ){ @@ -235,4 +358,83 @@ namespace gdpm::http{ } return list; } + + + size_t write_to_buffer( + char *contents, + size_t size, + size_t nmemb, + void *userdata + ){ + size_t realsize = size * nmemb; + utils::memory_buffer *m = (utils::memory_buffer*)userdata; + + m->addr = (char*)realloc(m->addr, m->size + realsize + 1); + if(m->addr == nullptr){ + /* Out of memory */ + log::error("Could not allocate memory (realloc returned NULL)."); + return 0; + } + + memcpy(&(m->addr[m->size]), contents, realsize); + m->size += realsize; + m->addr[m->size] = 0; + + return realsize; + } + + + size_t write_to_stream( + char *ptr, + size_t size, + size_t nmemb, + void *userdata + ){ + if(nmemb == 0) + return 0; + + return fwrite(ptr, size, nmemb, (FILE*)userdata); + } + + + int show_download_progress( + void *ptr, + curl_off_t total_download, + curl_off_t current_downloaded, + curl_off_t total_upload, + curl_off_t current_upload + ){ + if(current_downloaded >= total_download) + return 0; + using namespace indicators; + show_console_cursor(false); + // if(total_download != 0){ + // // double percent = std::floor((current_downloaded / (total_download)) * 100); + // bar.set_option(option::MaxProgress{total_download}); + // // bar.set_option(option::HideBarWhenComplete{false}); + // bar.set_progress(current_downloaded); + // bar.set_option(option::PostfixText{ + // utils::convert_size(current_downloaded) + " / " + + // utils::convert_size(total_download) + // }); + // if(bar.is_completed()){ + // bar.set_option(option::PrefixText{"Download completed."}); + // bar.mark_as_completed(); + // } + // } else { + // if(bar_unknown.is_completed()){ + // bar_unknown.set_option(option::PrefixText{"Download completed."}); + // bar_unknown.mark_as_completed(); + // } else { + // bar.tick(); + // bar_unknown.set_option( + // option::PostfixText(std::format("{}", utils::convert_size(current_downloaded))) + // ); + + // } + // } + show_console_cursor(true); + utils::memory_buffer *m = (utils::memory_buffer*)ptr; + return 0; + } } \ No newline at end of file diff --git a/src/package.cpp b/src/package.cpp index 75f66bc..03a061a 100644 --- a/src/package.cpp +++ b/src/package.cpp @@ -40,97 +40,110 @@ namespace gdpm::package{ */ + /* Synchronize database information and then try to get data again from + cache if possible. */ + if(config.enable_sync){ + result_t result = fetch(config, package_titles); + error error = result.get_error(); + if(error.has_occurred()){ + return log::error_rc(ec::UNKNOWN, "package::install(): could not synchronize database."); + } + } /* Append files from --file option */ read_file_inputs(package_titles, params.input_files); - result_t result = cache::get_package_info_by_title(package_titles); package::info_list p_cache = result.unwrap_unsafe(); - /* Synchronize database information and then try to get data again from - cache if possible. */ - if(config.enable_sync || p_cache.empty()){ - result_t result = synchronize_database(config, package_titles); - p_cache = result.unwrap_unsafe(); - } - - /* Match queried package titles with those found in cache. */ - package::info_list p_found = find_cached_packages(package_titles); - - // if(p_found.size() == package_titles.size()){ - // log::info("Found all packages stored in local cache."); - // } - /* Found nothing to install so there's nothing to do at this point. */ - if(p_found.empty()){ - error error( - constants::error::NOT_FOUND, - "No packages found to install." + if(p_cache.empty()){ + return log::error_rc( + ec::NOT_FOUND, /* TODO: change to PACKAGE_NOT_FOUND */ + "package::install(): no packages found to install." ); - log::error(error); - return error; } - log::println("Packages to install: "); - for(const auto& p : p_found){ - string output((p.is_installed) ? p.title + " (reinstall)" : p.title); - log::print(" {} ", (p.is_installed) ? p.title + " (reinstall)" : p.title); + /* Show packages to install */ + { + using namespace tabulate; + Table table; + table.format() + .border_top("") + .border_bottom("") + .border_left("") + .border_right("") + .corner("") + .padding_top(0) + .padding_bottom(0); + table.add_row({"Title", "Author", "Category", "Version", "Godot", "Last Modified", "Installed?"}); + table[0].format() + .font_style({FontStyle::underline, FontStyle::bold}); + for(const auto& p : p_cache){ + table.add_row({p.title, p.author, p.category, p.version, p.godot_version, p.modify_date, (p.is_installed) ? "✔️": "❌"}); + size_t index = table.size() - 1; + table[index][3].format().font_align(FontAlign::center); + table[index][4].format().font_align(FontAlign::center); + table[index][6].format().font_align(FontAlign::center); + + // string output(p.title + GDPM_COLOR_CYAN " v" + p.version + GDPM_COLOR_RESET); + // output += GDPM_COLOR_BLUE " last updated: " + p.modify_date + GDPM_COLOR_RESET; + // output += (p.is_installed) ? GDPM_COLOR_LIGHT_CYAN " (reinstall)" : ""; + // output += GDPM_COLOR_RESET; + // log::print(" {}\n", output); + } + table.print(std::cout); + log::println(""); } - log::println(""); + /* Skip prompt if set in config */ if(!config.skip_prompt){ if(!utils::prompt_user_yn("Do you want to install these packages? (Y/n)")) return error(); } - /* Check if provided param is in remote sources*/ + /* Check if provided remote param is in remote sources */ if(!config.remote_sources.contains(params.remote_source)){ - error error( - constants::error::NOT_FOUND, - "Remote source not found in config." + return log::error_rc( + ec::NOT_FOUND, /* TODO: change to REMOTE_NOT_FOUND */ + "package::install(): remote source not found in config." ); - log::error(error); - return error; } - /* Try and obtain all requested packages. */ - std::vector dir_pairs; - task_list tasks; + // task_list tasks; + /* Retrieve necessary asset data if it was found already in cache */ + std::vector target_extract_dirs; rest_api::request_params rest_api_params = rest_api::make_from_config(config); - for(auto& p : p_found){ // TODO: Execute each in parallel using coroutines?? - - /* Check if a remote source was provided. If not, then try to get packages - in global storage location only. */ - - log::info("Fetching asset data for \"{}\"...", p.title); + package::title_list p_download_urls; + package::path_list p_storage_paths; + for(auto& p : p_cache){ + log::info_n("Fetching asset data for \"{}\"...", p.title); string url{config.remote_sources.at(params.remote_source) + rest_api::endpoints::GET_AssetId}; - string package_dir, tmp_dir, tmp_zip; + string package_dir = config.packages_dir + "/" + p.title; + string tmp_dir = config.tmp_dir + "/" + p.title; + string tmp_zip = tmp_dir + ".zip"; - /* Retrieve necessary asset data if it was found already in cache */ Document doc; - bool is_missing_data = p.download_url.empty() || p.category.empty() || p.description.empty() || p.support_level.empty(); - if(is_missing_data){ + bool is_data_missing = p.download_url.empty() || p.category.empty() || p.description.empty() || p.support_level.empty(); + if(is_data_missing){ doc = rest_api::get_asset(url, p.asset_id, rest_api_params); if(doc.HasParseError() || doc.IsNull()){ - return log::error_rc(error( - constants::error::JSON_ERR, - std::format("Error parsing JSON: {}", GetParseError_En(doc.GetParseError())) - )); + return log::error_rc( + ec::JSON_ERR, + std::format("package::install(): error parsing JSON: {}", + GetParseError_En(doc.GetParseError())) + ); } p.category = doc["category"].GetString(); p.description = doc["description"].GetString(); p.support_level = doc["support_level"].GetString(); p.download_url = doc["download_url"].GetString(); p.download_hash = doc["download_hash"].GetString(); + log::println("Done"); } else{ - log::info("Found asset data found for \"{}\"", p.title); + log::println(""); + log::info("Found asset data for \"{}\".", p.title); } - - /* Set directory and temp paths for storage */ - package_dir = config.packages_dir + "/" + p.title; - tmp_dir = config.tmp_dir + "/" + p.title; - tmp_zip = tmp_dir + ".zip"; - + /* Make directories for packages if they don't exist to keep everything organized */ if(!std::filesystem::exists(config.tmp_dir)) std::filesystem::create_directories(config.tmp_dir); @@ -145,60 +158,83 @@ namespace gdpm::package{ OStreamWrapper osw(ofs); PrettyWriter writer(osw); doc.Accept(writer); + target_extract_dirs.emplace_back(string_pair(tmp_zip, package_dir + "/")); /* Check if we already have a stored temporary file before attempting to download */ if(std::filesystem::exists(tmp_zip) && std::filesystem::is_regular_file(tmp_zip)){ - log::info("Found cached package. Skipping download.", p.title); + log::info("Found cached package for \"{}\".", p.title); } else{ - /* Download all the package files and place them in tmp directory. */ - log::info_n("Downloading \"{}\"...", p.title); - http::context http; - http::response response = http.download_file(p.download_url, tmp_zip); - if(response.code == http::OK){ - log::println("Done."); - }else{ - return log::error_rc(error( - constants::error::HTTP_RESPONSE_ERR, - std::format("HTTP Error: {}", response.code) - )); - } + p_download_urls.emplace_back(p.download_url); + p_storage_paths.emplace_back(tmp_zip); } - - dir_pairs.emplace_back(string_pair(tmp_zip, package_dir + "/")); - - p.is_installed = true; - p.install_path = package_dir; - - /* Extract all the downloaded packages to their appropriate directory location. */ - for(const auto& p : dir_pairs){ - int ec = utils::extract_zip(p.first.c_str(), p.second.c_str()); - if(ec){ - log::error_rc(error( - constants::error::LIBZIP_ERR, - std::format("libzip returned an error code {}", ec) - )); - } - } - - /* Update the cache data with information from */ - log::info_n("Updating local asset data..."); - error error = cache::update_package_info(p_found); - if(error.has_occurred()){ - string prefix = std::format(log::get_error_prefix(), utils::timestamp()); - log::println(GDPM_COLOR_LOG_ERROR"\n{}{}" GDPM_COLOR_RESET, prefix, error.get_message()); - return error; - } - if(config.clean_temporary){ - clean_temporary(config, package_titles); - } - - log::println("Done."); - // }) - // ); } - return error(); + /* Make sure the number of urls matches storage locations */ + if(p_download_urls.size() != p_storage_paths.size()){ + return log::error_rc(error(ec::ASSERTION_FAILED, + "package::install(): p_left.size() != p_storage.size()" + )); + } + + /* Attempt to download ZIPs in parallel */ + if(config.jobs > 1){ + http::multi http(config.jobs); + ptr transfers = http.make_downloads(p_download_urls, p_storage_paths); + ptr responses = http.execute(std::move(transfers)); + + /* Check for HTTP response errors */ + for(const auto& r : *responses){ + if(r.code != http::OK){ + log::error(error(ec::HTTP_RESPONSE_ERR, + std::format("HTTP error: {}", r.code) + )); + } + } + } + else{ + http::context http; + for(size_t i = 0; i < p_download_urls.size(); i++){ + http::response r = http.download_file( + p_download_urls[i], + p_storage_paths[i] + ); + if(r.code != http::OK){ + log::error(error(ec::HTTP_RESPONSE_ERR, + std::format("HTTP error: {}", r.code) + )); + } + } + } + + /* Extract all the downloaded packages to their appropriate directory location. */ + for(const auto& p : target_extract_dirs){ + error error = utils::extract_zip(p.first.c_str(), p.second.c_str()); + if(error.has_occurred()){ + return error; + } + log::println("Done."); + } + + /* Update the cache data */ + for(auto& p : p_cache){ + p.is_installed = true; + p.install_path = config.packages_dir + "/" + p.title; + } + + log::info_n("Updating local asset data..."); + error error = cache::update_package_info(p_cache); + if(error.has_occurred()){ + string prefix = std::format(log::get_error_prefix(), utils::timestamp()); + log::println(GDPM_COLOR_LOG_ERROR"\n{}{}" GDPM_COLOR_RESET, prefix, error.get_message()); + return error; + } + if(config.clean_temporary){ + clean(config, package_titles); + } + log::println("Done."); + + return error; } @@ -233,18 +269,16 @@ namespace gdpm::package{ /* Check if provided param is in remote sources*/ if(!config.remote_sources.contains(params.remote_source)){ - error error( - constants::error::NOT_FOUND, + return log::error_rc(ec::NOT_FOUND, "Remote source not found in config." ); - log::error(error); - return error; } /* Install the other packages from remte source. */ std::vector dir_pairs; task_list tasks; rest_api::request_params rest_api_params = rest_api::make_from_config(config); + package::info_list p_left; for(auto& p : p_found){ // TODO: Execute each in parallel using coroutines?? /* Check if a remote source was provided. If not, then try to get packages @@ -299,20 +333,36 @@ namespace gdpm::package{ if(std::filesystem::exists(tmp_zip) && std::filesystem::is_regular_file(tmp_zip)){ log::info("Found cached package. Skipping download.", p.title); } - else{ - /* Download all the package files and place them in tmp directory. */ - log::info_n("Downloading \"{}\"...", p.title); - http::context http; - http::response response = http.download_file(p.download_url, tmp_zip); - if(response.code == http::OK){ - log::println("Done."); - }else{ - return log::error_rc(error( - constants::error::HTTP_RESPONSE_ERR, - std::format("HTTP Error: {}", response.code) - )); - } + else { + p_left.emplace_back(p); } + } // for loop + + /* Get the packages not found in cache and download */ + string_list urls; + for(const auto& p : p_left){ + urls.emplace_back(p.download_url); + } + http::multi http; + ptr transfers = http.make_requests(urls); + ptr responses = http.execute(std::move(transfers)); + + for(const auto& response : *responses){ + if(response.code == http::OK){ + log::println("Done."); + }else{ + return log::error_rc(error( + constants::error::HTTP_RESPONSE_ERR, + std::format("HTTP Error: {}", response.code) + )); + } + } + + /* Extract all packages and update cache database */ + for(auto& p : p_found){ + string package_dir = std::filesystem::current_path().string() + "/" + p.title;//config.packages_dir + "/" + p.title; + string tmp_dir = std::filesystem::current_path().string() + "/" + p.title + ".tmp"; + string tmp_zip = tmp_dir + ".zip"; dir_pairs.emplace_back(string_pair(tmp_zip, package_dir + "/")); @@ -321,13 +371,7 @@ namespace gdpm::package{ /* Extract all the downloaded packages to their appropriate directory location. */ for(const auto& p : dir_pairs){ - int ec = utils::extract_zip(p.first.c_str(), p.second.c_str()); - if(ec){ - log::error_rc(error( - constants::error::LIBZIP_ERR, - std::format("libzip returned an error code {}", ec) - )); - } + error error = utils::extract_zip(p.first.c_str(), p.second.c_str()); } /* Remove temporary download archive */ @@ -357,12 +401,9 @@ namespace gdpm::package{ result_t result = cache::get_package_info_by_title(package_titles); package::info_list p_cache = result.unwrap_unsafe(); if(p_cache.empty()){ - error error( - constants::error::NOT_FOUND, + return log::error_rc(ec::NOT_FOUND, "Could not find any packages to remove." ); - log::error(error); - return error; } /* Count number packages in cache flagged as is_installed. If there are none, then there's nothing to do. */ @@ -372,19 +413,36 @@ namespace gdpm::package{ }); if(p_count == 0){ - error error( - constants::error::NOT_FOUND, + return log::error_rc(ec::NOT_FOUND, "No packages to remove." ); - log::error(error); - return error; } - log::println("Packages to remove:"); - for(const auto& p : p_cache) - if(p.is_installed) - log::print(" {} ", p.title); - log::println(""); + { + using namespace tabulate; + Table table; + table.format() + .border_top("") + .border_bottom("") + .border_left("") + .border_right("") + .corner("") + .padding_top(0) + .padding_bottom(0); + table.add_row({"Title", "Author", "Category", "Version", "Godot", "Last Modified", "Installed?"}); + table[0].format() + .font_style({FontStyle::underline, FontStyle::bold}); + for(const auto& p : p_cache){ + table.add_row({p.title, p.author, p.category, p.version, p.godot_version, p.modify_date, (p.is_installed) ? "✔️": "❌"}); + // string output(p.title + GDPM_COLOR_CYAN " v" + p.version + GDPM_COLOR_RESET); + // output += GDPM_COLOR_BLUE " last updated: " + p.modify_date + GDPM_COLOR_RESET; + // output += (p.is_installed) ? GDPM_COLOR_LIGHT_CYAN " (reinstall)" : ""; + // output += GDPM_COLOR_RESET; + // log::print(" {}\n", output); + } + table.print(std::cout); + log::println(""); + } if(!config.skip_prompt){ if(!utils::prompt_user_yn("Do you want to remove these packages? (Y/n)")) @@ -428,13 +486,13 @@ namespace gdpm::package{ } log::println("Done."); if(config.clean_temporary){ - clean_temporary(config, package_titles); + clean(config, package_titles); } log::info_n("Updating local asset data..."); { error error = cache::update_package_info(p_cache); if(error.has_occurred()){ - log::error("\n{}", error.get_message()); + log::error("\nsqlite: {}", error.get_message()); return error; } } @@ -473,9 +531,9 @@ namespace gdpm::package{ string url{constants::HostUrl + rest_api::endpoints::GET_AssetId}; Document doc = rest_api::get_assets_list(url, rest_api_params); if(doc.IsNull()){ - constexpr const char *message = "Could not get response from server. Aborting."; - log::error(message); - return error(constants::error::HOST_UNREACHABLE, message); + return log::error_rc(ec::HOST_UNREACHABLE, + "package::update(): could not get response from server. Aborting." + ); } return error(); } @@ -568,10 +626,9 @@ namespace gdpm::package{ remote::print_repositories(config); } else{ - log::error(error( - constants::error::UNKNOWN_COMMAND, - "Unrecognized subcommand. Try either 'packages' or 'remote' instead." - )); + log::error_rc(ec::UNKNOWN_COMMAND, + "package::list(): unrecognized subcommand...try either 'packages' or 'remote' instead." + ); } return error(); } @@ -598,9 +655,9 @@ namespace gdpm::package{ } else { - constexpr const char *message = "File or directory exists!"; - log::error(message); - return error(constants::error::FILE_EXISTS, message); + return log::error_rc(ec::FILE_EXISTS, + "package::export_to(): file or directory exists!" + ); } } std::ofstream of(path); @@ -613,6 +670,45 @@ namespace gdpm::package{ } + error clean( + const config::context& config, + const title_list& package_titles + ){ + if(package_titles.empty()){ + if(!config.skip_prompt){ + if(!utils::prompt_user_yn("Are you sure you want to clean all temporary files? (Y/n)")){ + return error(); + } + } + std::filesystem::remove_all(config.tmp_dir); + return error(); + } + + /* Find the path of each packages is_installed then delete temporaries */ + log::info_n("Cleaning temporary files..."); + for(const auto& p_title : package_titles){ + string tmp_zip = config.tmp_dir + "/" + p_title + ".zip"; + if(config.verbose > 0) + log::info("Removed '{}'", tmp_zip); + std::filesystem::remove_all(tmp_zip); + } + log::println("Done."); + return error(); + } + + + error purge(const config::context& config){ + if(!config.skip_prompt){ + if(!utils::prompt_user_yn("Are you sure you want to purge all installed packages? (Y/n)")){ + return error(); + } + } + /* Remove all packages installed in global location */ + std::filesystem::remove_all(config.packages_dir); + return cache::drop_package_database(); + } + + error link( const config::context& config, const title_list& package_titles, @@ -622,7 +718,7 @@ namespace gdpm::package{ if(params.paths.empty()){ return log::error_rc(error( - constants::error::PATH_NOT_DEFINED, + constants::error::MALFORMED_PATH, "Path is required" )); } @@ -685,7 +781,7 @@ namespace gdpm::package{ if(params.paths.empty()){ return log::error_rc(error( - constants::error::PATH_NOT_DEFINED, + constants::error::MALFORMED_PATH, "Path is required" )); } @@ -739,6 +835,88 @@ namespace gdpm::package{ return error(); } + + result_t fetch( + const config::context& config, + const title_list& package_titles + ){ + using namespace rapidjson; + + rest_api::request_params rest_api_params = rest_api::make_from_config(config); + rest_api_params.page = 0; + int page = 0; + int page_length = 0; + int total_items = 0; + int items_left = 0; + // int total_pages = 0; + + log::info_n("Sychronizing database..."); + do{ + /* Make the GET request to get page data and store it in the local + package database. Also, check to see if we need to keep going. */ + string url{constants::HostUrl + rest_api::endpoints::GET_Asset}; + Document doc = rest_api::get_assets_list(url, rest_api_params); + rest_api_params.page += 1; + + if(doc.IsNull()){ + log::println(""); + return result_t(info_list(), log::error_rc( + ec::EMPTY_RESPONSE, + "Could not get response from server. Aborting." + )); + } + + /* Need to know how many pages left to get and how many we get per + request. */ + page = doc["page"].GetInt(); + page_length = doc["page_length"].GetInt(); + // total_pages = doc["pages"].GetInt(); + total_items = doc["total_items"].GetInt(); + items_left = total_items - (page + 1) * page_length; + + // log::info("page: {}, page length: {}, total pages: {}, total items: {}, items left: {}", page, page_length, total_pages, total_items, items_left); + + if(page == 0){ + error error; + error = cache::drop_package_database(); + error = cache::create_package_database(); + } + + info_list packages; + for(const auto& o : doc["result"].GetArray()){ + // log::println("======================="); + info p{ + .asset_id = std::stoul(o["asset_id"].GetString()), + .title = o["title"].GetString(), + .author = o["author"].GetString(), + .author_id = std::stoul(o["author_id"].GetString()), + .version = o["version"].GetString(), + .godot_version = o["godot_version"].GetString(), + .cost = o["cost"].GetString(), + .modify_date = o["modify_date"].GetString(), + .category = o["category"].GetString(), + .remote_source = url + }; + packages.emplace_back(p); + } + error error = cache::insert_package_info(packages); + if (error.has_occurred()){ + log::error(error); + /* FIXME: Should this stop here or keep going? */ + } + /* Make the same request again to get the rest of the needed data + using the same request, but with a different page, then update + variables as needed. */ + + + } while(items_left > 0); + + log::println("Done."); + + return cache::get_package_info_by_title(package_titles); + } + + void print_list(const info_list& packages){ for(const auto& p : packages){ log::println( @@ -856,26 +1034,6 @@ namespace gdpm::package{ } - void clean_temporary( - const config::context& config, - const title_list& package_titles - ){ - if(package_titles.empty()){ - log::info("No temporary files found to clean."); - std::filesystem::remove_all(config.tmp_dir); - } - /* Find the path of each packages is_installed then delete temporaries */ - log::info_n("Cleaning temporary files..."); - for(const auto& p_title : package_titles){ - string tmp_zip = config.tmp_dir + "/" + p_title + ".zip"; - if(config.verbose > 0) - log::info("Removed '{}'", tmp_zip); - std::filesystem::remove_all(tmp_zip); - } - log::println("Done."); - } - - template auto set_if_key_exists( const var_opts& o, @@ -886,89 +1044,6 @@ namespace gdpm::package{ } - result_t synchronize_database( - const config::context& config, - const title_list& package_titles - ){ - using namespace rapidjson; - - rest_api::request_params rest_api_params = rest_api::make_from_config(config); - rest_api_params.page = 0; - int page = 0; - int page_length = 0; - int total_items = 0; - int items_left = 0; - // int total_pages = 0; - - log::info("Sychronizing database..."); - do{ - /* Make the GET request to get page data and store it in the local - package database. Also, check to see if we need to keep going. */ - std::string url{constants::HostUrl}; - url += rest_api::endpoints::GET_Asset; - Document doc = rest_api::get_assets_list(url, rest_api_params); - rest_api_params.page += 1; - - if(doc.IsNull()){ - error error( - constants::error::EMPTY_RESPONSE, - "Could not get response from server. Aborting." - ); - log::error(error); - return result_t(info_list(), error); - } - - /* Need to know how many pages left to get and how many we get per - request. */ - page = doc["page"].GetInt(); - page_length = doc["page_length"].GetInt(); - // total_pages = doc["pages"].GetInt(); - total_items = doc["total_items"].GetInt(); - items_left = total_items - (page + 1) * page_length; - - // log::info("page: {}, page length: {}, total pages: {}, total items: {}, items left: {}", page, page_length, total_pages, total_items, items_left); - - if(page == 0){ - error error; - error = cache::drop_package_database(); - error = cache::create_package_database(); - } - - info_list packages; - for(const auto& o : doc["result"].GetArray()){ - // log::println("======================="); - info p{ - .asset_id = std::stoul(o["asset_id"].GetString()), - .title = o["title"].GetString(), - .author = o["author"].GetString(), - .author_id = std::stoul(o["author_id"].GetString()), - .version = o["version"].GetString(), - .godot_version = o["godot_version"].GetString(), - .cost = o["cost"].GetString(), - .modify_date = o["modify_date"].GetString(), - .category = o["category"].GetString(), - .remote_source = url - }; - packages.emplace_back(p); - } - error error = cache::insert_package_info(packages); - if (error.has_occurred()){ - log::error(error); - /* FIXME: Should this stop here or keep going? */ - } - /* Make the same request again to get the rest of the needed data - using the same request, but with a different page, then update - variables as needed. */ - - - } while(items_left > 0); - - log::println("Done."); - - return cache::get_package_info_by_title(package_titles); - } - - result_t resolve_dependencies( const config::context& config, const title_list& package_titles diff --git a/src/package_manager.cpp b/src/package_manager.cpp index 48167be..e1fe56f 100644 --- a/src/package_manager.cpp +++ b/src/package_manager.cpp @@ -111,6 +111,7 @@ namespace gdpm::package_manager{ ArgumentParser update_command("update"); ArgumentParser search_command("search"); ArgumentParser export_command("export"); + ArgumentParser purge_command("purge"); ArgumentParser list_command("list"); ArgumentParser link_command("link"); ArgumentParser clone_command("clone"); @@ -150,16 +151,16 @@ namespace gdpm::package_manager{ .implicit_value(true) .default_value(false) .nargs(0); - install_command.add_argument("--disable-sync") + install_command.add_argument("--sync") .help("enable syncing with remote before installing") .implicit_value(true) - .default_value(false) - .nargs(0); - install_command.add_argument("--disable-cache") + .default_value(true) + .nargs(nargs_pattern::any); + install_command.add_argument("--cache") .help("disable caching asset data") .implicit_value(true) .default_value(false) - .nargs(0); + .nargs(nargs_pattern::any); install_command.add_argument("--remote") .help("set the remote to use") .nargs(1); @@ -200,17 +201,25 @@ namespace gdpm::package_manager{ remove_command.add_description("remove package(s)"); remove_command.add_argument("packages") .nargs(nargs_pattern::any); - remove_command.add_argument("--clean"); + remove_command.add_argument("--clean") + .help("clean temporary files") + .implicit_value(true) + .default_value(false) + .nargs(0); remove_command.add_argument("-y", "--skip-prompt"); remove_command.add_argument("-f", "--file") .help("set the file(s) to read as input") .append() - .nargs(nargs_pattern::at_least_one); + .nargs(1); update_command.add_description("update package(s)"); update_command.add_argument("packages") .nargs(nargs_pattern::any); - update_command.add_argument("--clean"); + update_command.add_argument("--clean") + .help("clean temporary files") + .implicit_value(true) + .default_value(false) + .nargs(0); update_command.add_argument("--remote"); update_command.add_argument("-f", "--file") .help("set the file(s) to read as input") @@ -301,8 +310,19 @@ namespace gdpm::package_manager{ clean_command.add_description("clean package(s) temporary files"); clean_command.add_argument("packages") .help("package(s) to clean") - .required() - .nargs(nargs_pattern::at_least_one); + .nargs(nargs_pattern::any); + clean_command.add_argument("-y", "--skip-prompt") + .help("skip the yes/no prompt") + .implicit_value(true) + .default_value(false) + .nargs(0); + + purge_command.add_description("purge cache database"); + purge_command.add_argument("-y", "--skip-prompt") + .help("skip the yes/no prompt") + .implicit_value(true) + .default_value(false) + .nargs(0); fetch_command.add_description("fetch and sync asset data"); fetch_command.add_argument("remote") @@ -362,6 +382,7 @@ namespace gdpm::package_manager{ program.add_subparser(update_command); program.add_subparser(search_command); program.add_subparser(export_command); + program.add_subparser(purge_command); program.add_subparser(list_command); program.add_subparser(link_command); program.add_subparser(clone_command); @@ -377,25 +398,41 @@ namespace gdpm::package_manager{ program.parse_args(argc, argv); // program.parse_known_args(argc, argv); } catch(const std::runtime_error& e){ - return log::error_rc(error( - constants::error::ARGPARSE_ERROR, - e.what()) - ); + return log::error_rc(ec::ARGPARSE_ERROR, e.what()); } if(program.is_subcommand_used(install_command)){ action = action_e::install; - if(install_command.is_used("packages")) - package_titles = install_command.get("packages"); + // if(install_command.is_used("packages")) + // package_titles = install_command.get("packages"); + package_titles = get_values_from_parser(install_command); set_if_used(install_command, config.rest_api_params.godot_version, "godot-version"); set_if_used(install_command, config.clean_temporary, "clean"); - set_if_used(install_command, config.enable_sync, "disable-sync"); - set_if_used(install_command, config.enable_cache, "disable-cache"); + // set_if_used(install_command, config.enable_sync, "disable-sync"); + // set_if_used(install_command, config.enable_cache, "disable-cache"); set_if_used(install_command, params.remote_source, "remote"); set_if_used(install_command, config.jobs, "jobs"); set_if_used(install_command, config.skip_prompt, "skip-prompt"); set_if_used(install_command, params.input_files, "file"); set_if_used(install_command, config.timeout, "timeout"); + if(install_command.is_used("sync")){ + string sync = install_command.get("sync"); + if(!sync.compare("enable") || !sync.compare("true") || sync.empty()){ + config.enable_sync = true; + } + else if(!sync.compare("disable") || !sync.compare("false")){ + config.enable_sync = false; + } + } + if(install_command.is_used("cache")){ + string cache = install_command.get("sync"); + if(!cache.compare("enable") || !cache.compare("true") || cache.empty()){ + config.enable_sync = true; + } + else if(!cache.compare("disable") || !cache.compare("false")){ + config.enable_sync = false; + } + } } else if(program.is_subcommand_used(get_command)){ action = action_e::get; @@ -437,6 +474,15 @@ namespace gdpm::package_manager{ action = action_e::p_export; params.paths = export_command.get("paths"); } + else if(program.is_subcommand_used(clean_command)){ + action = action_e::clean; + package_titles = get_values_from_parser(clean_command); + set_if_used(clean_command, config.skip_prompt, "skip-prompt"); + } + else if(program.is_subcommand_used(purge_command)){ + action = action_e::purge; + set_if_used(purge_command, config.skip_prompt, "skip-prompt"); + } else if(program.is_subcommand_used(list_command)){ action = action_e::list; if(list_command.is_used("show")) @@ -471,10 +517,6 @@ namespace gdpm::package_manager{ params.paths = clone_command.get("path"); } } - else if(program.is_subcommand_used(clean_command)){ - action = action_e::clean; - package_titles = get_values_from_parser(clean_command); - } else if(program.is_subcommand_used(config_command)){ if(config_command.is_used("style")){ string style = config_command.get("style"); @@ -550,15 +592,16 @@ namespace gdpm::package_manager{ case action_e::update: package::update(config, package_titles, params); break; case action_e::search: package::search(config, package_titles, params); break; case action_e::p_export: package::export_to(params.paths); break; + case action_e::purge: package::purge(config); break; case action_e::list: package::list(config, params); break; /* ...opts are the paths here */ case action_e::link: package::link(config, package_titles, params); break; case action_e::clone: package::clone(config, package_titles, params); break; - case action_e::clean: package::clean_temporary(config, package_titles); break; + case action_e::clean: package::clean(config, package_titles); break; case action_e::config_get: config::print_properties(config, params.args); break; case action_e::config_set: config::set_property(config, params.args[0], params.args[1]); break; - case action_e::fetch: package::synchronize_database(config, package_titles); break; - case action_e::sync: package::synchronize_database(config, package_titles); break; + case action_e::fetch: package::fetch(config, package_titles); break; + case action_e::sync: package::fetch(config, package_titles); break; case action_e::remote_list: remote::print_repositories(config); break; case action_e::remote_add: remote::add_repository(config, params.args); break; case action_e::remote_remove: remote::remove_respositories(config, params.args); break; diff --git a/src/rest_api.cpp b/src/rest_api.cpp index 11ca39f..6a76f74 100644 --- a/src/rest_api.cpp +++ b/src/rest_api.cpp @@ -119,7 +119,10 @@ namespace gdpm::rest_api{ return request_url; } - error print_params(const request_params& params, const string& filter){ + error print_params( + const request_params& params, + const string& filter + ){ log::println("params: \n" "\ttype: {}\n" "\tcategory: {}\n" @@ -147,7 +150,7 @@ namespace gdpm::rest_api{ Document doc = rest_api::get_assets_list(request_url, rest_api_params); if(doc.IsNull()){ return log::error_rc(error( - constants::error::HOST_UNREACHABLE, + ec::HOST_UNREACHABLE, "Could not fetch metadata. Aborting." )); } @@ -166,9 +169,9 @@ namespace gdpm::rest_api{ http::context http; string request_url{url}; request_url += to_string(type); - http::response r = http.request_get(url); + http::response r = http.request(url); if(verbose > 0) - log::info("URL: {}", url); + log::info("rest_api::configure::url: {}", url); return _parse_json(r.body); } @@ -179,15 +182,15 @@ namespace gdpm::rest_api{ const string& filter ){ http::context http; - http::request_params http_params; - http_params.headers.insert(http::header("Accept", "*/*")); - http_params.headers.insert(http::header("Accept-Encoding", "application/gzip")); - http_params.headers.insert(http::header("Content-Encoding", "application/gzip")); - http_params.headers.insert(http::header("Connection", "keep-alive")); - string request_url = _prepare_request(url, c, http.url_escape(filter)); - http::response r = http.request_get(request_url, http_params); + http::request params; + params.headers.insert(http::header("Accept", "*/*")); + params.headers.insert(http::header("Accept-Encoding", "application/gzip")); + params.headers.insert(http::header("Content-Encoding", "application/gzip")); + params.headers.insert(http::header("Connection", "keep-alive")); + string prepared_url = _prepare_request(url, c, http.url_escape(filter)); + http::response r = http.request(prepared_url, params); if(c.verbose >= log::INFO) - log::info("rest_api::get_asset_list()::URL: {}", request_url); + log::info("rest_api::get_asset_list()::url: {}", prepared_url); return _parse_json(r.body, c.verbose); } @@ -200,16 +203,18 @@ namespace gdpm::rest_api{ ){ /* Set up HTTP request */ http::context http; - http::request_params http_params; - http_params.headers.insert(http::header("Accept", "*/*")); - http_params.headers.insert(http::header("Accept-Encoding", "application/gzip")); - http_params.headers.insert(http::header("Content-Encoding", "application/gzip")); - http_params.headers.insert(http::header("Connection", "keep-alive")); - string request_url = utils::replace_all(_prepare_request(url, api_params, http.url_escape(filter)), "{id}", std::to_string(asset_id)); - http::response r = http.request_get(request_url.c_str(), http_params); + http::request params; + params.headers.insert(http::header("Accept", "*/*")); + params.headers.insert(http::header("Accept-Encoding", "application/gzip")); + params.headers.insert(http::header("Content-Encoding", "application/gzip")); + params.headers.insert(http::header("Connection", "keep-alive")); + string prepared_url = utils::replace_all( + _prepare_request(url, api_params, + http.url_escape(filter) + ), "{id}", std::to_string(asset_id)); + http::response r = http.request(prepared_url, params); if(api_params.verbose >= log::INFO) - log::info("get_asset().URL: {}", request_url); - + log::info("rest_api::get_asset()::url: {}", prepared_url); return _parse_json(r.body); } @@ -228,27 +233,56 @@ namespace gdpm::rest_api{ return false; } + + namespace multi{ + json::documents get_assets( + const string_list& urls, + id_list asset_ids, + const request_params& api_params, + const string_list& filters + ){ + if(urls.size() == asset_ids.size() && urls.size() == filters.size()){ + log::error(ec::ASSERTION_FAILED, + "multi::get_assets(): urls.size() != filters.size()" + ); + } + http::multi http; + http::request params; + json::documents docs; + params.headers.insert(http::header("Accept", "*/*")); + params.headers.insert(http::header("Accept-Encoding", "application/gzip")); + params.headers.insert(http::header("Content-Encoding", "application/gzip")); + params.headers.insert(http::header("Connection", "keep-alive")); + string_list prepared_urls = {}; + + /* Prepare the URLs for the request_multi() call z*/ + for(size_t i = 0; i < urls.size(); i++){ + const string& url = urls.at(i); + const string& filter = filters.at(i); + int asset_id = asset_ids.at(i); + string prepared_url = utils::replace_all( + _prepare_request(url, api_params, http.url_escape(filter)), + "{id}", std::to_string(asset_id)); + prepared_urls.emplace_back(prepared_url); + if(api_params.verbose >= log::INFO) + log::info("get_assets(i={})::url: {}", i, prepared_url); + } + + /* Parse JSON string into objects */ + ptr transfers = http.make_requests(prepared_urls, params); + ptr responses = http.execute(std::move(transfers)); + for(const auto& response : *responses){ + docs.emplace_back(_parse_json(response.body)); + } + return docs; + } + } + namespace edits{ - - void edit_asset(){ - - } - - void get_asset_edit(int asset_id){ - - } - - string review_asset_edit(int asset_id){ - return string(); - } - - string accept_asset_edit(int asset_id){ - return string(); - } - - string reject_asset_edit(int asset_id){ - return string(); - } - + void edit_asset(){} + void get_asset_edit(int asset_id){} + string review_asset_edit(int asset_id){ return string(); } + string accept_asset_edit(int asset_id){ return string(); } + string reject_asset_edit(int asset_id){ return string(); } } // namespace edits } \ No newline at end of file diff --git a/src/utils.cpp b/src/utils.cpp index 2b1a863..b71c2c2 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -2,11 +2,9 @@ #include "utils.hpp" #include "config.hpp" #include "constants.hpp" +#include "error.hpp" #include "log.hpp" -#include "indicators/indeterminate_progress_bar.hpp" -#include "indicators/dynamic_progress.hpp" -#include "indicators/progress_bar.hpp" -#include "indicators/block_progress_bar.hpp" + #include "csv2/reader.hpp" @@ -30,38 +28,13 @@ namespace gdpm::utils{ - using namespace indicators; - BlockProgressBar bar { - option::BarWidth{50}, - // option::Start{"["}, - // option::Fill{"="}, - // option::Lead{">"}, - // option::Remainder{" "}, - // option::End{"]"}, - option::PrefixText{"Downloading file "}, - option::PostfixText{""}, - option::ForegroundColor{Color::green}, - option::FontStyles{std::vector{FontStyle::bold}}, - }; - // option::ShowElapsedTime{true}, - // option::ShowRemainingTime{true}, - IndeterminateProgressBar bar_unknown { - option::BarWidth{50}, - option::Start{"["}, - option::Fill{"."}, - option::Lead{"<==>"}, - option::PrefixText{"Downloading file "}, - option::End{"]"}, - option::PostfixText{""}, - option::ForegroundColor{Color::green}, - option::FontStyles{std::vector{FontStyle::bold}}, - }; + - bool to_bool(const std::string& s){ + bool to_bool(const string& s){ return to_lower(s) == "true"; } - std::vector split_lines(const std::string& contents){ + std::vector split_lines(const string& contents){ using namespace csv2; csv2::Reader< delimiter<'\n'>, @@ -69,7 +42,7 @@ namespace gdpm::utils{ first_row_is_header, trim_policy::trim_whitespace > csv; - std::vector lines; + std::vector lines; if(csv.parse(contents)){ for(const auto& row : csv){ for(const auto& cell : row){ @@ -82,13 +55,13 @@ namespace gdpm::utils{ #if (GDPM_READFILE_IMPL == 0) - std::string readfile(const std::string& path){ + string readfile(const string& path){ constexpr auto read_size = std::size_t{4096}; auto stream = std::ifstream{path.data()}; stream.exceptions(std::ios_base::badbit); - auto out = std::string{}; - auto buf = std::string(read_size, '\0'); + auto out = string{}; + auto buf = string(read_size, '\0'); while (stream.read(& buf[0], read_size)) { out.append(buf, 0, stream.gcount()); } @@ -97,15 +70,15 @@ namespace gdpm::utils{ } #elif(GDPM_READFILE_IMPL == 1) - std::string readfile(const std::string& path){ + string readfile(const string& path){ std::ifstream ifs(path); - return std::string( + return string( (std::istreambuf_iterator(ifs)), (std::istreambuf_iterator()) ); } #elif(GDPM_READFILE_IMPL == 2) - std::string readfile(const std::string& path){ + string readfile(const string& path){ std::ifstream ifs(path); std::stringstream buffer; buffer << ifs.rdbuf(); @@ -113,43 +86,43 @@ namespace gdpm::utils{ } #endif - std::string to_lower(const std::string& s){ - std::string copy = s; + string to_lower(const string& s){ + string copy = s; std::transform(copy.begin(), copy.end(), copy.begin(), tolower); return copy; } - std::string trim(const std::string& s){ + string trim(const string& s){ return trim_right(trim_left(s)); } - std::string trim_left(const std::string& s){ + string trim_left(const string& s){ return trim_left(s, constants::WHITESPACE); } - std::string trim_left( - const std::string& s, - const std::string& ref + string trim_left( + const string& s, + const string& ref ){ size_t start = s.find_first_not_of(ref); - return (start == std::string::npos) ? "" : s.substr(start); + return (start == string::npos) ? "" : s.substr(start); } - std::string trim_right(const std::string& s){ + string trim_right(const string& s){ return trim_right(s, constants::WHITESPACE); } - std::string trim_right( - const std::string& s, - const std::string& ref + string trim_right( + const string& s, + const string& ref ){ size_t end = s.find_last_not_of(ref); - return (end == std::string::npos) ? "" : s.substr(0, end + 1); + return (end == string::npos) ? "" : s.substr(0, end + 1); } - std::vector parse_lines(const std::string &s){ - std::string line; - std::vector result; + std::vector parse_lines(const string &s){ + string line; + std::vector result; std::stringstream ss(s); while(std::getline(ss, line)){ result.emplace_back(line); @@ -157,26 +130,26 @@ namespace gdpm::utils{ return result; } - std::string replace_first( - const std::string &s, - const std::string &from, - const std::string &to + string replace_first( + const string &s, + const string &from, + const string &to ){ - std::string copy = s; // make string copy + string copy = s; // make string copy size_t pos = copy.find(from); - if(pos == std::string::npos) + if(pos == string::npos) return copy; return copy.replace(pos, from.length(), to); } - std::string replace_all( - const std::string& s, - const std::string& from, - const std::string& to + string replace_all( + const string& s, + const string& from, + const string& to ){ - std::string copy = s; // make string copy + string copy = s; // make string copy size_t pos = 0; - while((pos = copy.find(from, pos)) != std::string::npos){ + while((pos = copy.find(from, pos)) != string::npos){ copy.replace(pos, s.length(), to); pos += to.length(); } @@ -184,45 +157,47 @@ namespace gdpm::utils{ } /* Ref: https://gist.github.com/mobius/1759816 */ - int extract_zip( + error extract_zip( const char *archive, const char *dest, int verbose ){ - constexpr const char *prog = "gpdm"; - struct zip *zip; + constexpr int SIZE = 1024; + struct zip *za; struct zip_file *zf; struct zip_stat sb; - char buf[100]; + char buf[SIZE]; int err; int i, len, fd; zip_uint64_t sum; - // log::info_n("Extracting package contents to '{}'...", dest); - log::info_n("Extracting package contents..."); - if((zip = zip_open(archive, 0, &err)) == nullptr){ + std::filesystem::path path(archive); + log::info_n("Extracting \"{}\" archive...", path.filename().string()); + if((za = zip_open(path.c_str(), ZIP_RDONLY, &err)) == NULL){ zip_error_to_str(buf, sizeof(buf), err, errno); - log::error("{}: can't open zip archive {}: {}", prog, archive, buf); - return 1; + log::println(""); + return log::error_rc(error( + ec::LIBZIP_ERR, + std::format("utils::extract_zip(): can't open zip archive \"{}\": {}", path.filename().string(), buf)) + ); } - for(i = 0; i < zip_get_num_entries(zip, 0); i++){ - if(zip_stat_index(zip, i, 0, &sb) == 0){ + for(i = 0; i < zip_get_num_entries(za, 0); i++){ + if(zip_stat_index(za, i, 0, &sb) == 0){ len = strlen(sb.name); if(verbose > 1){ - log::print("{}, ", sb.name); - log::println("size: {}, ", sb.size); + log::println("utils::extract_zip(): {}, size: {}", sb.name, sb.size); } - std::string path{dest}; + string path{dest}; path += sb.name; if(sb.name[len-1] == '/'){ - // safe_create_dir(sb.name); std::filesystem::create_directory(path); } else { - zf = zip_fopen_index(zip, i, 0); + zf = zip_fopen_index(za, i, 0); if(!zf){ - log::error("extract_zip: zip_fopen_index() failed."); - return 100; + return log::error_rc(error(ec::LIBZIP_ERR, + "utils::extract_zip(): zip_fopen_index() failed.") + ); } #ifdef _WIN32 fd = open(sb.name, O_RDWR | O_TRUNC | O_CREAT | O_BINARY, 0644); @@ -230,16 +205,19 @@ namespace gdpm::utils{ fd = open(path.c_str(), O_RDWR | O_TRUNC | O_CREAT, 0644); #endif if(fd < 0){ - log::error("extract_zip: open() failed. (path: {}, fd={})", path, fd); - return 101; + log::error_rc(error(ec::LIBZIP_ERR, + std::format("utils::extract_zip(): open() failed. (path: {}, fd={})", path, fd)) + ); } sum = 0; while(sum != sb.size){ len = zip_fread(zf, buf, 100); if(len < 0){ - log::error("extract_zip: zip_fread() returned len < 0 (len={})", len); - return 102; + return log::error_rc(error( + ec::LIBZIP_ERR, + std::format("utils::extract_zip(): zip_fread() returned len < 0 (len={})", len)) + ); } write(fd, buf, len); sum += len; @@ -252,24 +230,23 @@ namespace gdpm::utils{ } } - if(zip_close(zip) == -1){ - log::error("{}: can't close zip archive '{}'", prog, archive); - return 1; + if(zip_close(za) == -1){ + return log::error_rc(error(ec::LIBZIP_ERR, + std::format("utils::extract_zip: can't close zip archive '{}'", archive)) + ); } - log::println("Done."); - return 0; + return error(); } - std::string prompt_user(const char *message){ + string prompt_user(const char *message){ log::print("{} ", message); - std::string input; - // std::cin >> input; + string input; getline(std::cin, input); return input; } bool prompt_user_yn(const char *message){ - std::string input{""}; + string input{""}; while( input != "y" && input != "n" ){ input = to_lower(utils::prompt_user(message)); bool is_default = (input == "\0" || input == "\n" || input == "\r\n" || input.empty()); @@ -287,15 +264,15 @@ namespace gdpm::utils{ // sleep_until(system_clock::now() + millis); } - std::string join( - const std::vector& target, - const std::string& delimiter + string join( + const std::vector& target, + const string& delimiter ){ - std::string o; + string o; std::for_each( target.begin(), target.end(), - [&o, &delimiter](const std::string& s){ + [&o, &delimiter](const string& s){ o += s + delimiter; } ); @@ -303,16 +280,16 @@ namespace gdpm::utils{ return o; } - std::string join( - const std::unordered_map& target, - const std::string& prefix, - const std::string& delimiter + string join( + const std::unordered_map& target, + const string& prefix, + const string& delimiter ){ - std::string o; + string o; std::for_each( target.begin(), target.end(), - [&o, &prefix, &delimiter](const std::pair& p){ + [&o, &prefix, &delimiter](const std::pair& p){ o += prefix + p.first + ": " + p.second + delimiter; } ); @@ -320,13 +297,13 @@ namespace gdpm::utils{ return o; } - std::string convert_size(long size){ + string convert_size(long size){ int digit = 0; while(size > 1000){ size /= 1000; digit += 1; } - std::string s = std::to_string(size); + string s = std::to_string(size); switch(digit){ case 0: return s + " B"; case 1: return s + " KB"; @@ -338,93 +315,13 @@ namespace gdpm::utils{ return std::to_string(size); } - - namespace curl { - size_t write_to_buffer( - char *contents, - size_t size, - size_t nmemb, - void *userdata - ){ - size_t realsize = size * nmemb; - struct memory_buffer *m = (struct memory_buffer*)userdata; - - m->addr = (char*)realloc(m->addr, m->size + realsize + 1); - if(m->addr == nullptr){ - /* Out of memory */ - fprintf(stderr, "Not enough memory (realloc returned NULL)\n"); - return 0; - } - - memcpy(&(m->addr[m->size]), contents, realsize); - m->size += realsize; - m->addr[m->size] = 0; - - return realsize; - } - - size_t write_to_stream( - char *ptr, - size_t size, - size_t nmemb, - void *userdata - ){ - if(nmemb == 0) - return 0; - - return fwrite(ptr, size, nmemb, (FILE*)userdata); - } - - int show_progress( - void *ptr, - curl_off_t total_download, - curl_off_t current_downloaded, - curl_off_t total_upload, - curl_off_t current_upload - ){ - - if(current_downloaded >= total_download) - return 0; - using namespace indicators; - show_console_cursor(false); - if(total_download != 0){ - // double percent = std::floor((current_downloaded / (total_download)) * 100); - bar.set_option(option::MaxProgress{total_download}); - // bar.set_option(option::HideBarWhenComplete{false}); - bar.set_progress(current_downloaded); - bar.set_option(option::PostfixText{ - convert_size(current_downloaded) + " / " + - convert_size(total_download) - }); - if(bar.is_completed()){ - bar.set_option(option::PrefixText{"Download completed."}); - bar.mark_as_completed(); - } - } else { - if(bar_unknown.is_completed()){ - bar_unknown.set_option(option::PrefixText{"Download completed."}); - bar_unknown.mark_as_completed(); - } else { - bar.tick(); - bar_unknown.set_option( - option::PostfixText(std::format("{}", convert_size(current_downloaded))) - ); - - } - } - show_console_cursor(true); - memory_buffer *m = (memory_buffer*)ptr; - return 0; - } - } - namespace json { - std::string from_array( - const std::set& a, - const std::string& prefix + string from_array( + const std::set& a, + const string& prefix ){ - std::string o{"["}; - for(const std::string& src : a) + string o{"["}; + for(const string& src : a) o += prefix + "\t\"" + src + "\","; if(o.back() == ',') o.pop_back(); @@ -433,14 +330,14 @@ namespace gdpm::utils{ }; - std::string from_object( - const std::unordered_map& m, - const std::string& prefix, - const std::string& spaces + string from_object( + const std::unordered_map& m, + const string& prefix, + const string& spaces ){ - std::string o{"{"}; + string o{"{"}; std::for_each(m.begin(), m.end(), - [&o, &prefix, &spaces](const std::pair& p){ + [&o, &prefix, &spaces](const std::pair& p){ o += std::format("{}\t\"{}\":{}\"{}\",", prefix, p.first, spaces, p.second); } );