Implemented parallel downloads through CURL multi interface and added purge command

- Update `README.md` file with examples
- Fixed error messages showing no or wrong message
- Changed the prompt message when installing, removing, etc.
- Changed how http::request works
- Added `http::multi` class for parallel downloads
- Removed separate `concepts.hpp` file
TODO: Fix ZIP not extracting after running the `install` command
This commit is contained in:
David Allen 2023-07-10 20:26:15 -06:00
parent 766eabd5b2
commit 807aa8e5b2
21 changed files with 1158 additions and 758 deletions

View file

@ -30,7 +30,7 @@ find_package(SQLiteCpp CONFIG REQUIRED)
set(CMAKE_CXX_COMPILER "clang++") set(CMAKE_CXX_COMPILER "clang++")
set(CMAKE_BUILD_RPATH "build/cmake") set(CMAKE_BUILD_RPATH "build/cmake")
set(CMAKE_CXX_FLAGS set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} -std=c++20 -Ofast -fPIC -fPIE -fpermissive -Wall -Wno-switch -Wno-unused-variable -Wno-unused-function -Wno-sign-conversion -Wno-deprecated-declarations -pedantic-errors" "${CMAKE_CXX_FLAGS} -std=c++20 -g -Ofast -fPIC -fPIE -fpermissive -Wall -Wno-switch -Wno-unused-variable -Wno-unused-function -Wno-sign-conversion -Wno-deprecated-declarations -pedantic-errors"
) )
set(INCLUDE_DIRS set(INCLUDE_DIRS
"include" "include"

View file

@ -324,11 +324,11 @@ gdpm config set username towk
## Planned Features ## Planned Features
- [x] Godot 4 Asset Library compatibility. - [x] Compatible with Godot 4
- [ ] PKGBUILD for ArchLinux/Manjaro - [x] Parallel downloading.
- [ ] Multithreaded downloading. - [ ] PKGBUILD for ArchLinux/Manjaro.
- [ ] Proper updating. - [ ] Proper updating.
@ -340,6 +340,8 @@ gdpm config set username towk
- [ ] Adapted to new Asset API. - [ ] Adapted to new Asset API.
- [ ] User interface.
- [ ] Experimental dependency management. There is no way of handling dependencies using the Godot Asset API. This is a [hot topic](https://github.com/godotengine/godot-proposals/issues/142) and might change in the near future. - [ ] Experimental dependency management. There is no way of handling dependencies using the Godot Asset API. This is a [hot topic](https://github.com/godotengine/godot-proposals/issues/142) and might change in the near future.
## Known Issues ## Known Issues

View file

@ -1,15 +1,71 @@
# GDPM HTTP Example # GDPM Rest API Example
This is an example showing how to use the GDPM HTTP library to download files. The library uses RapidJSON to get results. This is an example showing how to use the REST API designed to query the Godot Asset library in C++. It is built using the `libcurl` library.
Here is a snippet making a HTTP get and post request.
Here is a quick snippet to get started:
```c++ ```c++
// Get a full list of assets #include "http.hpp"
rest_api::context context = rest_api::make_context();
rapidjson::Document doc = http::get_asset_list( #include <string>
"https://godotengine.org/asset-library/api", #include <unordered_map>
context
) using string = std::string;
// ... parse the rapidjson::Document using headers_t = std::unordered_map<string, string>;
string url = "www.example.com";
http::context http;
http::request params = {{"user-agent", "firefox"}};
http::response r = http::request(url, params);
if(r.code == http::OK){ /* ...do something... */ }
r = http::request(url, params, http::method::POST);
if(r.code == http::OK){ /* ...do something.. */ }
```
Here's an example using the `multi` interface for parallel requests and downloads:
```c++
#include "http.hpp"
#include <memory>
template <typename T>
using ptr = std::unique_ptr<T>;
using strings = std::vector<std::string>;
using transfers = std::vector<http::transfer>;
using responses = std::vector<http::response>;
http::multi http;
http::request params = {{"user-agent", "firefox"}}
params.headers.insert(http::header("Accept", "*/*"));
params.headers.insert(http::header("Accept-Encoding", "application/gzip"));
params.headers.insert(http::header("Content-Encoding", "application/gzip"));
params.headers.insert(http::header("Connection", "keep-alive"));
strings request_urls = {
"www.example.com",
"www.google.com"
}
ptr<http::transfers> requests = http::make_requests(request_url, params);
ptr<http::responses> responses = http::execute(std::move(requests));
for(const auto& r : *responses){
if(r.code == http::OK){
// ...do something
}
}
strings download_urls = {
""
}
strings storage_paths = {
"./download1"
}
ptr<http::transfers> downloads = http::make_downloads(download_url, storage_paths, params);
ptr<http::responses> responses = http::execute(std::move(downloads));
for(const auto& r : *responses){
if(r.code == http::OK){
// ...do something
}
}
``` ```

View file

@ -1,26 +1,15 @@
# GDPM Rest API Example # GDPM HTTP Example
This is an example showing how to use the REST API designed to query the Godot Asset library in C++. It is built using the `libcurl` library. This is an example showing how to use the GDPM HTTP library to download files. The library uses RapidJSON to get results.
Here is a snippet making a HTTP get and post request.
Here is a quick snippet to get started:
```c++ ```c++
using string = std::string; // Get a full list of assets
using headers_t = std::unordered_map<string, string>; rest_api::context context = rest_api::make_context();
rapidjson::Document doc = http::get_asset_list(
std::string url = "www.example.com"; "https://godotengine.org/asset-library/api",
http::response r_get = http::request_get(url) context
if(r_get.response_code == http::response_code::OK){ )
// ...do something... // ... parse the rapidjson::Document
}
http::request_params params;
params.headers = {
{"user-agent", "firefox"},
{"content-type", "application/json"}
}
http::response r_post = http::request_post(url, params);
if(r_post.response_code == http::response_code::OK){
// ...do something...
}
``` ```

View file

@ -1,6 +0,0 @@
#pragma once
#include <string>
namespace gdpm::concepts{
template <typename...Args> concept RequireMinArgs = requires (std::size_t min){ sizeof...(Args) > min; };
}

View file

@ -36,8 +36,8 @@ namespace gdpm::print{
#define GDPM_CONFIG_REMOTE_SOURCES std::pair<std::string, std::string>(constants::RemoteName, constants::HostUrl) #define GDPM_CONFIG_REMOTE_SOURCES std::pair<std::string, std::string>(constants::RemoteName, constants::HostUrl)
#define GDPM_CONFIG_THREADS 1 #define GDPM_CONFIG_THREADS 1
#define GDPM_CONFIG_TIMEOUT_MS 30000 #define GDPM_CONFIG_TIMEOUT_MS 30000
#define GDPM_CONFIG_ENABLE_SYNC 1 #define GDPM_CONFIG_ENABLE_SYNC true
#define GDPM_CONFIG_ENABLE_FILE_LOGGING 0 #define GDPM_CONFIG_ENABLE_FILE_LOGGING true
#define GDPM_CONFIG_VERBOSE 0 #define GDPM_CONFIG_VERBOSE 0
/* Defines the default package cache for local storage */ /* Defines the default package cache for local storage */

View file

@ -2,6 +2,7 @@
#include "log.hpp" #include "log.hpp"
#include "types.hpp" #include "types.hpp"
#include "utils.hpp"
#include <fmt/core.h> #include <fmt/core.h>
#include <new> #include <new>
#include <string> #include <string>
@ -15,11 +16,14 @@ namespace gdpm::constants::error{
UNKNOWN_COMMAND, UNKNOWN_COMMAND,
UNKNOWN_ARGUMENT, UNKNOWN_ARGUMENT,
ARGPARSE_ERROR, ARGPARSE_ERROR,
ASSERTION_FAILED,
PRECONDITION_FAILED,
POSTCONDITION_FAILED,
NOT_FOUND, NOT_FOUND,
NOT_DEFINED, NOT_DEFINED,
NOT_IMPLEMENTED, NOT_IMPLEMENTED,
NO_PACKAGE_FOUND, NO_PACKAGE_FOUND,
PATH_NOT_DEFINED, MALFORMED_PATH,
FILE_EXISTS, FILE_EXISTS,
FILE_NOT_FOUND, FILE_NOT_FOUND,
DIRECTORY_EXISTS, DIRECTORY_EXISTS,
@ -43,12 +47,17 @@ namespace gdpm::constants::error{
"", "",
"An unknown error has occurred.", "An unknown error has occurred.",
"Unknown command.", "Unknown command.",
"Unknown argument.",
"Could not parse argument.",
"Assertion condition failed.",
"Pre-condition failed.",
"Post-condition failed.",
"Resource not found.", "Resource not found.",
"Function not defined.", "Resource not defined.",
"Function not implemented.", "Resource not implemented.",
"No package found.", "No package found.",
"Path is not well-defined", "Path is malformed.",
"File found.", "File already exists",
"File does not exist.", "File does not exist.",
"Directory exists.", "Directory exists.",
"Directory not found.", "Directory not found.",
@ -58,6 +67,10 @@ namespace gdpm::constants::error{
"Invalid configuration.", "Invalid configuration.",
"Invalid key.", "Invalid key.",
"An HTTP response error has occurred.", "An HTTP response error has occurred.",
"A SQLite error has occurred.",
"A libzip error has occurred.",
"A libcurl error has occurred.",
"A JSON error has occurred.",
"An error has occurred." "An error has occurred."
}; };
@ -73,10 +86,12 @@ namespace gdpm::constants::error{
}; };
namespace gdpm{ namespace gdpm{
namespace ec = constants::error;
class error { class error {
public: public:
constexpr explicit error(int code = 0, const string& message = "{code}"): constexpr explicit error(int code = 0, const string& message = "{default}"):
m_code(code), m_message(message == "{code}" ? constants::error::get_message(code): message) m_code(code),
m_message(utils::replace_all(message, "{default}", ec::get_message(code)))
{} {}
void set_code(int code) { m_code = code; } void set_code(int code) { m_code = code; }
@ -106,13 +121,21 @@ namespace gdpm{
#endif #endif
} }
// static constexpr void error(int code, const string& message = "{default}"){
// log::error(gdpm::error(code, message));
// }
static constexpr gdpm::error error_rc(const gdpm::error& e){ static constexpr gdpm::error error_rc(const gdpm::error& e){
error(e); log::error(e);
return e; return e;
} }
static void error(const char *p, const gdpm::error& e){ static constexpr gdpm::error error_rc(int code, const string& message = "{default}"){
println("{}{}{}", p, prefix.contents, e.get_message()); return error_rc(gdpm::error(code, message));
} }
} }
namespace concepts{
template <typename T>concept error_t = requires{ std::is_same<error, T>::value; };
}
} }

View file

@ -2,6 +2,12 @@
#include "constants.hpp" #include "constants.hpp"
#include "types.hpp" #include "types.hpp"
#include "indicators/indeterminate_progress_bar.hpp"
#include "indicators/dynamic_progress.hpp"
#include "indicators/progress_bar.hpp"
#include "indicators/block_progress_bar.hpp"
#include "utils.hpp"
#include <memory>
#include <unordered_map> #include <unordered_map>
#include <curl/curl.h> #include <curl/curl.h>
#include <curl/easy.h> #include <curl/easy.h>
@ -10,6 +16,13 @@ namespace gdpm::http{
using headers_t = std::unordered_map<string, string>; using headers_t = std::unordered_map<string, string>;
using header = std::pair<string, string>; using header = std::pair<string, string>;
enum method{
GET,
POST,
PUT,
DELETE
};
// REF: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status // REF: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
enum response_code{ enum response_code{
CONTINUE = 100, CONTINUE = 100,
@ -77,6 +90,11 @@ namespace gdpm::http{
NETWORK_AUTHENTICATION_REQUIRED = 511 NETWORK_AUTHENTICATION_REQUIRED = 511
}; };
enum transfer_type{
REQUEST,
DOWNLOAD
};
struct response{ struct response{
long code = 0; long code = 0;
string body{}; string body{};
@ -85,31 +103,91 @@ namespace gdpm::http{
}; };
struct request_params { struct request {
headers_t headers = {}; headers_t headers = {};
method method = method::GET;
size_t timeout = GDPM_CONFIG_TIMEOUT_MS; size_t timeout = GDPM_CONFIG_TIMEOUT_MS;
int verbose = 0; int verbose = 0;
}; };
using namespace indicators;
// BlockProgressBar bar {
// option::BarWidth{50},
// // option::Start{"["},
// // option::Fill{"="},
// // option::Lead{">"},
// // option::Remainder{" "},
// // option::End{"]"},
// option::PrefixText{"Downloading file "},
// option::PostfixText{""},
// option::ForegroundColor{Color::green},
// option::FontStyles{std::vector<FontStyle>{FontStyle::bold}},
// };
// // option::ShowElapsedTime{true},
// // option::ShowRemainingTime{true},
// IndeterminateProgressBar bar_unknown {
// option::BarWidth{50},
// option::Start{"["},
// option::Fill{"."},
// option::Lead{"<==>"},
// option::PrefixText{"Downloading file "},
// option::End{"]"},
// option::PostfixText{""},
// option::ForegroundColor{Color::green},
// option::FontStyles{std::vector<FontStyle>{FontStyle::bold}},
// };
struct transfer : public non_copyable{
transfer(){ curl = curl_easy_init(); }
transfer(transfer&&){}
~transfer(){ }
CURLcode res;
int id;
CURL *curl = nullptr;
FILE *fp = nullptr;
utils::memory_buffer data = {0};
};
using transfers = std::vector<transfer>;
using responses = std::vector<response>;
class context : public non_copyable{ class context : public non_copyable{
public: public:
context(); context();
~context(); ~context();
inline CURL* const get_curl() const;
string url_escape(const string& url); string url_escape(const string& url);
response request_get(const string& url, const http::request_params& params = http::request_params()); response request(const string& url, const http::request& params = http::request());
response request_post(const string& url, const http::request_params& params = http::request_params()); response download_file(const string& url, const string& storage_path, const http::request& params = http::request());
response download_file(const string& url, const string& storage_path, const http::request_params& params = http::request_params());
long get_download_size(const string& url); long get_download_size(const string& url);
long get_bytes_downloaded(const string& url); long get_bytes_downloaded(const string& url);
private: private:
CURL *curl; CURL *curl = nullptr;
curl_slist* _add_headers(CURL *curl, const headers_t& headers);
}; };
class multi{
public:
multi(long max_allowed_transfers = 2);
~multi();
string url_escape(const string& url);
ptr<transfers> make_requests(const string_list& urls, const http::request& params = http::request());
ptr<transfers> make_downloads(const string_list& url, const string_list& storage_path, const http::request& params = http::request());
ptr<responses> execute(ptr<transfers> transfers, size_t timeout = 1000);
private:
DynamicProgress<BlockProgressBar> progress_bars;
CURLM *cm = nullptr;
CURLMsg *cmessage = nullptr;
CURLMcode cres;
int messages_left = -1;
};
curl_slist* add_headers(CURL *curl, const headers_t& headers);
static size_t write_to_buffer(char *contents, size_t size, size_t nmemb, void *userdata);
static size_t write_to_stream(char *ptr, size_t size, size_t nmemb, void *userdata);
static int show_download_progress(void *ptr, curl_off_t total_download, curl_off_t current_downloaded, curl_off_t total_upload, curl_off_t current_upload);
extern context http;
} }

View file

@ -110,8 +110,11 @@ namespace gdpm::package {
GDPM_DLL_EXPORT error search(const config::context& config, const title_list& package_titles, const params& params = package::params()); GDPM_DLL_EXPORT error search(const config::context& config, const title_list& package_titles, const params& params = package::params());
GDPM_DLL_EXPORT error list(const config::context& config, const params& params = package::params()); GDPM_DLL_EXPORT error list(const config::context& config, const params& params = package::params());
GDPM_DLL_EXPORT error export_to(const path_list& paths); GDPM_DLL_EXPORT error export_to(const path_list& paths);
GDPM_DLL_EXPORT error clean(const config::context& config, const title_list& package_titles);
GDPM_DLL_EXPORT error purge(const config::context& config);
GDPM_DLL_EXPORT error link(const config::context& config, const title_list& package_titles, const params& params = package::params()); GDPM_DLL_EXPORT error link(const config::context& config, const title_list& package_titles, const params& params = package::params());
GDPM_DLL_EXPORT error clone(const config::context& config, const title_list& package_titles, const params& params = package::params()); GDPM_DLL_EXPORT error clone(const config::context& config, const title_list& package_titles, const params& params = package::params());
GDPM_DLL_EXPORT result_t<info_list> fetch(const config::context& config, const title_list& package_titles);
GDPM_DLL_EXPORT void print_list(const rapidjson::Document& json); GDPM_DLL_EXPORT void print_list(const rapidjson::Document& json);
@ -120,12 +123,10 @@ namespace gdpm::package {
GDPM_DLL_EXPORT void print_table(const rapidjson::Document& json); GDPM_DLL_EXPORT void print_table(const rapidjson::Document& json);
GDPM_DLL_EXPORT result_t<info_list> get_package_info(const opts_t& opts); GDPM_DLL_EXPORT result_t<info_list> get_package_info(const opts_t& opts);
GDPM_DLL_EXPORT result_t<title_list> get_package_titles(const info_list& packages); GDPM_DLL_EXPORT result_t<title_list> get_package_titles(const info_list& packages);
GDPM_DLL_EXPORT void clean_temporary(const config::context& config, const title_list& package_titles);
GDPM_DLL_EXPORT void read_file_inputs(title_list& package_titles, const path_list& paths); GDPM_DLL_EXPORT void read_file_inputs(title_list& package_titles, const path_list& paths);
GDPM_DLL_EXPORT info_list find_cached_packages(const title_list& package_titles); GDPM_DLL_EXPORT info_list find_cached_packages(const title_list& package_titles);
GDPM_DLL_EXPORT info_list find_installed_packages(const title_list& package_titles); GDPM_DLL_EXPORT info_list find_installed_packages(const title_list& package_titles);
/* Dependency Management API */ /* Dependency Management API */
GDPM_DLL_EXPORT result_t<info_list> synchronize_database(const config::context& config, const title_list& package_titles);
GDPM_DLL_EXPORT result_t<info_list> resolve_dependencies(const config::context& config, const title_list& package_titles); GDPM_DLL_EXPORT result_t<info_list> resolve_dependencies(const config::context& config, const title_list& package_titles);
GDPM_DLL_EXPORT string to_json(const info& info, bool pretty_print = false); GDPM_DLL_EXPORT string to_json(const info& info, bool pretty_print = false);

View file

@ -27,6 +27,7 @@ namespace gdpm::package_manager {
update, update,
search, search,
p_export, /* reserved keyword */ p_export, /* reserved keyword */
purge,
list, list,
link, link,
clone, clone,

View file

@ -63,26 +63,30 @@ namespace gdpm::rest_api{
}; };
request_params make_from_config(const config::context& config); request_params make_from_config(const config::context& config);
string to_json(const rapidjson::Document& doc); string to_json(const json::document& doc);
string to_string(type_e type); string to_string(type_e type);
string to_string(support_e support); string to_string(support_e support);
string to_string(sort_e sort); string to_string(sort_e sort);
error print_params(const request_params& params, const string& filter = ""); error print_params(const request_params& params, const string& filter = "");
error print_asset(const request_params& params, const string& filter = "", const print::style& style = print::style::list); error print_asset(const request_params& params, const string& filter = "", const print::style& style = print::style::list);
rapidjson::Document _parse_json(const string& r, int verbose = 0); json::document _parse_json(const string& r, int verbose = 0);
string _prepare_request(const string& url, const request_params& context, const string& filter); string _prepare_request(const string& url, const request_params& context, const string& filter);
bool register_account(const string& username, const string& password, const string& email); bool register_account(const string& username, const string& password, const string& email);
bool login(const string& username, const string& password); bool login(const string& username, const string& password);
bool logout(); bool logout();
rapidjson::Document configure(const string& url = constants::HostUrl, type_e type = any, int verbose = 0); json::document configure(const string& url = constants::HostUrl, type_e type = any, int verbose = 0);
rapidjson::Document get_assets_list(const string& url, const request_params& params = {}, const string& filter = ""); json::document get_assets_list(const string& url, const request_params& params = {}, const string& filter = "");
rapidjson::Document get_asset(const string& url, int asset_id, const request_params& params = {}, const string& filter = ""); json::document get_asset(const string& url, int asset_id, const request_params& params = {}, const string& filter = "");
bool delete_asset(int asset_id); // ...for moderators bool delete_asset(int asset_id); // ...for moderators
bool undelete_asset(int asset_id); // ...for moderators bool undelete_asset(int asset_id); // ...for moderators
bool set_support_level(int asset_id); // ...for moderators bool set_support_level(int asset_id); // ...for moderators
namespace multi{
json::documents get_assets(const string_list& urls, id_list aset_ids, const request_params& api_params, const string_list& filters);
}
/* /*
POST /asset POST /asset
POST /asset/{id} POST /asset/{id}

View file

@ -2,13 +2,12 @@
#include "log.hpp" #include "log.hpp"
#include "error.hpp" #include "error.hpp"
#include "types.hpp"
#include <functional> #include <functional>
#include <type_traits> #include <type_traits>
namespace gdpm{ namespace gdpm{
template <class T, error_t U = error> template <class T, concepts::error_t U = error>
class result_t { class result_t {
public: public:
result_t() = delete; result_t() = delete;
@ -35,13 +34,14 @@ namespace gdpm{
fn_error = error; fn_error = error;
} }
constexpr U get_error() const{
return std::get<U>(data);
}
constexpr std::unique_ptr<T> unwrap() const { constexpr std::unique_ptr<T> unwrap() const {
/* First, check if ok() and error() are defined. */ /* First, check if ok() and error() are defined. */
if(!fn_error || !fn_ok){ if(!fn_error || !fn_ok){
error error( log::error(error(ec::NOT_DEFINED));
constants::error::NOT_DEFINED
);
log::error(error);
return nullptr; return nullptr;
} }
/* Then, attempt unwrap the data. */ /* Then, attempt unwrap the data. */

View file

@ -1,5 +1,6 @@
#pragma once #pragma once
#include <rapidjson/document.h>
#include <tuple> #include <tuple>
#include <functional> #include <functional>
#include <type_traits> #include <type_traits>
@ -41,13 +42,11 @@ namespace gdpm{
SIZE_T = 6, SIZE_T = 6,
}; };
template <typename T>
concept error_t = requires{ std::is_same<error, T>::value; };
using string = std::string; using string = std::string;
using string_list = std::vector<string>; using string_list = std::vector<string>;
using string_map = std::unordered_map<string, string>; using string_map = std::unordered_map<string, string>;
using string_pair = std::pair<string, string>; using string_pair = std::pair<string, string>;
using id_list = std::vector<int>;
using any = std::any; using any = std::any;
using var = std::variant<int, float, bool, string, string_list, string_map, size_t>; using var = std::variant<int, float, bool, string, string_list, string_map, size_t>;
template <typename T = var> template <typename T = var>
@ -65,6 +64,15 @@ namespace gdpm{
template <typename T = error> template <typename T = error>
using _task_list = std::vector<std::future<T>>; using _task_list = std::vector<std::future<T>>;
using task_list = _task_list<error>; using task_list = _task_list<error>;
template <typename T>
using ptr = std::unique_ptr<T>;
namespace json{
using document = rapidjson::Document;
using documents = std::vector<rapidjson::Document>;
}
namespace concepts{
template <typename...Args> concept require_min_args = requires (std::size_t min){ sizeof...(Args) > min; };
}
inline string_list unwrap(const var_args& args){ inline string_list unwrap(const var_args& args){
string_list sl; string_list sl;

View file

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "constants.hpp" #include "constants.hpp"
#include "types.hpp"
#include <algorithm> #include <algorithm>
#include <cstddef> #include <cstddef>
@ -71,6 +72,11 @@ namespace gdpm::utils {
std::move(part, from.end(), std::back_inserter(from)); std::move(part, from.end(), std::back_inserter(from));
from.erase(part); from.erase(part);
} }
template <class T>
std::vector<T> append(const std::vector<T>& a, const std::vector<T>& b){
a.insert(std::end(a), std::begin(b), std::end(b));
return a;
}
bool to_bool(const std::string& s); bool to_bool(const std::string& s);
std::vector<std::string> split_lines(const std::string& contents); std::vector<std::string> split_lines(const std::string& contents);
@ -84,7 +90,7 @@ namespace gdpm::utils {
std::vector<std::string> parse_lines(const std::string& s); std::vector<std::string> parse_lines(const std::string& s);
std::string replace_first(const std::string& s, const std::string& from, const std::string& to); std::string replace_first(const std::string& s, const std::string& from, const std::string& to);
std::string replace_all(const std::string& s, const std::string& from, const std::string& to); std::string replace_all(const std::string& s, const std::string& from, const std::string& to);
int extract_zip(const char *archive, const char *dest, int verbose = 0); error extract_zip(const char *archive, const char *dest, int verbose = 0);
std::string prompt_user(const char *message); std::string prompt_user(const char *message);
bool prompt_user_yn(const char *message); bool prompt_user_yn(const char *message);
void delay(std::chrono::milliseconds milliseconds = GDPM_REQUEST_DELAY); void delay(std::chrono::milliseconds milliseconds = GDPM_REQUEST_DELAY);
@ -93,11 +99,6 @@ namespace gdpm::utils {
std::string convert_size(long size); std::string convert_size(long size);
// TODO: Add function to get size of decompressed zip // TODO: Add function to get size of decompressed zip
namespace curl {
extern size_t write_to_buffer(char *contents, size_t size, size_t nmemb, void *userdata);
extern size_t write_to_stream(char *ptr, size_t size, size_t nmemb, void *userdata);
extern int show_progress(void *ptr, curl_off_t total_download, curl_off_t current_downloaded, curl_off_t total_upload, curl_off_t current_upload);
}
namespace json { namespace json {
std::string from_array(const std::set<std::string>& a, const std::string& prefix); std::string from_array(const std::set<std::string>& a, const std::string& prefix);
std::string from_object(const std::unordered_map<std::string, std::string>& m, const std::string& prefix, const std::string& spaces); std::string from_object(const std::unordered_map<std::string, std::string>& m, const std::string& prefix, const std::string& spaces);

View file

@ -29,15 +29,15 @@ namespace gdpm::cache{
namespace fs = std::filesystem; namespace fs = std::filesystem;
fs::path dir_path = fs::path(params.cache_path).parent_path(); fs::path dir_path = fs::path(params.cache_path).parent_path();
if(!fs::exists(dir_path)){ if(!fs::exists(dir_path)){
log::info("Creating cache directories...{}", params.cache_path); log::debug("Creating cache directories...{}", params.cache_path);
fs::create_directories(dir_path); fs::create_directories(dir_path);
} }
int rc = sqlite3_open(params.cache_path.c_str(), &db); int rc = sqlite3_open(params.cache_path.c_str(), &db);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, error error(ec::SQLITE_ERR,
std::format( std::format(
"create_package_database.sqlite3_open(): {}", "cache::create_package_database::sqlite3_open(): {}",
sqlite3_errmsg(db) sqlite3_errmsg(db)
) )
); );
@ -70,8 +70,8 @@ namespace gdpm::cache{
rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
// log::error("Failed to fetch data: {}\n", sqlite3_errmsg(db)); // log::error("Failed to fetch data: {}\n", sqlite3_errmsg(db));
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"create_package_database.sqlite3_exec(): {}", "cache::create_package_database::sqlite3_exec(): {}",
errmsg errmsg
)); ));
sqlite3_free(errmsg); sqlite3_free(errmsg);
@ -101,8 +101,8 @@ namespace gdpm::cache{
// log::println("{}", sql); // log::println("{}", sql);
int rc = sqlite3_open(params.cache_path.c_str(), &db); int rc = sqlite3_open(params.cache_path.c_str(), &db);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"insert_package_info.sqlite3_open(): {}", "cache::insert_package_info::sqlite3_open(): {}",
sqlite3_errmsg(db) sqlite3_errmsg(db)
)); ));
sqlite3_close(db); sqlite3_close(db);
@ -110,10 +110,9 @@ namespace gdpm::cache{
} }
rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(rc, std::format( error error = log::error_rc(ec::SQLITE_ERR,
"insert_package_info.sqlite3_exec(): {}", errmsg std::format("cache::insert_package_info::sqlite3_exec(): {}", errmsg)
)); );
log::error(error);
sqlite3_free(errmsg); sqlite3_free(errmsg);
sqlite3_close(db); sqlite3_close(db);
return error; return error;
@ -135,8 +134,6 @@ namespace gdpm::cache{
string sql{"BEGIN TRANSACTION;\n"}; string sql{"BEGIN TRANSACTION;\n"};
auto callback = [](void *data, int argc, char **argv, char **colnames){ auto callback = [](void *data, int argc, char **argv, char **colnames){
// log::error("{}", (const char*)data);
// p_data *_data = (p_data*)data;
package::info_list *_p_vector = (package::info_list*) data; package::info_list *_p_vector = (package::info_list*) data;
package::info p{ package::info p{
.asset_id = std::stoul(argv[1]), .asset_id = std::stoul(argv[1]),
@ -163,8 +160,8 @@ namespace gdpm::cache{
int rc = sqlite3_open(params.cache_path.c_str(), &db); int rc = sqlite3_open(params.cache_path.c_str(), &db);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"get_package_info_by_id.sqlite3_open(): {}", sqlite3_errmsg(db) "cache::get_package_info_by_id::sqlite3_open(): {}", sqlite3_errmsg(db)
)); ));
sqlite3_close(db); sqlite3_close(db);
return result_t(package::info_list(), error); return result_t(package::info_list(), error);
@ -177,7 +174,7 @@ namespace gdpm::cache{
rc = sqlite3_exec(db, sql.c_str(), callback, (void*)&p_vector, &errmsg); rc = sqlite3_exec(db, sql.c_str(), callback, (void*)&p_vector, &errmsg);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(constants::error::SQLITE_ERR, std::format(
"get_package_info_by_id.sqlite3_exec(): {}", errmsg "cache::get_package_info_by_id::sqlite3_exec(): {}", errmsg
)); ));
sqlite3_free(errmsg); sqlite3_free(errmsg);
sqlite3_close(db); sqlite3_close(db);
@ -231,8 +228,8 @@ namespace gdpm::cache{
int rc = sqlite3_open(params.cache_path.c_str(), &db); int rc = sqlite3_open(params.cache_path.c_str(), &db);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"get_package_info_by_title.sqlite3_open(): {}", sqlite3_errmsg(db) "cache::get_package_info_by_title::sqlite3_open(): {}", sqlite3_errmsg(db)
)); ));
sqlite3_close(db); sqlite3_close(db);
return result_t(package::info_list(), error); return result_t(package::info_list(), error);
@ -246,8 +243,8 @@ namespace gdpm::cache{
// log::println(sql); // log::println(sql);
rc = sqlite3_exec(db, sql.c_str(), callback, (void*)&p_vector, &errmsg); rc = sqlite3_exec(db, sql.c_str(), callback, (void*)&p_vector, &errmsg);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"get_package_info_by_title.sqlite3_exec(): {}", errmsg "cache::get_package_info_by_title::sqlite3_exec(): {}", errmsg
)); ));
sqlite3_free(errmsg); sqlite3_free(errmsg);
sqlite3_close(db); sqlite3_close(db);
@ -292,8 +289,8 @@ namespace gdpm::cache{
int rc = sqlite3_open(params.cache_path.c_str(), &db); int rc = sqlite3_open(params.cache_path.c_str(), &db);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"get_installed_packages.sqlite3_open(): {}", sqlite3_errmsg(db) "cache::get_installed_packages::sqlite3_open(): {}", sqlite3_errmsg(db)
)); ));
sqlite3_close(db); sqlite3_close(db);
return result_t(package::info_list(), error); return result_t(package::info_list(), error);
@ -302,8 +299,8 @@ namespace gdpm::cache{
sql += "SELECT * FROM " + params.table_name + " WHERE is_installed=1; COMMIT;"; sql += "SELECT * FROM " + params.table_name + " WHERE is_installed=1; COMMIT;";
rc = sqlite3_exec(db, sql.c_str(), callback, (void*)&p_vector, &errmsg); rc = sqlite3_exec(db, sql.c_str(), callback, (void*)&p_vector, &errmsg);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(rc, std::format( error error(ec::SQLITE_ERR, std::format(
"get_installed_packages.sqlite3_exec(): {}", errmsg "cache::get_installed_packages::sqlite3_exec(): {}", errmsg
)); ));
log::error(error); log::error(error);
sqlite3_free(errmsg); sqlite3_free(errmsg);
@ -326,8 +323,8 @@ namespace gdpm::cache{
int rc = sqlite3_open(params.cache_path.c_str(), &db); int rc = sqlite3_open(params.cache_path.c_str(), &db);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error( error error(
constants::error::SQLITE_ERR, std::format( ec::SQLITE_ERR, std::format(
"update_package_info.sqlite3_open(): {}", sqlite3_errmsg(db) "cache::update_package_info::sqlite3_open(): {}", sqlite3_errmsg(db)
)); ));
sqlite3_close(db); sqlite3_close(db);
return error; return error;
@ -359,8 +356,8 @@ namespace gdpm::cache{
} }
rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"update_package_info.sqlite3_exec(): {}\n\t{}", errmsg, sql "cache::update_package_info::sqlite3_exec(): {}\n\t{}", errmsg, sql
)); ));
sqlite3_free(errmsg); sqlite3_free(errmsg);
sqlite3_close(db); sqlite3_close(db);
@ -382,8 +379,8 @@ namespace gdpm::cache{
int rc = sqlite3_open(params.cache_path.c_str(), &db); int rc = sqlite3_open(params.cache_path.c_str(), &db);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"delete_packages.sqlite3_open(): {}", sqlite3_errmsg(db) "cache::delete_packages::sqlite3_open(): {}", sqlite3_errmsg(db)
)); ));
sqlite3_close(db); sqlite3_close(db);
return error; return error;
@ -395,8 +392,8 @@ namespace gdpm::cache{
} }
rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"delete_packages.sqlite3_exec(): {}", errmsg "cache::delete_packages::sqlite3_exec(): {}", errmsg
)); ));
sqlite3_free(errmsg); sqlite3_free(errmsg);
sqlite3_close(db); sqlite3_close(db);
@ -418,8 +415,8 @@ namespace gdpm::cache{
int rc = sqlite3_open(params.cache_path.c_str(), &db); int rc = sqlite3_open(params.cache_path.c_str(), &db);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"delete_packages.sqlite3_open(): {}", errmsg "cache::delete_packages::sqlite3_open(): {}", errmsg
)); ));
sqlite3_close(db); sqlite3_close(db);
return error; return error;
@ -431,8 +428,8 @@ namespace gdpm::cache{
} }
rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"delete_packages.sqlite3_exec(): {}", errmsg "cache::delete_packages::sqlite3_exec(): {}", errmsg
)); ));
sqlite3_free(errmsg); sqlite3_free(errmsg);
sqlite3_close(db); sqlite3_close(db);
@ -451,8 +448,8 @@ namespace gdpm::cache{
int rc = sqlite3_open(params.cache_path.c_str(), &db); int rc = sqlite3_open(params.cache_path.c_str(), &db);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"drop_package_database.sqlite3_open(): {}", sqlite3_errmsg(db) "cache::drop_package_database::sqlite3_open(): {}", sqlite3_errmsg(db)
)); ));
sqlite3_close(db); sqlite3_close(db);
return error; return error;
@ -460,8 +457,8 @@ namespace gdpm::cache{
rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg); rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errmsg);
if(rc != SQLITE_OK){ if(rc != SQLITE_OK){
error error(constants::error::SQLITE_ERR, std::format( error error(ec::SQLITE_ERR, std::format(
"drop_package_database.sqlite3_exec(): {}", errmsg "cache::drop_package_database::sqlite3_exec(): {}", errmsg
)); ));
sqlite3_free(errmsg); sqlite3_free(errmsg);
sqlite3_close(db); sqlite3_close(db);

View file

@ -106,18 +106,15 @@ namespace gdpm::config{
ParseErrorCode status = doc.Parse(contents.c_str()).GetParseError(); ParseErrorCode status = doc.Parse(contents.c_str()).GetParseError();
if(!doc.IsObject()){ if(!doc.IsObject()){
error error( return log::error_rc(
constants::error::FILE_NOT_FOUND, ec::FILE_NOT_FOUND,
"Could not load config file." "Could not load config file."
); );
log::error(error);
return error;
} }
error error = validate(doc); error error = validate(doc);
if(error()){ if(error.has_occurred()){
log::error(error); return log::error_rc(error);
return error;
} }
/* Make sure contents were read correctly. */ /* Make sure contents were read correctly. */
@ -138,12 +135,10 @@ namespace gdpm::config{
); );
} }
} else { } else {
gdpm::error error( return log::error_rc(
constants::error::INVALID_KEY, ec::INVALID_KEY,
"Could not read key `remote_sources`." "Could not read key `remote_sources`."
); );
log::error(error);
return error;
} }
} }
auto _get_value_string = [](Document& doc, const char *property){ auto _get_value_string = [](Document& doc, const char *property){

View file

@ -2,10 +2,14 @@
#include "http.hpp" #include "http.hpp"
#include "utils.hpp" #include "utils.hpp"
#include "log.hpp" #include "log.hpp"
#include "error.hpp"
#include <curl/curl.h> #include <curl/curl.h>
#include <curl/easy.h> #include <curl/easy.h>
#include <curl/multi.h>
#include <memory>
#include <stdio.h> #include <stdio.h>
#include <chrono> #include <chrono>
#include <type_traits>
namespace gdpm::http{ namespace gdpm::http{
@ -15,75 +19,34 @@ namespace gdpm::http{
curl = curl_easy_init(); curl = curl_easy_init();
} }
context::~context(){ context::~context(){
curl_global_cleanup();
curl_easy_cleanup(curl); curl_easy_cleanup(curl);
curl_global_cleanup();
} }
CURL* const context::get_curl() const{
return curl;
}
string context::url_escape(const string &url){ string context::url_escape(const string &url){
return curl_easy_escape(curl, url.c_str(), url.size());; return curl_easy_escape(curl, url.c_str(), url.size());;
} }
response context::request_get(
response context::request(
const string& url, const string& url,
const http::request_params& params const http::request& params
){ ){
CURLcode res; CURLcode res;
utils::memory_buffer buf = utils::make_buffer(); utils::memory_buffer buf = utils::make_buffer();
utils::memory_buffer data = utils::make_buffer();
response r; response r;
#if (GDPM_DELAY_HTTP_REQUESTS == 1) #if (GDPM_DELAY_HTTP_REQUESTS == 1)
using namespace std::chrono_literals; using namespace std::chrono_literals;
utils::delay(); utils::delay();
#endif #endif
// curl_global_init(CURL_GLOBAL_ALL);
// curl = curl_easy_init();
if(curl){ if(curl){
utils::memory_buffer *data; curl_slist *list = add_headers(curl, params.headers);
curl_slist *list = _add_headers(curl, params.headers); if(params.method == method::POST){
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
// curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_buffer);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress);
curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
res = curl_easy_perform(curl);
curl_slist_free_all(list);
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code);
if(res != CURLE_OK && params.verbose > 0)
log::error("_make_request.curl_easy_perform(): {}", curl_easy_strerror(res));
curl_easy_cleanup(curl);
}
r.body = buf.addr;
utils::free_buffer(buf);
// curl_global_cleanup();
return r;
}
response context::request_post(
const string& url,
const http::request_params& params
){
// CURL *curl = nullptr;
CURLcode res;
utils::memory_buffer buf = utils::make_buffer();
response r;
#if (GDPM_DELAY_HTTP_REQUESTS == 1)
using namespace std::chrono_literals;
utils::delay();
#endif
string h; string h;
std::for_each( std::for_each(
params.headers.begin(), params.headers.begin(),
@ -94,35 +57,32 @@ namespace gdpm::http{
); );
h.pop_back(); h.pop_back();
h = url_escape(h); h = url_escape(h);
// const char *post_fields = "";
// curl_global_init(CURL_GLOBAL_ALL);
// curl = curl_easy_init();
if(curl){
utils::memory_buffer *data;
curl_slist *list = _add_headers(curl, params.headers);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
// curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, h.size()); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, h.size());
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, h.c_str()); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, h.c_str());
}
else if(params.method == method::GET){
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
}
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_buffer); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_to_buffer);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data); curl_easy_setopt(curl, CURLOPT_XFERINFODATA, (void*)&data);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress); curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, show_download_progress);
curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str()); curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout); curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
res = curl_easy_perform(curl); res = curl_easy_perform(curl);
curl_slist_free_all(list); curl_slist_free_all(list);
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code);
if(res != CURLE_OK && params.verbose > 0) if(res != CURLE_OK && params.verbose > 0)
log::error("_make_request.curl_easy_perform(): {}", curl_easy_strerror(res)); log::error(ec::LIBCURL_ERR,
std::format("http::context::request::curl_easy_perform(): {}", curl_easy_strerror(res))
);
curl_easy_cleanup(curl); curl_easy_cleanup(curl);
} }
r.body = buf.addr; r.body = buf.addr;
utils::free_buffer(buf); utils::free_buffer(buf);
// curl_global_cleanup();
return r; return r;
} }
@ -130,7 +90,7 @@ namespace gdpm::http{
response context::download_file( response context::download_file(
const string& url, const string& url,
const string& storage_path, const string& storage_path,
const http::request_params& params const http::request& params
){ ){
// CURL *curl = nullptr; // CURL *curl = nullptr;
CURLcode res; CURLcode res;
@ -141,34 +101,19 @@ namespace gdpm::http{
using namespace std::chrono_literals; using namespace std::chrono_literals;
utils::delay(); utils::delay();
#endif #endif
// curl_global_init(CURL_GLOBAL_ALL);
// curl = curl_easy_init();
if(curl){ if(curl){
fp = fopen(storage_path.c_str(), "wb"); fp = fopen(storage_path.c_str(), "wb");
// if(!config.username.empty() && !config.password.empty()){
// std::string curlopt_userpwd{config.username + ":" + config.password};
// curl_easy_setopt(curl, CURLOPT_USERPWD, curlopt_userpwd.c_str());
// }
// /* Switch on full protocol/debug output while testing and disable
// * progress meter by setting to 0L */
// if(config.verbose){
// curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
// curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
// }
utils::memory_buffer *data; utils::memory_buffer *data;
curl_slist *list = _add_headers(curl, params.headers); curl_slist *list = add_headers(curl, params.headers);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
// curl_easy_setopt(curl, CURLOPT_USERPWD, "user:pass");
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(curl, CURLOPT_HEADER, 0); curl_easy_setopt(curl, CURLOPT_HEADER, 0);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_stream); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_to_stream);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data); curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress); curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, show_download_progress);
curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str()); curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout); curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
res = curl_easy_perform(curl); res = curl_easy_perform(curl);
@ -177,16 +122,16 @@ namespace gdpm::http{
/* Get response code, process error, save data, and close file. */ /* Get response code, process error, save data, and close file. */
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code);
if(res != CURLE_OK && params.verbose > 0){ if(res != CURLE_OK && params.verbose > 0){
log::error("download_file.curl_easy_perform() failed: {}", curl_easy_strerror(res)); log::error(ec::LIBCURL_ERR,
std::format("http::context::download_file::curl_easy_perform() failed: {}", curl_easy_strerror(res))
);
} }
fclose(fp); fclose(fp);
} }
// curl_global_cleanup();
return r; return r;
} }
long context::get_download_size(const string& url){ long context::get_download_size(const string& url){
// CURL *curl = curl_easy_init();
CURLcode res; CURLcode res;
if(curl){ if(curl){
curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
@ -221,7 +166,185 @@ namespace gdpm::http{
} }
curl_slist* context::_add_headers( multi::multi(long max_allowed_transfers){
curl_global_init(CURL_GLOBAL_ALL);
if(max_allowed_transfers > 1)
cm = curl_multi_init();
curl_multi_setopt(cm, CURLMOPT_MAXCONNECTS, (long)max_allowed_transfers);
}
multi::~multi(){
if(cm != nullptr)
curl_multi_cleanup(cm);
curl_global_cleanup();
}
string multi::url_escape(const string &url){
return curl_easy_escape(cm, url.c_str(), url.size());;
}
ptr<transfers> multi::make_requests(
const string_list& urls,
const http::request& params
){
if(cm == nullptr){
log::error(error(PRECONDITION_FAILED,
"http::multi::make_downloads(): multi client not initialized.")
);
return std::make_unique<transfers>();
}
if(urls.size() <= 0){
log::warn("No requests to make.");
return std::make_unique<transfers>();
}
ptr<transfers> ts = std::make_unique<transfers>();
for(const auto& url : urls){
transfer t;
if(t.curl){
curl_slist *list = add_headers(t.curl, params.headers);
if(params.method == method::POST){
string h;
std::for_each(
params.headers.begin(),
params.headers.end(),
[&h](const string_pair& kv){
h += kv.first + "=" + kv.second + "&";
}
);
h.pop_back();
h = url_escape(h);
curl_easy_setopt(t.curl, CURLOPT_POSTFIELDSIZE, h.size());
curl_easy_setopt(t.curl, CURLOPT_POSTFIELDS, h.c_str());
}
else if(params.method == method::GET){
curl_easy_setopt(t.curl, CURLOPT_CUSTOMREQUEST, "GET");
}
curl_easy_setopt(t.curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(t.curl, CURLOPT_WRITEDATA, (void*)&t.data);
curl_easy_setopt(t.curl, CURLOPT_WRITEFUNCTION, write_to_buffer);
curl_easy_setopt(t.curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(t.curl, CURLOPT_XFERINFODATA, &t.data);
curl_easy_setopt(t.curl, CURLOPT_XFERINFOFUNCTION, show_download_progress);
curl_easy_setopt(t.curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
curl_easy_setopt(t.curl, CURLOPT_TIMEOUT_MS, params.timeout);
}
}
return ts;
}
ptr<transfers> multi::make_downloads(
const string_list& urls,
const string_list& storage_paths,
const http::request& params
){
if(cm == nullptr){
log::error(error(ec::PRECONDITION_FAILED,
"http::multi::make_downloads(): multi client not initialized.")
);
return std::make_unique<transfers>();
}
if(urls.size() != storage_paths.size()){
log::error(error(ec::ASSERTION_FAILED,
"http::context::make_downloads(): urls.size() != storage_paths.size()"
));
}
ptr<transfers> ts = std::make_unique<transfers>();
for(size_t i = 0; i < urls.size(); i++){
const string& url = urls.at(i);
const string& storage_path = storage_paths.at(i);
response r;
transfer t;
t.id = i;
if(t.curl){
t.fp = fopen(storage_path.c_str(), "wb");
curl_slist *list = add_headers(t.curl, params.headers);
curl_easy_setopt(t.curl, CURLOPT_URL, url.c_str());
// curl_easy_setopt(t.curl, CURLOPT_PRIVATE, url.c_str());
curl_easy_setopt(t.curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(t.curl, CURLOPT_HEADER, 0);
curl_easy_setopt(t.curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(t.curl, CURLOPT_WRITEDATA, t.fp);
curl_easy_setopt(t.curl, CURLOPT_WRITEFUNCTION, write_to_stream);
curl_easy_setopt(t.curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(t.curl, CURLOPT_XFERINFODATA, &t.data);
curl_easy_setopt(t.curl, CURLOPT_XFERINFOFUNCTION, show_download_progress);
curl_easy_setopt(t.curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
curl_easy_setopt(t.curl, CURLOPT_TIMEOUT_MS, params.timeout);
cres = curl_multi_add_handle(cm, t.curl);
curl_slist_free_all(list);
if(cres != CURLM_OK){
log::error(ec::LIBCURL_ERR,
std::format("http::context::make_downloads(): {}", curl_multi_strerror(cres))
);
}
ts->emplace_back(std::move(t));
/* NOTE: Should the file pointer be closed here? */
}
}
return ts;
}
ptr<responses> multi::execute(
ptr<transfers> transfers,
size_t timeout
){
if(cm == nullptr){
log::error(error(PRECONDITION_FAILED,
"http::multi::execute(): multi client not initialized")
);
return std::make_unique<responses>();
}
if(transfers->empty()){
log::debug("http::multi::execute(): no transfers found");
return std::make_unique<responses>();
}
size_t transfers_left = transfers->size();
ptr<responses> responses = std::make_unique<http::responses>(transfers->size());
do{
int still_alive = 1;
cres = curl_multi_perform(cm, &still_alive);
while((cmessage = curl_multi_info_read(cm, &messages_left))){
if(cmessage->msg == CURLMSG_DONE){
char *url = nullptr;
transfer& t = transfers->at(transfers_left-1);
response& r = responses->at(transfers_left-1);
t.curl = cmessage->easy_handle;
curl_easy_getinfo(cmessage->easy_handle, CURLINFO_EFFECTIVE_URL, &url);
curl_easy_getinfo(cmessage->easy_handle, CURLINFO_RESPONSE_CODE, &r.code);
if((int)cmessage->data.result != CURLM_OK){
log::error(error(ec::LIBCURL_ERR,
std::format("http::context::execute({}): {} <url: {}>", (int)cmessage->data.result, curl_easy_strerror(cmessage->data.result), url))
);
}
curl_multi_remove_handle(cm, t.curl);
curl_easy_cleanup(t.curl);
if(t.fp) fclose(t.fp);
transfers->pop_back();
transfers_left -= 1;
}
else{
log::error(error(ec::LIBCURL_ERR,
std::format("http::context::execute(): {}", (int)cmessage->msg))
);
}
}
if(transfers_left)
curl_multi_wait(cm, NULL, 0, timeout, NULL);
}while(transfers_left);
return responses;
}
curl_slist* add_headers(
CURL *curl, CURL *curl,
const headers_t& headers const headers_t& headers
){ ){
@ -235,4 +358,83 @@ namespace gdpm::http{
} }
return list; return list;
} }
size_t write_to_buffer(
char *contents,
size_t size,
size_t nmemb,
void *userdata
){
size_t realsize = size * nmemb;
utils::memory_buffer *m = (utils::memory_buffer*)userdata;
m->addr = (char*)realloc(m->addr, m->size + realsize + 1);
if(m->addr == nullptr){
/* Out of memory */
log::error("Could not allocate memory (realloc returned NULL).");
return 0;
}
memcpy(&(m->addr[m->size]), contents, realsize);
m->size += realsize;
m->addr[m->size] = 0;
return realsize;
}
size_t write_to_stream(
char *ptr,
size_t size,
size_t nmemb,
void *userdata
){
if(nmemb == 0)
return 0;
return fwrite(ptr, size, nmemb, (FILE*)userdata);
}
int show_download_progress(
void *ptr,
curl_off_t total_download,
curl_off_t current_downloaded,
curl_off_t total_upload,
curl_off_t current_upload
){
if(current_downloaded >= total_download)
return 0;
using namespace indicators;
show_console_cursor(false);
// if(total_download != 0){
// // double percent = std::floor((current_downloaded / (total_download)) * 100);
// bar.set_option(option::MaxProgress{total_download});
// // bar.set_option(option::HideBarWhenComplete{false});
// bar.set_progress(current_downloaded);
// bar.set_option(option::PostfixText{
// utils::convert_size(current_downloaded) + " / " +
// utils::convert_size(total_download)
// });
// if(bar.is_completed()){
// bar.set_option(option::PrefixText{"Download completed."});
// bar.mark_as_completed();
// }
// } else {
// if(bar_unknown.is_completed()){
// bar_unknown.set_option(option::PrefixText{"Download completed."});
// bar_unknown.mark_as_completed();
// } else {
// bar.tick();
// bar_unknown.set_option(
// option::PostfixText(std::format("{}", utils::convert_size(current_downloaded)))
// );
// }
// }
show_console_cursor(true);
utils::memory_buffer *m = (utils::memory_buffer*)ptr;
return 0;
}
} }

View file

@ -40,97 +40,110 @@ namespace gdpm::package{
*/ */
/* Synchronize database information and then try to get data again from
cache if possible. */
if(config.enable_sync){
result_t result = fetch(config, package_titles);
error error = result.get_error();
if(error.has_occurred()){
return log::error_rc(ec::UNKNOWN, "package::install(): could not synchronize database.");
}
}
/* Append files from --file option */ /* Append files from --file option */
read_file_inputs(package_titles, params.input_files); read_file_inputs(package_titles, params.input_files);
result_t result = cache::get_package_info_by_title(package_titles); result_t result = cache::get_package_info_by_title(package_titles);
package::info_list p_cache = result.unwrap_unsafe(); package::info_list p_cache = result.unwrap_unsafe();
/* Synchronize database information and then try to get data again from
cache if possible. */
if(config.enable_sync || p_cache.empty()){
result_t result = synchronize_database(config, package_titles);
p_cache = result.unwrap_unsafe();
}
/* Match queried package titles with those found in cache. */
package::info_list p_found = find_cached_packages(package_titles);
// if(p_found.size() == package_titles.size()){
// log::info("Found all packages stored in local cache.");
// }
/* Found nothing to install so there's nothing to do at this point. */ /* Found nothing to install so there's nothing to do at this point. */
if(p_found.empty()){ if(p_cache.empty()){
error error( return log::error_rc(
constants::error::NOT_FOUND, ec::NOT_FOUND, /* TODO: change to PACKAGE_NOT_FOUND */
"No packages found to install." "package::install(): no packages found to install."
); );
log::error(error);
return error;
} }
log::println("Packages to install: "); /* Show packages to install */
for(const auto& p : p_found){ {
string output((p.is_installed) ? p.title + " (reinstall)" : p.title); using namespace tabulate;
log::print(" {} ", (p.is_installed) ? p.title + " (reinstall)" : p.title); Table table;
table.format()
.border_top("")
.border_bottom("")
.border_left("")
.border_right("")
.corner("")
.padding_top(0)
.padding_bottom(0);
table.add_row({"Title", "Author", "Category", "Version", "Godot", "Last Modified", "Installed?"});
table[0].format()
.font_style({FontStyle::underline, FontStyle::bold});
for(const auto& p : p_cache){
table.add_row({p.title, p.author, p.category, p.version, p.godot_version, p.modify_date, (p.is_installed) ? "✔️": ""});
size_t index = table.size() - 1;
table[index][3].format().font_align(FontAlign::center);
table[index][4].format().font_align(FontAlign::center);
table[index][6].format().font_align(FontAlign::center);
// string output(p.title + GDPM_COLOR_CYAN " v" + p.version + GDPM_COLOR_RESET);
// output += GDPM_COLOR_BLUE " last updated: " + p.modify_date + GDPM_COLOR_RESET;
// output += (p.is_installed) ? GDPM_COLOR_LIGHT_CYAN " (reinstall)" : "";
// output += GDPM_COLOR_RESET;
// log::print(" {}\n", output);
} }
table.print(std::cout);
log::println(""); log::println("");
}
/* Skip prompt if set in config */
if(!config.skip_prompt){ if(!config.skip_prompt){
if(!utils::prompt_user_yn("Do you want to install these packages? (Y/n)")) if(!utils::prompt_user_yn("Do you want to install these packages? (Y/n)"))
return error(); return error();
} }
/* Check if provided param is in remote sources*/ /* Check if provided remote param is in remote sources */
if(!config.remote_sources.contains(params.remote_source)){ if(!config.remote_sources.contains(params.remote_source)){
error error( return log::error_rc(
constants::error::NOT_FOUND, ec::NOT_FOUND, /* TODO: change to REMOTE_NOT_FOUND */
"Remote source not found in config." "package::install(): remote source not found in config."
); );
log::error(error);
return error;
} }
/* Try and obtain all requested packages. */ // task_list tasks;
std::vector<string_pair> dir_pairs;
task_list tasks;
rest_api::request_params rest_api_params = rest_api::make_from_config(config);
for(auto& p : p_found){ // TODO: Execute each in parallel using coroutines??
/* Check if a remote source was provided. If not, then try to get packages
in global storage location only. */
log::info("Fetching asset data for \"{}\"...", p.title);
string url{config.remote_sources.at(params.remote_source) + rest_api::endpoints::GET_AssetId};
string package_dir, tmp_dir, tmp_zip;
/* Retrieve necessary asset data if it was found already in cache */ /* Retrieve necessary asset data if it was found already in cache */
std::vector<string_pair> target_extract_dirs;
rest_api::request_params rest_api_params = rest_api::make_from_config(config);
package::title_list p_download_urls;
package::path_list p_storage_paths;
for(auto& p : p_cache){
log::info_n("Fetching asset data for \"{}\"...", p.title);
string url{config.remote_sources.at(params.remote_source) + rest_api::endpoints::GET_AssetId};
string package_dir = config.packages_dir + "/" + p.title;
string tmp_dir = config.tmp_dir + "/" + p.title;
string tmp_zip = tmp_dir + ".zip";
Document doc; Document doc;
bool is_missing_data = p.download_url.empty() || p.category.empty() || p.description.empty() || p.support_level.empty(); bool is_data_missing = p.download_url.empty() || p.category.empty() || p.description.empty() || p.support_level.empty();
if(is_missing_data){ if(is_data_missing){
doc = rest_api::get_asset(url, p.asset_id, rest_api_params); doc = rest_api::get_asset(url, p.asset_id, rest_api_params);
if(doc.HasParseError() || doc.IsNull()){ if(doc.HasParseError() || doc.IsNull()){
return log::error_rc(error( return log::error_rc(
constants::error::JSON_ERR, ec::JSON_ERR,
std::format("Error parsing JSON: {}", GetParseError_En(doc.GetParseError())) std::format("package::install(): error parsing JSON: {}",
)); GetParseError_En(doc.GetParseError()))
);
} }
p.category = doc["category"].GetString(); p.category = doc["category"].GetString();
p.description = doc["description"].GetString(); p.description = doc["description"].GetString();
p.support_level = doc["support_level"].GetString(); p.support_level = doc["support_level"].GetString();
p.download_url = doc["download_url"].GetString(); p.download_url = doc["download_url"].GetString();
p.download_hash = doc["download_hash"].GetString(); p.download_hash = doc["download_hash"].GetString();
log::println("Done");
} }
else{ else{
log::info("Found asset data found for \"{}\"", p.title); log::println("");
log::info("Found asset data for \"{}\".", p.title);
} }
/* Set directory and temp paths for storage */
package_dir = config.packages_dir + "/" + p.title;
tmp_dir = config.tmp_dir + "/" + p.title;
tmp_zip = tmp_dir + ".zip";
/* Make directories for packages if they don't exist to keep everything organized */ /* Make directories for packages if they don't exist to keep everything organized */
if(!std::filesystem::exists(config.tmp_dir)) if(!std::filesystem::exists(config.tmp_dir))
std::filesystem::create_directories(config.tmp_dir); std::filesystem::create_directories(config.tmp_dir);
@ -145,60 +158,83 @@ namespace gdpm::package{
OStreamWrapper osw(ofs); OStreamWrapper osw(ofs);
PrettyWriter<OStreamWrapper> writer(osw); PrettyWriter<OStreamWrapper> writer(osw);
doc.Accept(writer); doc.Accept(writer);
target_extract_dirs.emplace_back(string_pair(tmp_zip, package_dir + "/"));
/* Check if we already have a stored temporary file before attempting to download */ /* Check if we already have a stored temporary file before attempting to download */
if(std::filesystem::exists(tmp_zip) && std::filesystem::is_regular_file(tmp_zip)){ if(std::filesystem::exists(tmp_zip) && std::filesystem::is_regular_file(tmp_zip)){
log::info("Found cached package. Skipping download.", p.title); log::info("Found cached package for \"{}\".", p.title);
} }
else{ else{
/* Download all the package files and place them in tmp directory. */ p_download_urls.emplace_back(p.download_url);
log::info_n("Downloading \"{}\"...", p.title); p_storage_paths.emplace_back(tmp_zip);
http::context http; }
http::response response = http.download_file(p.download_url, tmp_zip); }
if(response.code == http::OK){
log::println("Done."); /* Make sure the number of urls matches storage locations */
}else{ if(p_download_urls.size() != p_storage_paths.size()){
return log::error_rc(error( return log::error_rc(error(ec::ASSERTION_FAILED,
constants::error::HTTP_RESPONSE_ERR, "package::install(): p_left.size() != p_storage.size()"
std::format("HTTP Error: {}", response.code) ));
}
/* Attempt to download ZIPs in parallel */
if(config.jobs > 1){
http::multi http(config.jobs);
ptr<http::transfers> transfers = http.make_downloads(p_download_urls, p_storage_paths);
ptr<http::responses> responses = http.execute(std::move(transfers));
/* Check for HTTP response errors */
for(const auto& r : *responses){
if(r.code != http::OK){
log::error(error(ec::HTTP_RESPONSE_ERR,
std::format("HTTP error: {}", r.code)
)); ));
} }
} }
}
dir_pairs.emplace_back(string_pair(tmp_zip, package_dir + "/")); else{
http::context http;
p.is_installed = true; for(size_t i = 0; i < p_download_urls.size(); i++){
p.install_path = package_dir; http::response r = http.download_file(
p_download_urls[i],
p_storage_paths[i]
);
if(r.code != http::OK){
log::error(error(ec::HTTP_RESPONSE_ERR,
std::format("HTTP error: {}", r.code)
));
}
}
}
/* Extract all the downloaded packages to their appropriate directory location. */ /* Extract all the downloaded packages to their appropriate directory location. */
for(const auto& p : dir_pairs){ for(const auto& p : target_extract_dirs){
int ec = utils::extract_zip(p.first.c_str(), p.second.c_str()); error error = utils::extract_zip(p.first.c_str(), p.second.c_str());
if(ec){ if(error.has_occurred()){
log::error_rc(error( return error;
constants::error::LIBZIP_ERR,
std::format("libzip returned an error code {}", ec)
));
} }
log::println("Done.");
}
/* Update the cache data */
for(auto& p : p_cache){
p.is_installed = true;
p.install_path = config.packages_dir + "/" + p.title;
} }
/* Update the cache data with information from */
log::info_n("Updating local asset data..."); log::info_n("Updating local asset data...");
error error = cache::update_package_info(p_found); error error = cache::update_package_info(p_cache);
if(error.has_occurred()){ if(error.has_occurred()){
string prefix = std::format(log::get_error_prefix(), utils::timestamp()); string prefix = std::format(log::get_error_prefix(), utils::timestamp());
log::println(GDPM_COLOR_LOG_ERROR"\n{}{}" GDPM_COLOR_RESET, prefix, error.get_message()); log::println(GDPM_COLOR_LOG_ERROR"\n{}{}" GDPM_COLOR_RESET, prefix, error.get_message());
return error; return error;
} }
if(config.clean_temporary){ if(config.clean_temporary){
clean_temporary(config, package_titles); clean(config, package_titles);
} }
log::println("Done."); log::println("Done.");
// })
// );
}
return error(); return error;
} }
@ -233,18 +269,16 @@ namespace gdpm::package{
/* Check if provided param is in remote sources*/ /* Check if provided param is in remote sources*/
if(!config.remote_sources.contains(params.remote_source)){ if(!config.remote_sources.contains(params.remote_source)){
error error( return log::error_rc(ec::NOT_FOUND,
constants::error::NOT_FOUND,
"Remote source not found in config." "Remote source not found in config."
); );
log::error(error);
return error;
} }
/* Install the other packages from remte source. */ /* Install the other packages from remte source. */
std::vector<string_pair> dir_pairs; std::vector<string_pair> dir_pairs;
task_list tasks; task_list tasks;
rest_api::request_params rest_api_params = rest_api::make_from_config(config); rest_api::request_params rest_api_params = rest_api::make_from_config(config);
package::info_list p_left;
for(auto& p : p_found){ // TODO: Execute each in parallel using coroutines?? for(auto& p : p_found){ // TODO: Execute each in parallel using coroutines??
/* Check if a remote source was provided. If not, then try to get packages /* Check if a remote source was provided. If not, then try to get packages
@ -299,11 +333,21 @@ namespace gdpm::package{
if(std::filesystem::exists(tmp_zip) && std::filesystem::is_regular_file(tmp_zip)){ if(std::filesystem::exists(tmp_zip) && std::filesystem::is_regular_file(tmp_zip)){
log::info("Found cached package. Skipping download.", p.title); log::info("Found cached package. Skipping download.", p.title);
} }
else{ else {
/* Download all the package files and place them in tmp directory. */ p_left.emplace_back(p);
log::info_n("Downloading \"{}\"...", p.title); }
http::context http; } // for loop
http::response response = http.download_file(p.download_url, tmp_zip);
/* Get the packages not found in cache and download */
string_list urls;
for(const auto& p : p_left){
urls.emplace_back(p.download_url);
}
http::multi http;
ptr<http::transfers> transfers = http.make_requests(urls);
ptr<http::responses> responses = http.execute(std::move(transfers));
for(const auto& response : *responses){
if(response.code == http::OK){ if(response.code == http::OK){
log::println("Done."); log::println("Done.");
}else{ }else{
@ -314,6 +358,12 @@ namespace gdpm::package{
} }
} }
/* Extract all packages and update cache database */
for(auto& p : p_found){
string package_dir = std::filesystem::current_path().string() + "/" + p.title;//config.packages_dir + "/" + p.title;
string tmp_dir = std::filesystem::current_path().string() + "/" + p.title + ".tmp";
string tmp_zip = tmp_dir + ".zip";
dir_pairs.emplace_back(string_pair(tmp_zip, package_dir + "/")); dir_pairs.emplace_back(string_pair(tmp_zip, package_dir + "/"));
p.is_installed = true; p.is_installed = true;
@ -321,13 +371,7 @@ namespace gdpm::package{
/* Extract all the downloaded packages to their appropriate directory location. */ /* Extract all the downloaded packages to their appropriate directory location. */
for(const auto& p : dir_pairs){ for(const auto& p : dir_pairs){
int ec = utils::extract_zip(p.first.c_str(), p.second.c_str()); error error = utils::extract_zip(p.first.c_str(), p.second.c_str());
if(ec){
log::error_rc(error(
constants::error::LIBZIP_ERR,
std::format("libzip returned an error code {}", ec)
));
}
} }
/* Remove temporary download archive */ /* Remove temporary download archive */
@ -357,12 +401,9 @@ namespace gdpm::package{
result_t result = cache::get_package_info_by_title(package_titles); result_t result = cache::get_package_info_by_title(package_titles);
package::info_list p_cache = result.unwrap_unsafe(); package::info_list p_cache = result.unwrap_unsafe();
if(p_cache.empty()){ if(p_cache.empty()){
error error( return log::error_rc(ec::NOT_FOUND,
constants::error::NOT_FOUND,
"Could not find any packages to remove." "Could not find any packages to remove."
); );
log::error(error);
return error;
} }
/* Count number packages in cache flagged as is_installed. If there are none, then there's nothing to do. */ /* Count number packages in cache flagged as is_installed. If there are none, then there's nothing to do. */
@ -372,19 +413,36 @@ namespace gdpm::package{
}); });
if(p_count == 0){ if(p_count == 0){
error error( return log::error_rc(ec::NOT_FOUND,
constants::error::NOT_FOUND,
"No packages to remove." "No packages to remove."
); );
log::error(error);
return error;
} }
log::println("Packages to remove:"); {
for(const auto& p : p_cache) using namespace tabulate;
if(p.is_installed) Table table;
log::print(" {} ", p.title); table.format()
.border_top("")
.border_bottom("")
.border_left("")
.border_right("")
.corner("")
.padding_top(0)
.padding_bottom(0);
table.add_row({"Title", "Author", "Category", "Version", "Godot", "Last Modified", "Installed?"});
table[0].format()
.font_style({FontStyle::underline, FontStyle::bold});
for(const auto& p : p_cache){
table.add_row({p.title, p.author, p.category, p.version, p.godot_version, p.modify_date, (p.is_installed) ? "✔️": ""});
// string output(p.title + GDPM_COLOR_CYAN " v" + p.version + GDPM_COLOR_RESET);
// output += GDPM_COLOR_BLUE " last updated: " + p.modify_date + GDPM_COLOR_RESET;
// output += (p.is_installed) ? GDPM_COLOR_LIGHT_CYAN " (reinstall)" : "";
// output += GDPM_COLOR_RESET;
// log::print(" {}\n", output);
}
table.print(std::cout);
log::println(""); log::println("");
}
if(!config.skip_prompt){ if(!config.skip_prompt){
if(!utils::prompt_user_yn("Do you want to remove these packages? (Y/n)")) if(!utils::prompt_user_yn("Do you want to remove these packages? (Y/n)"))
@ -428,13 +486,13 @@ namespace gdpm::package{
} }
log::println("Done."); log::println("Done.");
if(config.clean_temporary){ if(config.clean_temporary){
clean_temporary(config, package_titles); clean(config, package_titles);
} }
log::info_n("Updating local asset data..."); log::info_n("Updating local asset data...");
{ {
error error = cache::update_package_info(p_cache); error error = cache::update_package_info(p_cache);
if(error.has_occurred()){ if(error.has_occurred()){
log::error("\n{}", error.get_message()); log::error("\nsqlite: {}", error.get_message());
return error; return error;
} }
} }
@ -473,9 +531,9 @@ namespace gdpm::package{
string url{constants::HostUrl + rest_api::endpoints::GET_AssetId}; string url{constants::HostUrl + rest_api::endpoints::GET_AssetId};
Document doc = rest_api::get_assets_list(url, rest_api_params); Document doc = rest_api::get_assets_list(url, rest_api_params);
if(doc.IsNull()){ if(doc.IsNull()){
constexpr const char *message = "Could not get response from server. Aborting."; return log::error_rc(ec::HOST_UNREACHABLE,
log::error(message); "package::update(): could not get response from server. Aborting."
return error(constants::error::HOST_UNREACHABLE, message); );
} }
return error(); return error();
} }
@ -568,10 +626,9 @@ namespace gdpm::package{
remote::print_repositories(config); remote::print_repositories(config);
} }
else{ else{
log::error(error( log::error_rc(ec::UNKNOWN_COMMAND,
constants::error::UNKNOWN_COMMAND, "package::list(): unrecognized subcommand...try either 'packages' or 'remote' instead."
"Unrecognized subcommand. Try either 'packages' or 'remote' instead." );
));
} }
return error(); return error();
} }
@ -598,9 +655,9 @@ namespace gdpm::package{
} }
else { else {
constexpr const char *message = "File or directory exists!"; return log::error_rc(ec::FILE_EXISTS,
log::error(message); "package::export_to(): file or directory exists!"
return error(constants::error::FILE_EXISTS, message); );
} }
} }
std::ofstream of(path); std::ofstream of(path);
@ -613,6 +670,45 @@ namespace gdpm::package{
} }
error clean(
const config::context& config,
const title_list& package_titles
){
if(package_titles.empty()){
if(!config.skip_prompt){
if(!utils::prompt_user_yn("Are you sure you want to clean all temporary files? (Y/n)")){
return error();
}
}
std::filesystem::remove_all(config.tmp_dir);
return error();
}
/* Find the path of each packages is_installed then delete temporaries */
log::info_n("Cleaning temporary files...");
for(const auto& p_title : package_titles){
string tmp_zip = config.tmp_dir + "/" + p_title + ".zip";
if(config.verbose > 0)
log::info("Removed '{}'", tmp_zip);
std::filesystem::remove_all(tmp_zip);
}
log::println("Done.");
return error();
}
error purge(const config::context& config){
if(!config.skip_prompt){
if(!utils::prompt_user_yn("Are you sure you want to purge all installed packages? (Y/n)")){
return error();
}
}
/* Remove all packages installed in global location */
std::filesystem::remove_all(config.packages_dir);
return cache::drop_package_database();
}
error link( error link(
const config::context& config, const config::context& config,
const title_list& package_titles, const title_list& package_titles,
@ -622,7 +718,7 @@ namespace gdpm::package{
if(params.paths.empty()){ if(params.paths.empty()){
return log::error_rc(error( return log::error_rc(error(
constants::error::PATH_NOT_DEFINED, constants::error::MALFORMED_PATH,
"Path is required" "Path is required"
)); ));
} }
@ -685,7 +781,7 @@ namespace gdpm::package{
if(params.paths.empty()){ if(params.paths.empty()){
return log::error_rc(error( return log::error_rc(error(
constants::error::PATH_NOT_DEFINED, constants::error::MALFORMED_PATH,
"Path is required" "Path is required"
)); ));
} }
@ -739,6 +835,88 @@ namespace gdpm::package{
return error(); return error();
} }
result_t<info_list> fetch(
const config::context& config,
const title_list& package_titles
){
using namespace rapidjson;
rest_api::request_params rest_api_params = rest_api::make_from_config(config);
rest_api_params.page = 0;
int page = 0;
int page_length = 0;
int total_items = 0;
int items_left = 0;
// int total_pages = 0;
log::info_n("Sychronizing database...");
do{
/* Make the GET request to get page data and store it in the local
package database. Also, check to see if we need to keep going. */
string url{constants::HostUrl + rest_api::endpoints::GET_Asset};
Document doc = rest_api::get_assets_list(url, rest_api_params);
rest_api_params.page += 1;
if(doc.IsNull()){
log::println("");
return result_t(info_list(), log::error_rc(
ec::EMPTY_RESPONSE,
"Could not get response from server. Aborting."
));
}
/* Need to know how many pages left to get and how many we get per
request. */
page = doc["page"].GetInt();
page_length = doc["page_length"].GetInt();
// total_pages = doc["pages"].GetInt();
total_items = doc["total_items"].GetInt();
items_left = total_items - (page + 1) * page_length;
// log::info("page: {}, page length: {}, total pages: {}, total items: {}, items left: {}", page, page_length, total_pages, total_items, items_left);
if(page == 0){
error error;
error = cache::drop_package_database();
error = cache::create_package_database();
}
info_list packages;
for(const auto& o : doc["result"].GetArray()){
// log::println("=======================");
info p{
.asset_id = std::stoul(o["asset_id"].GetString()),
.title = o["title"].GetString(),
.author = o["author"].GetString(),
.author_id = std::stoul(o["author_id"].GetString()),
.version = o["version"].GetString(),
.godot_version = o["godot_version"].GetString(),
.cost = o["cost"].GetString(),
.modify_date = o["modify_date"].GetString(),
.category = o["category"].GetString(),
.remote_source = url
};
packages.emplace_back(p);
}
error error = cache::insert_package_info(packages);
if (error.has_occurred()){
log::error(error);
/* FIXME: Should this stop here or keep going? */
}
/* Make the same request again to get the rest of the needed data
using the same request, but with a different page, then update
variables as needed. */
} while(items_left > 0);
log::println("Done.");
return cache::get_package_info_by_title(package_titles);
}
void print_list(const info_list& packages){ void print_list(const info_list& packages){
for(const auto& p : packages){ for(const auto& p : packages){
log::println( log::println(
@ -856,26 +1034,6 @@ namespace gdpm::package{
} }
void clean_temporary(
const config::context& config,
const title_list& package_titles
){
if(package_titles.empty()){
log::info("No temporary files found to clean.");
std::filesystem::remove_all(config.tmp_dir);
}
/* Find the path of each packages is_installed then delete temporaries */
log::info_n("Cleaning temporary files...");
for(const auto& p_title : package_titles){
string tmp_zip = config.tmp_dir + "/" + p_title + ".zip";
if(config.verbose > 0)
log::info("Removed '{}'", tmp_zip);
std::filesystem::remove_all(tmp_zip);
}
log::println("Done.");
}
template <typename T> template <typename T>
auto set_if_key_exists( auto set_if_key_exists(
const var_opts& o, const var_opts& o,
@ -886,89 +1044,6 @@ namespace gdpm::package{
} }
result_t<info_list> synchronize_database(
const config::context& config,
const title_list& package_titles
){
using namespace rapidjson;
rest_api::request_params rest_api_params = rest_api::make_from_config(config);
rest_api_params.page = 0;
int page = 0;
int page_length = 0;
int total_items = 0;
int items_left = 0;
// int total_pages = 0;
log::info("Sychronizing database...");
do{
/* Make the GET request to get page data and store it in the local
package database. Also, check to see if we need to keep going. */
std::string url{constants::HostUrl};
url += rest_api::endpoints::GET_Asset;
Document doc = rest_api::get_assets_list(url, rest_api_params);
rest_api_params.page += 1;
if(doc.IsNull()){
error error(
constants::error::EMPTY_RESPONSE,
"Could not get response from server. Aborting."
);
log::error(error);
return result_t(info_list(), error);
}
/* Need to know how many pages left to get and how many we get per
request. */
page = doc["page"].GetInt();
page_length = doc["page_length"].GetInt();
// total_pages = doc["pages"].GetInt();
total_items = doc["total_items"].GetInt();
items_left = total_items - (page + 1) * page_length;
// log::info("page: {}, page length: {}, total pages: {}, total items: {}, items left: {}", page, page_length, total_pages, total_items, items_left);
if(page == 0){
error error;
error = cache::drop_package_database();
error = cache::create_package_database();
}
info_list packages;
for(const auto& o : doc["result"].GetArray()){
// log::println("=======================");
info p{
.asset_id = std::stoul(o["asset_id"].GetString()),
.title = o["title"].GetString(),
.author = o["author"].GetString(),
.author_id = std::stoul(o["author_id"].GetString()),
.version = o["version"].GetString(),
.godot_version = o["godot_version"].GetString(),
.cost = o["cost"].GetString(),
.modify_date = o["modify_date"].GetString(),
.category = o["category"].GetString(),
.remote_source = url
};
packages.emplace_back(p);
}
error error = cache::insert_package_info(packages);
if (error.has_occurred()){
log::error(error);
/* FIXME: Should this stop here or keep going? */
}
/* Make the same request again to get the rest of the needed data
using the same request, but with a different page, then update
variables as needed. */
} while(items_left > 0);
log::println("Done.");
return cache::get_package_info_by_title(package_titles);
}
result_t<info_list> resolve_dependencies( result_t<info_list> resolve_dependencies(
const config::context& config, const config::context& config,
const title_list& package_titles const title_list& package_titles

View file

@ -111,6 +111,7 @@ namespace gdpm::package_manager{
ArgumentParser update_command("update"); ArgumentParser update_command("update");
ArgumentParser search_command("search"); ArgumentParser search_command("search");
ArgumentParser export_command("export"); ArgumentParser export_command("export");
ArgumentParser purge_command("purge");
ArgumentParser list_command("list"); ArgumentParser list_command("list");
ArgumentParser link_command("link"); ArgumentParser link_command("link");
ArgumentParser clone_command("clone"); ArgumentParser clone_command("clone");
@ -150,16 +151,16 @@ namespace gdpm::package_manager{
.implicit_value(true) .implicit_value(true)
.default_value(false) .default_value(false)
.nargs(0); .nargs(0);
install_command.add_argument("--disable-sync") install_command.add_argument("--sync")
.help("enable syncing with remote before installing") .help("enable syncing with remote before installing")
.implicit_value(true) .implicit_value(true)
.default_value(false) .default_value(true)
.nargs(0); .nargs(nargs_pattern::any);
install_command.add_argument("--disable-cache") install_command.add_argument("--cache")
.help("disable caching asset data") .help("disable caching asset data")
.implicit_value(true) .implicit_value(true)
.default_value(false) .default_value(false)
.nargs(0); .nargs(nargs_pattern::any);
install_command.add_argument("--remote") install_command.add_argument("--remote")
.help("set the remote to use") .help("set the remote to use")
.nargs(1); .nargs(1);
@ -200,17 +201,25 @@ namespace gdpm::package_manager{
remove_command.add_description("remove package(s)"); remove_command.add_description("remove package(s)");
remove_command.add_argument("packages") remove_command.add_argument("packages")
.nargs(nargs_pattern::any); .nargs(nargs_pattern::any);
remove_command.add_argument("--clean"); remove_command.add_argument("--clean")
.help("clean temporary files")
.implicit_value(true)
.default_value(false)
.nargs(0);
remove_command.add_argument("-y", "--skip-prompt"); remove_command.add_argument("-y", "--skip-prompt");
remove_command.add_argument("-f", "--file") remove_command.add_argument("-f", "--file")
.help("set the file(s) to read as input") .help("set the file(s) to read as input")
.append() .append()
.nargs(nargs_pattern::at_least_one); .nargs(1);
update_command.add_description("update package(s)"); update_command.add_description("update package(s)");
update_command.add_argument("packages") update_command.add_argument("packages")
.nargs(nargs_pattern::any); .nargs(nargs_pattern::any);
update_command.add_argument("--clean"); update_command.add_argument("--clean")
.help("clean temporary files")
.implicit_value(true)
.default_value(false)
.nargs(0);
update_command.add_argument("--remote"); update_command.add_argument("--remote");
update_command.add_argument("-f", "--file") update_command.add_argument("-f", "--file")
.help("set the file(s) to read as input") .help("set the file(s) to read as input")
@ -301,8 +310,19 @@ namespace gdpm::package_manager{
clean_command.add_description("clean package(s) temporary files"); clean_command.add_description("clean package(s) temporary files");
clean_command.add_argument("packages") clean_command.add_argument("packages")
.help("package(s) to clean") .help("package(s) to clean")
.required() .nargs(nargs_pattern::any);
.nargs(nargs_pattern::at_least_one); clean_command.add_argument("-y", "--skip-prompt")
.help("skip the yes/no prompt")
.implicit_value(true)
.default_value(false)
.nargs(0);
purge_command.add_description("purge cache database");
purge_command.add_argument("-y", "--skip-prompt")
.help("skip the yes/no prompt")
.implicit_value(true)
.default_value(false)
.nargs(0);
fetch_command.add_description("fetch and sync asset data"); fetch_command.add_description("fetch and sync asset data");
fetch_command.add_argument("remote") fetch_command.add_argument("remote")
@ -362,6 +382,7 @@ namespace gdpm::package_manager{
program.add_subparser(update_command); program.add_subparser(update_command);
program.add_subparser(search_command); program.add_subparser(search_command);
program.add_subparser(export_command); program.add_subparser(export_command);
program.add_subparser(purge_command);
program.add_subparser(list_command); program.add_subparser(list_command);
program.add_subparser(link_command); program.add_subparser(link_command);
program.add_subparser(clone_command); program.add_subparser(clone_command);
@ -377,25 +398,41 @@ namespace gdpm::package_manager{
program.parse_args(argc, argv); program.parse_args(argc, argv);
// program.parse_known_args(argc, argv); // program.parse_known_args(argc, argv);
} catch(const std::runtime_error& e){ } catch(const std::runtime_error& e){
return log::error_rc(error( return log::error_rc(ec::ARGPARSE_ERROR, e.what());
constants::error::ARGPARSE_ERROR,
e.what())
);
} }
if(program.is_subcommand_used(install_command)){ if(program.is_subcommand_used(install_command)){
action = action_e::install; action = action_e::install;
if(install_command.is_used("packages")) // if(install_command.is_used("packages"))
package_titles = install_command.get<string_list>("packages"); // package_titles = install_command.get<string_list>("packages");
package_titles = get_values_from_parser(install_command);
set_if_used(install_command, config.rest_api_params.godot_version, "godot-version"); set_if_used(install_command, config.rest_api_params.godot_version, "godot-version");
set_if_used(install_command, config.clean_temporary, "clean"); set_if_used(install_command, config.clean_temporary, "clean");
set_if_used(install_command, config.enable_sync, "disable-sync"); // set_if_used(install_command, config.enable_sync, "disable-sync");
set_if_used(install_command, config.enable_cache, "disable-cache"); // set_if_used(install_command, config.enable_cache, "disable-cache");
set_if_used(install_command, params.remote_source, "remote"); set_if_used(install_command, params.remote_source, "remote");
set_if_used(install_command, config.jobs, "jobs"); set_if_used(install_command, config.jobs, "jobs");
set_if_used(install_command, config.skip_prompt, "skip-prompt"); set_if_used(install_command, config.skip_prompt, "skip-prompt");
set_if_used(install_command, params.input_files, "file"); set_if_used(install_command, params.input_files, "file");
set_if_used(install_command, config.timeout, "timeout"); set_if_used(install_command, config.timeout, "timeout");
if(install_command.is_used("sync")){
string sync = install_command.get<string>("sync");
if(!sync.compare("enable") || !sync.compare("true") || sync.empty()){
config.enable_sync = true;
}
else if(!sync.compare("disable") || !sync.compare("false")){
config.enable_sync = false;
}
}
if(install_command.is_used("cache")){
string cache = install_command.get<string>("sync");
if(!cache.compare("enable") || !cache.compare("true") || cache.empty()){
config.enable_sync = true;
}
else if(!cache.compare("disable") || !cache.compare("false")){
config.enable_sync = false;
}
}
} }
else if(program.is_subcommand_used(get_command)){ else if(program.is_subcommand_used(get_command)){
action = action_e::get; action = action_e::get;
@ -437,6 +474,15 @@ namespace gdpm::package_manager{
action = action_e::p_export; action = action_e::p_export;
params.paths = export_command.get<string_list>("paths"); params.paths = export_command.get<string_list>("paths");
} }
else if(program.is_subcommand_used(clean_command)){
action = action_e::clean;
package_titles = get_values_from_parser(clean_command);
set_if_used(clean_command, config.skip_prompt, "skip-prompt");
}
else if(program.is_subcommand_used(purge_command)){
action = action_e::purge;
set_if_used(purge_command, config.skip_prompt, "skip-prompt");
}
else if(program.is_subcommand_used(list_command)){ else if(program.is_subcommand_used(list_command)){
action = action_e::list; action = action_e::list;
if(list_command.is_used("show")) if(list_command.is_used("show"))
@ -471,10 +517,6 @@ namespace gdpm::package_manager{
params.paths = clone_command.get<string_list>("path"); params.paths = clone_command.get<string_list>("path");
} }
} }
else if(program.is_subcommand_used(clean_command)){
action = action_e::clean;
package_titles = get_values_from_parser(clean_command);
}
else if(program.is_subcommand_used(config_command)){ else if(program.is_subcommand_used(config_command)){
if(config_command.is_used("style")){ if(config_command.is_used("style")){
string style = config_command.get<string>("style"); string style = config_command.get<string>("style");
@ -550,15 +592,16 @@ namespace gdpm::package_manager{
case action_e::update: package::update(config, package_titles, params); break; case action_e::update: package::update(config, package_titles, params); break;
case action_e::search: package::search(config, package_titles, params); break; case action_e::search: package::search(config, package_titles, params); break;
case action_e::p_export: package::export_to(params.paths); break; case action_e::p_export: package::export_to(params.paths); break;
case action_e::purge: package::purge(config); break;
case action_e::list: package::list(config, params); break; case action_e::list: package::list(config, params); break;
/* ...opts are the paths here */ /* ...opts are the paths here */
case action_e::link: package::link(config, package_titles, params); break; case action_e::link: package::link(config, package_titles, params); break;
case action_e::clone: package::clone(config, package_titles, params); break; case action_e::clone: package::clone(config, package_titles, params); break;
case action_e::clean: package::clean_temporary(config, package_titles); break; case action_e::clean: package::clean(config, package_titles); break;
case action_e::config_get: config::print_properties(config, params.args); break; case action_e::config_get: config::print_properties(config, params.args); break;
case action_e::config_set: config::set_property(config, params.args[0], params.args[1]); break; case action_e::config_set: config::set_property(config, params.args[0], params.args[1]); break;
case action_e::fetch: package::synchronize_database(config, package_titles); break; case action_e::fetch: package::fetch(config, package_titles); break;
case action_e::sync: package::synchronize_database(config, package_titles); break; case action_e::sync: package::fetch(config, package_titles); break;
case action_e::remote_list: remote::print_repositories(config); break; case action_e::remote_list: remote::print_repositories(config); break;
case action_e::remote_add: remote::add_repository(config, params.args); break; case action_e::remote_add: remote::add_repository(config, params.args); break;
case action_e::remote_remove: remote::remove_respositories(config, params.args); break; case action_e::remote_remove: remote::remove_respositories(config, params.args); break;

View file

@ -119,7 +119,10 @@ namespace gdpm::rest_api{
return request_url; return request_url;
} }
error print_params(const request_params& params, const string& filter){ error print_params(
const request_params& params,
const string& filter
){
log::println("params: \n" log::println("params: \n"
"\ttype: {}\n" "\ttype: {}\n"
"\tcategory: {}\n" "\tcategory: {}\n"
@ -147,7 +150,7 @@ namespace gdpm::rest_api{
Document doc = rest_api::get_assets_list(request_url, rest_api_params); Document doc = rest_api::get_assets_list(request_url, rest_api_params);
if(doc.IsNull()){ if(doc.IsNull()){
return log::error_rc(error( return log::error_rc(error(
constants::error::HOST_UNREACHABLE, ec::HOST_UNREACHABLE,
"Could not fetch metadata. Aborting." "Could not fetch metadata. Aborting."
)); ));
} }
@ -166,9 +169,9 @@ namespace gdpm::rest_api{
http::context http; http::context http;
string request_url{url}; string request_url{url};
request_url += to_string(type); request_url += to_string(type);
http::response r = http.request_get(url); http::response r = http.request(url);
if(verbose > 0) if(verbose > 0)
log::info("URL: {}", url); log::info("rest_api::configure::url: {}", url);
return _parse_json(r.body); return _parse_json(r.body);
} }
@ -179,15 +182,15 @@ namespace gdpm::rest_api{
const string& filter const string& filter
){ ){
http::context http; http::context http;
http::request_params http_params; http::request params;
http_params.headers.insert(http::header("Accept", "*/*")); params.headers.insert(http::header("Accept", "*/*"));
http_params.headers.insert(http::header("Accept-Encoding", "application/gzip")); params.headers.insert(http::header("Accept-Encoding", "application/gzip"));
http_params.headers.insert(http::header("Content-Encoding", "application/gzip")); params.headers.insert(http::header("Content-Encoding", "application/gzip"));
http_params.headers.insert(http::header("Connection", "keep-alive")); params.headers.insert(http::header("Connection", "keep-alive"));
string request_url = _prepare_request(url, c, http.url_escape(filter)); string prepared_url = _prepare_request(url, c, http.url_escape(filter));
http::response r = http.request_get(request_url, http_params); http::response r = http.request(prepared_url, params);
if(c.verbose >= log::INFO) if(c.verbose >= log::INFO)
log::info("rest_api::get_asset_list()::URL: {}", request_url); log::info("rest_api::get_asset_list()::url: {}", prepared_url);
return _parse_json(r.body, c.verbose); return _parse_json(r.body, c.verbose);
} }
@ -200,16 +203,18 @@ namespace gdpm::rest_api{
){ ){
/* Set up HTTP request */ /* Set up HTTP request */
http::context http; http::context http;
http::request_params http_params; http::request params;
http_params.headers.insert(http::header("Accept", "*/*")); params.headers.insert(http::header("Accept", "*/*"));
http_params.headers.insert(http::header("Accept-Encoding", "application/gzip")); params.headers.insert(http::header("Accept-Encoding", "application/gzip"));
http_params.headers.insert(http::header("Content-Encoding", "application/gzip")); params.headers.insert(http::header("Content-Encoding", "application/gzip"));
http_params.headers.insert(http::header("Connection", "keep-alive")); params.headers.insert(http::header("Connection", "keep-alive"));
string request_url = utils::replace_all(_prepare_request(url, api_params, http.url_escape(filter)), "{id}", std::to_string(asset_id)); string prepared_url = utils::replace_all(
http::response r = http.request_get(request_url.c_str(), http_params); _prepare_request(url, api_params,
http.url_escape(filter)
), "{id}", std::to_string(asset_id));
http::response r = http.request(prepared_url, params);
if(api_params.verbose >= log::INFO) if(api_params.verbose >= log::INFO)
log::info("get_asset().URL: {}", request_url); log::info("rest_api::get_asset()::url: {}", prepared_url);
return _parse_json(r.body); return _parse_json(r.body);
} }
@ -228,27 +233,56 @@ namespace gdpm::rest_api{
return false; return false;
} }
namespace multi{
json::documents get_assets(
const string_list& urls,
id_list asset_ids,
const request_params& api_params,
const string_list& filters
){
if(urls.size() == asset_ids.size() && urls.size() == filters.size()){
log::error(ec::ASSERTION_FAILED,
"multi::get_assets(): urls.size() != filters.size()"
);
}
http::multi http;
http::request params;
json::documents docs;
params.headers.insert(http::header("Accept", "*/*"));
params.headers.insert(http::header("Accept-Encoding", "application/gzip"));
params.headers.insert(http::header("Content-Encoding", "application/gzip"));
params.headers.insert(http::header("Connection", "keep-alive"));
string_list prepared_urls = {};
/* Prepare the URLs for the request_multi() call z*/
for(size_t i = 0; i < urls.size(); i++){
const string& url = urls.at(i);
const string& filter = filters.at(i);
int asset_id = asset_ids.at(i);
string prepared_url = utils::replace_all(
_prepare_request(url, api_params, http.url_escape(filter)),
"{id}", std::to_string(asset_id));
prepared_urls.emplace_back(prepared_url);
if(api_params.verbose >= log::INFO)
log::info("get_assets(i={})::url: {}", i, prepared_url);
}
/* Parse JSON string into objects */
ptr<http::transfers> transfers = http.make_requests(prepared_urls, params);
ptr<http::responses> responses = http.execute(std::move(transfers));
for(const auto& response : *responses){
docs.emplace_back(_parse_json(response.body));
}
return docs;
}
}
namespace edits{ namespace edits{
void edit_asset(){}
void edit_asset(){ void get_asset_edit(int asset_id){}
string review_asset_edit(int asset_id){ return string(); }
} string accept_asset_edit(int asset_id){ return string(); }
string reject_asset_edit(int asset_id){ return string(); }
void get_asset_edit(int asset_id){
}
string review_asset_edit(int asset_id){
return string();
}
string accept_asset_edit(int asset_id){
return string();
}
string reject_asset_edit(int asset_id){
return string();
}
} // namespace edits } // namespace edits
} }

View file

@ -2,11 +2,9 @@
#include "utils.hpp" #include "utils.hpp"
#include "config.hpp" #include "config.hpp"
#include "constants.hpp" #include "constants.hpp"
#include "error.hpp"
#include "log.hpp" #include "log.hpp"
#include "indicators/indeterminate_progress_bar.hpp"
#include "indicators/dynamic_progress.hpp"
#include "indicators/progress_bar.hpp"
#include "indicators/block_progress_bar.hpp"
#include "csv2/reader.hpp" #include "csv2/reader.hpp"
@ -30,38 +28,13 @@
namespace gdpm::utils{ namespace gdpm::utils{
using namespace indicators;
BlockProgressBar bar {
option::BarWidth{50},
// option::Start{"["},
// option::Fill{"="},
// option::Lead{">"},
// option::Remainder{" "},
// option::End{"]"},
option::PrefixText{"Downloading file "},
option::PostfixText{""},
option::ForegroundColor{Color::green},
option::FontStyles{std::vector<FontStyle>{FontStyle::bold}},
};
// option::ShowElapsedTime{true},
// option::ShowRemainingTime{true},
IndeterminateProgressBar bar_unknown {
option::BarWidth{50},
option::Start{"["},
option::Fill{"."},
option::Lead{"<==>"},
option::PrefixText{"Downloading file "},
option::End{"]"},
option::PostfixText{""},
option::ForegroundColor{Color::green},
option::FontStyles{std::vector<FontStyle>{FontStyle::bold}},
};
bool to_bool(const std::string& s){
bool to_bool(const string& s){
return to_lower(s) == "true"; return to_lower(s) == "true";
} }
std::vector<std::string> split_lines(const std::string& contents){ std::vector<string> split_lines(const string& contents){
using namespace csv2; using namespace csv2;
csv2::Reader< csv2::Reader<
delimiter<'\n'>, delimiter<'\n'>,
@ -69,7 +42,7 @@ namespace gdpm::utils{
first_row_is_header<false>, first_row_is_header<false>,
trim_policy::trim_whitespace trim_policy::trim_whitespace
> csv; > csv;
std::vector<std::string> lines; std::vector<string> lines;
if(csv.parse(contents)){ if(csv.parse(contents)){
for(const auto& row : csv){ for(const auto& row : csv){
for(const auto& cell : row){ for(const auto& cell : row){
@ -82,13 +55,13 @@ namespace gdpm::utils{
#if (GDPM_READFILE_IMPL == 0) #if (GDPM_READFILE_IMPL == 0)
std::string readfile(const std::string& path){ string readfile(const string& path){
constexpr auto read_size = std::size_t{4096}; constexpr auto read_size = std::size_t{4096};
auto stream = std::ifstream{path.data()}; auto stream = std::ifstream{path.data()};
stream.exceptions(std::ios_base::badbit); stream.exceptions(std::ios_base::badbit);
auto out = std::string{}; auto out = string{};
auto buf = std::string(read_size, '\0'); auto buf = string(read_size, '\0');
while (stream.read(& buf[0], read_size)) { while (stream.read(& buf[0], read_size)) {
out.append(buf, 0, stream.gcount()); out.append(buf, 0, stream.gcount());
} }
@ -97,15 +70,15 @@ namespace gdpm::utils{
} }
#elif(GDPM_READFILE_IMPL == 1) #elif(GDPM_READFILE_IMPL == 1)
std::string readfile(const std::string& path){ string readfile(const string& path){
std::ifstream ifs(path); std::ifstream ifs(path);
return std::string( return string(
(std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>(ifs)),
(std::istreambuf_iterator<char>()) (std::istreambuf_iterator<char>())
); );
} }
#elif(GDPM_READFILE_IMPL == 2) #elif(GDPM_READFILE_IMPL == 2)
std::string readfile(const std::string& path){ string readfile(const string& path){
std::ifstream ifs(path); std::ifstream ifs(path);
std::stringstream buffer; std::stringstream buffer;
buffer << ifs.rdbuf(); buffer << ifs.rdbuf();
@ -113,43 +86,43 @@ namespace gdpm::utils{
} }
#endif #endif
std::string to_lower(const std::string& s){ string to_lower(const string& s){
std::string copy = s; string copy = s;
std::transform(copy.begin(), copy.end(), copy.begin(), tolower); std::transform(copy.begin(), copy.end(), copy.begin(), tolower);
return copy; return copy;
} }
std::string trim(const std::string& s){ string trim(const string& s){
return trim_right(trim_left(s)); return trim_right(trim_left(s));
} }
std::string trim_left(const std::string& s){ string trim_left(const string& s){
return trim_left(s, constants::WHITESPACE); return trim_left(s, constants::WHITESPACE);
} }
std::string trim_left( string trim_left(
const std::string& s, const string& s,
const std::string& ref const string& ref
){ ){
size_t start = s.find_first_not_of(ref); size_t start = s.find_first_not_of(ref);
return (start == std::string::npos) ? "" : s.substr(start); return (start == string::npos) ? "" : s.substr(start);
} }
std::string trim_right(const std::string& s){ string trim_right(const string& s){
return trim_right(s, constants::WHITESPACE); return trim_right(s, constants::WHITESPACE);
} }
std::string trim_right( string trim_right(
const std::string& s, const string& s,
const std::string& ref const string& ref
){ ){
size_t end = s.find_last_not_of(ref); size_t end = s.find_last_not_of(ref);
return (end == std::string::npos) ? "" : s.substr(0, end + 1); return (end == string::npos) ? "" : s.substr(0, end + 1);
} }
std::vector<std::string> parse_lines(const std::string &s){ std::vector<string> parse_lines(const string &s){
std::string line; string line;
std::vector<std::string> result; std::vector<string> result;
std::stringstream ss(s); std::stringstream ss(s);
while(std::getline(ss, line)){ while(std::getline(ss, line)){
result.emplace_back(line); result.emplace_back(line);
@ -157,26 +130,26 @@ namespace gdpm::utils{
return result; return result;
} }
std::string replace_first( string replace_first(
const std::string &s, const string &s,
const std::string &from, const string &from,
const std::string &to const string &to
){ ){
std::string copy = s; // make string copy string copy = s; // make string copy
size_t pos = copy.find(from); size_t pos = copy.find(from);
if(pos == std::string::npos) if(pos == string::npos)
return copy; return copy;
return copy.replace(pos, from.length(), to); return copy.replace(pos, from.length(), to);
} }
std::string replace_all( string replace_all(
const std::string& s, const string& s,
const std::string& from, const string& from,
const std::string& to const string& to
){ ){
std::string copy = s; // make string copy string copy = s; // make string copy
size_t pos = 0; size_t pos = 0;
while((pos = copy.find(from, pos)) != std::string::npos){ while((pos = copy.find(from, pos)) != string::npos){
copy.replace(pos, s.length(), to); copy.replace(pos, s.length(), to);
pos += to.length(); pos += to.length();
} }
@ -184,45 +157,47 @@ namespace gdpm::utils{
} }
/* Ref: https://gist.github.com/mobius/1759816 */ /* Ref: https://gist.github.com/mobius/1759816 */
int extract_zip( error extract_zip(
const char *archive, const char *archive,
const char *dest, const char *dest,
int verbose int verbose
){ ){
constexpr const char *prog = "gpdm"; constexpr int SIZE = 1024;
struct zip *zip; struct zip *za;
struct zip_file *zf; struct zip_file *zf;
struct zip_stat sb; struct zip_stat sb;
char buf[100]; char buf[SIZE];
int err; int err;
int i, len, fd; int i, len, fd;
zip_uint64_t sum; zip_uint64_t sum;
// log::info_n("Extracting package contents to '{}'...", dest); std::filesystem::path path(archive);
log::info_n("Extracting package contents..."); log::info_n("Extracting \"{}\" archive...", path.filename().string());
if((zip = zip_open(archive, 0, &err)) == nullptr){ if((za = zip_open(path.c_str(), ZIP_RDONLY, &err)) == NULL){
zip_error_to_str(buf, sizeof(buf), err, errno); zip_error_to_str(buf, sizeof(buf), err, errno);
log::error("{}: can't open zip archive {}: {}", prog, archive, buf); log::println("");
return 1; return log::error_rc(error(
ec::LIBZIP_ERR,
std::format("utils::extract_zip(): can't open zip archive \"{}\": {}", path.filename().string(), buf))
);
} }
for(i = 0; i < zip_get_num_entries(zip, 0); i++){ for(i = 0; i < zip_get_num_entries(za, 0); i++){
if(zip_stat_index(zip, i, 0, &sb) == 0){ if(zip_stat_index(za, i, 0, &sb) == 0){
len = strlen(sb.name); len = strlen(sb.name);
if(verbose > 1){ if(verbose > 1){
log::print("{}, ", sb.name); log::println("utils::extract_zip(): {}, size: {}", sb.name, sb.size);
log::println("size: {}, ", sb.size);
} }
std::string path{dest}; string path{dest};
path += sb.name; path += sb.name;
if(sb.name[len-1] == '/'){ if(sb.name[len-1] == '/'){
// safe_create_dir(sb.name);
std::filesystem::create_directory(path); std::filesystem::create_directory(path);
} else { } else {
zf = zip_fopen_index(zip, i, 0); zf = zip_fopen_index(za, i, 0);
if(!zf){ if(!zf){
log::error("extract_zip: zip_fopen_index() failed."); return log::error_rc(error(ec::LIBZIP_ERR,
return 100; "utils::extract_zip(): zip_fopen_index() failed.")
);
} }
#ifdef _WIN32 #ifdef _WIN32
fd = open(sb.name, O_RDWR | O_TRUNC | O_CREAT | O_BINARY, 0644); fd = open(sb.name, O_RDWR | O_TRUNC | O_CREAT | O_BINARY, 0644);
@ -230,16 +205,19 @@ namespace gdpm::utils{
fd = open(path.c_str(), O_RDWR | O_TRUNC | O_CREAT, 0644); fd = open(path.c_str(), O_RDWR | O_TRUNC | O_CREAT, 0644);
#endif #endif
if(fd < 0){ if(fd < 0){
log::error("extract_zip: open() failed. (path: {}, fd={})", path, fd); log::error_rc(error(ec::LIBZIP_ERR,
return 101; std::format("utils::extract_zip(): open() failed. (path: {}, fd={})", path, fd))
);
} }
sum = 0; sum = 0;
while(sum != sb.size){ while(sum != sb.size){
len = zip_fread(zf, buf, 100); len = zip_fread(zf, buf, 100);
if(len < 0){ if(len < 0){
log::error("extract_zip: zip_fread() returned len < 0 (len={})", len); return log::error_rc(error(
return 102; ec::LIBZIP_ERR,
std::format("utils::extract_zip(): zip_fread() returned len < 0 (len={})", len))
);
} }
write(fd, buf, len); write(fd, buf, len);
sum += len; sum += len;
@ -252,24 +230,23 @@ namespace gdpm::utils{
} }
} }
if(zip_close(zip) == -1){ if(zip_close(za) == -1){
log::error("{}: can't close zip archive '{}'", prog, archive); return log::error_rc(error(ec::LIBZIP_ERR,
return 1; std::format("utils::extract_zip: can't close zip archive '{}'", archive))
);
} }
log::println("Done."); return error();
return 0;
} }
std::string prompt_user(const char *message){ string prompt_user(const char *message){
log::print("{} ", message); log::print("{} ", message);
std::string input; string input;
// std::cin >> input;
getline(std::cin, input); getline(std::cin, input);
return input; return input;
} }
bool prompt_user_yn(const char *message){ bool prompt_user_yn(const char *message){
std::string input{""}; string input{""};
while( input != "y" && input != "n" ){ while( input != "y" && input != "n" ){
input = to_lower(utils::prompt_user(message)); input = to_lower(utils::prompt_user(message));
bool is_default = (input == "\0" || input == "\n" || input == "\r\n" || input.empty()); bool is_default = (input == "\0" || input == "\n" || input == "\r\n" || input.empty());
@ -287,15 +264,15 @@ namespace gdpm::utils{
// sleep_until(system_clock::now() + millis); // sleep_until(system_clock::now() + millis);
} }
std::string join( string join(
const std::vector<std::string>& target, const std::vector<string>& target,
const std::string& delimiter const string& delimiter
){ ){
std::string o; string o;
std::for_each( std::for_each(
target.begin(), target.begin(),
target.end(), target.end(),
[&o, &delimiter](const std::string& s){ [&o, &delimiter](const string& s){
o += s + delimiter; o += s + delimiter;
} }
); );
@ -303,16 +280,16 @@ namespace gdpm::utils{
return o; return o;
} }
std::string join( string join(
const std::unordered_map<std::string, std::string>& target, const std::unordered_map<string, string>& target,
const std::string& prefix, const string& prefix,
const std::string& delimiter const string& delimiter
){ ){
std::string o; string o;
std::for_each( std::for_each(
target.begin(), target.begin(),
target.end(), target.end(),
[&o, &prefix, &delimiter](const std::pair<std::string, std::string>& p){ [&o, &prefix, &delimiter](const std::pair<string, string>& p){
o += prefix + p.first + ": " + p.second + delimiter; o += prefix + p.first + ": " + p.second + delimiter;
} }
); );
@ -320,13 +297,13 @@ namespace gdpm::utils{
return o; return o;
} }
std::string convert_size(long size){ string convert_size(long size){
int digit = 0; int digit = 0;
while(size > 1000){ while(size > 1000){
size /= 1000; size /= 1000;
digit += 1; digit += 1;
} }
std::string s = std::to_string(size); string s = std::to_string(size);
switch(digit){ switch(digit){
case 0: return s + " B"; case 0: return s + " B";
case 1: return s + " KB"; case 1: return s + " KB";
@ -338,93 +315,13 @@ namespace gdpm::utils{
return std::to_string(size); return std::to_string(size);
} }
namespace curl {
size_t write_to_buffer(
char *contents,
size_t size,
size_t nmemb,
void *userdata
){
size_t realsize = size * nmemb;
struct memory_buffer *m = (struct memory_buffer*)userdata;
m->addr = (char*)realloc(m->addr, m->size + realsize + 1);
if(m->addr == nullptr){
/* Out of memory */
fprintf(stderr, "Not enough memory (realloc returned NULL)\n");
return 0;
}
memcpy(&(m->addr[m->size]), contents, realsize);
m->size += realsize;
m->addr[m->size] = 0;
return realsize;
}
size_t write_to_stream(
char *ptr,
size_t size,
size_t nmemb,
void *userdata
){
if(nmemb == 0)
return 0;
return fwrite(ptr, size, nmemb, (FILE*)userdata);
}
int show_progress(
void *ptr,
curl_off_t total_download,
curl_off_t current_downloaded,
curl_off_t total_upload,
curl_off_t current_upload
){
if(current_downloaded >= total_download)
return 0;
using namespace indicators;
show_console_cursor(false);
if(total_download != 0){
// double percent = std::floor((current_downloaded / (total_download)) * 100);
bar.set_option(option::MaxProgress{total_download});
// bar.set_option(option::HideBarWhenComplete{false});
bar.set_progress(current_downloaded);
bar.set_option(option::PostfixText{
convert_size(current_downloaded) + " / " +
convert_size(total_download)
});
if(bar.is_completed()){
bar.set_option(option::PrefixText{"Download completed."});
bar.mark_as_completed();
}
} else {
if(bar_unknown.is_completed()){
bar_unknown.set_option(option::PrefixText{"Download completed."});
bar_unknown.mark_as_completed();
} else {
bar.tick();
bar_unknown.set_option(
option::PostfixText(std::format("{}", convert_size(current_downloaded)))
);
}
}
show_console_cursor(true);
memory_buffer *m = (memory_buffer*)ptr;
return 0;
}
}
namespace json { namespace json {
std::string from_array( string from_array(
const std::set<std::string>& a, const std::set<string>& a,
const std::string& prefix const string& prefix
){ ){
std::string o{"["}; string o{"["};
for(const std::string& src : a) for(const string& src : a)
o += prefix + "\t\"" + src + "\","; o += prefix + "\t\"" + src + "\",";
if(o.back() == ',') if(o.back() == ',')
o.pop_back(); o.pop_back();
@ -433,14 +330,14 @@ namespace gdpm::utils{
}; };
std::string from_object( string from_object(
const std::unordered_map<std::string, std::string>& m, const std::unordered_map<string, string>& m,
const std::string& prefix, const string& prefix,
const std::string& spaces const string& spaces
){ ){
std::string o{"{"}; string o{"{"};
std::for_each(m.begin(), m.end(), std::for_each(m.begin(), m.end(),
[&o, &prefix, &spaces](const std::pair<std::string, std::string>& p){ [&o, &prefix, &spaces](const std::pair<string, string>& p){
o += std::format("{}\t\"{}\":{}\"{}\",", prefix, p.first, spaces, p.second); o += std::format("{}\t\"{}\":{}\"{}\",", prefix, p.first, spaces, p.second);
} }
); );