towk/gdpm
1
0
Fork 0
mirror of https://github.com/davidallendj/gdpm.git synced 2025-12-20 03:27:02 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixed not being able to download in parallel

Browse source 
- Removed `transfer` class in `http` namespace
- Consolidated the `http::context` and `http::multi` classes
This commit is contained in:
David Allen 2023-07-15 18:56:14 -06:00
parent 8ac3910730
commit 9d5c52f450
6 changed files with 346 additions and 244 deletions
Download patch file Download diff file Expand all files Collapse all files

3
include/constants.hpp
View file

@ -59,6 +59,9 @@ namespace gdpm::print{
#define GDPM_DEFAULT_ASSET_REVERSE false #define GDPM_DEFAULT_ASSET_REVERSE false
#define GDPM_DEFAULT_ASSET_VERBOSE 0 #define GDPM_DEFAULT_ASSET_VERBOSE 0
#define GDPM_MIN_JOBS 1
#define GDPM_MAX_JOBS 1024
/* Define misc. macros */ /* Define misc. macros */
#if defined(_WIN32) #if defined(_WIN32)
#define GDPM_DLL_EXPORT __declspec(dllexport) #define GDPM_DLL_EXPORT __declspec(dllexport)

55
include/http.hpp
View file

@ -136,55 +136,50 @@ namespace gdpm::http{
// option::ForegroundColor{Color::green}, // option::ForegroundColor{Color::green},
// option::FontStyles{std::vector<FontStyle>{FontStyle::bold}}, // option::FontStyles{std::vector<FontStyle>{FontStyle::bold}},
// }; // };
struct transfer : public non_copyable{
transfer(){ curl = curl_easy_init(); }
transfer(transfer&&){}
~transfer(){ }
CURLcode res;
int id;
CURL *curl = nullptr;
FILE *fp = nullptr;
utils::memory_buffer data = {0};
};
using transfers = std::vector<transfer>;
using responses = std::vector<response>; using responses = std::vector<response>;
class context : public non_copyable{ class context : public non_copyable{
public: public:
context(); context(int max_transfers = 1);
~context(); ~context();
string url_escape(const string& url); string url_escape(const string& url);
response request(const string& url, const http::request& params = http::request()); response request(const string& url, const http::request& params = http::request());
responses requests(const string_list& urls, const http::request& params = http::request());
response download_file(const string& url, const string& storage_path, const http::request& params = http::request()); response download_file(const string& url, const string& storage_path, const http::request& params = http::request());
responses download_files(const string_list& url, const string_list& storage_path, const http::request& params = http::request());
long get_download_size(const string& url); long get_download_size(const string& url);
long get_bytes_downloaded(const string& url); long get_bytes_downloaded(const string& url);
void set_max_transfers(int max_transfers);
private: private:
CURL *curl = nullptr; CURL *curl = nullptr;
};
class multi{
public:
multi(long max_allowed_transfers = 2);
~multi();
string url_escape(const string& url);
ptr<transfers> make_requests(const string_list& urls, const http::request& params = http::request());
ptr<transfers> make_downloads(const string_list& url, const string_list& storage_path, const http::request& params = http::request());
ptr<responses> execute(ptr<transfers> transfers, size_t timeout = 1000);
private:
DynamicProgress<BlockProgressBar> progress_bars;
CURLM *cm = nullptr; CURLM *cm = nullptr;
CURLMsg *cmessage = nullptr; CURLMsg *cmessage = nullptr;
CURLMcode cres; CURLMcode cres;
int messages_left = -1; int max_transfers = 1;
int transfers_index = 0;
int transfers_left = -1;
DynamicProgress<BlockProgressBar> progress_bars;
}; };
// class multi{
// public:
// multi(long max_allowed_transfers = 2);
// ~multi();
// string url_escape(const string& url);
// ptr<responses> execute(ptr<transfers> transfers, size_t timeout = 1000);
// void cleanup();
// private:
// CURLM *cm = nullptr;
// CURLMsg *cmessage = nullptr;
// CURLMcode cres;
// int transfers_left = -1;
// };
curl_slist* add_headers(CURL *curl, const headers_t& headers); curl_slist* add_headers(CURL *curl, const headers_t& headers);
static size_t write_to_buffer(char *contents, size_t size, size_t nmemb, void *userdata); static size_t write_to_buffer(char *contents, size_t size, size_t nmemb, void *userdata);
static size_t write_to_stream(char *ptr, size_t size, size_t nmemb, void *userdata); static size_t write_to_stream(char *ptr, size_t size, size_t nmemb, void *userdata);

456
src/http.cpp
View file

@ -3,6 +3,7 @@
#include "utils.hpp" #include "utils.hpp"
#include "log.hpp" #include "log.hpp"
#include "error.hpp" #include "error.hpp"
#include <atomic>
#include <curl/curl.h> #include <curl/curl.h>
#include <curl/easy.h> #include <curl/easy.h>
#include <curl/multi.h> #include <curl/multi.h>
@ -14,14 +15,17 @@
namespace gdpm::http{ namespace gdpm::http{
context::context(){ context::context(int max_transfers): max_transfers(max_transfers){
curl_global_init(CURL_GLOBAL_ALL); curl_global_init(CURL_GLOBAL_ALL);
curl = curl_easy_init(); curl = curl_easy_init();
cm = curl_multi_init();
set_max_transfers(max_transfers);
} }
context::~context(){ context::~context(){
curl_easy_cleanup(curl); curl_easy_cleanup(curl);
curl_multi_cleanup(cm);
curl_global_cleanup(); curl_global_cleanup();
} }
@ -39,11 +43,6 @@ namespace gdpm::http{
utils::memory_buffer buf = utils::make_buffer(); utils::memory_buffer buf = utils::make_buffer();
utils::memory_buffer data = utils::make_buffer(); utils::memory_buffer data = utils::make_buffer();
response r; response r;
#if (GDPM_DELAY_HTTP_REQUESTS == 1)
using namespace std::chrono_literals;
utils::delay();
#endif
if(curl){ if(curl){
curl_slist *list = add_headers(curl, params.headers); curl_slist *list = add_headers(curl, params.headers);
if(params.method == method::POST){ if(params.method == method::POST){
@ -87,6 +86,122 @@ namespace gdpm::http{
} }
responses context::requests(
const string_list& urls,
const http::request& params
){
if(cm == nullptr){
log::error(error(PRECONDITION_FAILED,
"http::multi::make_downloads(): multi client not initialized.")
);
}
if(urls.size() <= 0){
log::warn("No requests to make.");
}
transfers_left = urls.size();
transfers_index = 0;
auto add_transfer = [this](const string& url, const http::request& params)-> void {
CURL *curl = curl_easy_init();
utils::memory_buffer *data;
if(curl){
curl_slist *list = add_headers(curl, params.headers);
if(params.method == method::POST){
string h;
std::for_each(
params.headers.begin(),
params.headers.end(),
[&h](const string_pair& kv){
h += kv.first + "=" + kv.second + "&";
}
);
h.pop_back();
h = url_escape(h);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, h.size());
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, h.c_str());
}
else if(params.method == method::GET){
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
}
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&data);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_to_buffer);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, show_download_progress);
curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
}
transfers_index += 1;
};
for(int i = 0; i < max_transfers; i++){
add_transfer(urls.at(i), params);
}
do{
int still_running = 1;
int numfds = 0;
cres = curl_multi_perform(cm, &still_running);
if(cres == CURLM_OK){
/* wait for activity, timeout or "nothing" */
cres = curl_multi_poll(cm, NULL, 0, params.timeout, &numfds);
}
if(cres != CURLM_OK){
log::error(error(
ec::LIBCURL_ERR,
"http::multi::execute(): curl_multi_poll() returned an error"
));
break;
}
int messages_left = -1;
while((cmessage = curl_multi_info_read(cm, &messages_left))){
if(cmessage->msg == CURLMSG_DONE){
char *url;
CURL *eh = cmessage->easy_handle;
int code = 0;
curl_easy_getinfo(cmessage->easy_handle, CURLINFO_EFFECTIVE_URL, &url);
curl_easy_getinfo(cmessage->easy_handle, CURLINFO_RESPONSE_CODE, &code);
if((int)cmessage->data.result != CURLM_OK){
log::error(error(ec::LIBCURL_ERR,
std::format("http::context::execute({}): {} <url: {}>",
(int)cmessage->data.result, curl_easy_strerror(cmessage->data.result), url))
);
}
log::println("transfers left: {}", transfers_left);
cres = curl_multi_remove_handle(cm, eh);
if(cres != CURLM_OK){
log::error("http::context::execute(): curl_multi_remove_handle() returned error {}", cres);
}
curl_easy_cleanup(eh);
transfers_left -= 1;
}
else{
log::error(error(ec::LIBCURL_ERR,
std::format("http::context::execute(): {}", (int)cmessage->msg))
);
}
if(transfers_index < urls.size()-1){
add_transfer(urls.at(transfers_index), params);
}
}
if(transfers_left){
cres = curl_multi_wait(cm, NULL, 0, params.timeout, NULL);
if(cres != CURLM_OK){
log::error("http::context::execute(): curl_multi_wait() returned an error {}", cres);
}
}
}while(transfers_left);
cres = curl_multi_cleanup(cm);
if(cres != CURLM_OK){
log::error("http::context::execute(): curl_multi_cleanup() returned an error {}", cres);
}
return responses();
}
response context::download_file( response context::download_file(
const string& url, const string& url,
const string& storage_path, const string& storage_path,
@ -97,10 +212,6 @@ namespace gdpm::http{
response r; response r;
FILE *fp; FILE *fp;
#if (GDPM_DELAY_HTTP_REQUESTS == 1)
using namespace std::chrono_literals;
utils::delay();
#endif
if(curl){ if(curl){
fp = fopen(storage_path.c_str(), "wb"); fp = fopen(storage_path.c_str(), "wb");
utils::memory_buffer *data; utils::memory_buffer *data;
@ -131,6 +242,124 @@ namespace gdpm::http{
return r; return r;
} }
responses context::download_files(
const string_list &urls,
const string_list &storage_paths,
const http::request& params
){
if(cm == nullptr){
log::error(error(ec::PRECONDITION_FAILED,
"http::multi::make_downloads(): multi client not initialized.")
);
return responses();
}
if(urls.size() != storage_paths.size()){
log::error(error(ec::ASSERTION_FAILED,
"http::context::make_downloads(): urls.size() != storage_paths.size()"
));
}
transfers_left = urls.size();
transfers_index = 0;
auto add_transfer = [this](const string& url, const string& storage_path, const http::request& params){
CURL *curl = curl_easy_init();
utils::memory_buffer *data;
if(curl){
FILE *fp = fopen(storage_path.c_str(), "wb");
curl_slist *list = add_headers(curl, params.headers);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(curl, CURLOPT_HEADER, 0);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(curl, CURLOPT_PRIVATE, fp);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_to_stream);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, show_download_progress);
curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
if(params.verbose >= log::INFO){
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
}
cres = curl_multi_add_handle(cm, curl);
curl_slist_free_all(list);
if(cres != CURLM_OK){
log::error(ec::LIBCURL_ERR,
std::format("http::context::make_downloads(): {}", curl_multi_strerror(cres))
);
}
}
transfers_index += 1;
};
for(size_t i = 0; i < urls.size(); i++){
add_transfer(urls.at(i), storage_paths.at(i), params);
}
int still_running = 1;
int numfds = 0;
do{
cres = curl_multi_perform(cm, &still_running);
if(cres == CURLM_OK){
/* wait for activity, timeout or "nothing" */
cres = curl_multi_poll(cm, NULL, 0, 1000, &numfds);
}
if(cres != CURLM_OK){
log::error(error(
ec::LIBCURL_ERR,
"http::multi::execute(): curl_multi_poll() returned an error"
));
break;
}
int messages_left = -1;
while((cmessage = curl_multi_info_read(cm, &messages_left))){
if(cmessage->msg == CURLMSG_DONE){
char *url = nullptr;
FILE *fp = nullptr;
CURL *eh = cmessage->easy_handle;
int code = 0;
curl_easy_getinfo(cmessage->easy_handle, CURLINFO_EFFECTIVE_URL, &url);
curl_easy_getinfo(cmessage->easy_handle, CURLINFO_RESPONSE_CODE, &code);
curl_easy_getinfo(cmessage->easy_handle, CURLINFO_PRIVATE, &fp);
if((int)cmessage->data.result != CURLM_OK){
log::error(error(ec::LIBCURL_ERR,
std::format("http::context::execute({}): {} <url: {}>", (int)cmessage->data.result, curl_easy_strerror(cmessage->data.result), url))
);
}
cres = curl_multi_remove_handle(cm, eh);
if(cres != CURLM_OK){
log::error("http::context::execute(): curl_multi_remove_handle() returned error {}", cres);
}
transfers_left -= 1;
curl_easy_cleanup(eh);
if(fp)
fclose(fp);
}
else{
log::error(error(ec::LIBCURL_ERR,
std::format("http::context::execute(): {}", (int)cmessage->msg))
);
}
if(transfers_index < urls.size()-1){
add_transfer(urls.at(transfers_index), storage_paths.at(transfers_index), params);
}
}
if(transfers_left){
cres = curl_multi_wait(cm, NULL, 0, params.timeout, NULL);
if(cres != CURLM_OK){
log::error("http::context::execute(): curl_multi_wait() returned an error {}", cres);
}
}
}while(transfers_left);
cres = curl_multi_cleanup(cm);
if(cres != CURLM_OK){
log::error("http::context::execute(): curl_multi_cleanup() returned an error {}", cres);
}
return responses();
}
long context::get_download_size(const string& url){ long context::get_download_size(const string& url){
CURLcode res; CURLcode res;
if(curl){ if(curl){
@ -166,182 +395,55 @@ namespace gdpm::http{
} }
multi::multi(long max_allowed_transfers){ void context::set_max_transfers(int max_transfers){
curl_global_init(CURL_GLOBAL_ALL); curl_multi_setopt(cm, CURLMOPT_MAX_TOTAL_CONNECTIONS, max_transfers);
if(max_allowed_transfers > 1)
cm = curl_multi_init();
curl_multi_setopt(cm, CURLMOPT_MAXCONNECTS, (long)max_allowed_transfers);
}
multi::~multi(){
if(cm != nullptr)
curl_multi_cleanup(cm);
curl_global_cleanup();
} }
string multi::url_escape(const string &url){ // multi::multi(long max_allowed_transfers){
return curl_easy_escape(cm, url.c_str(), url.size());; // curl_global_init(CURL_GLOBAL_ALL);
} // if(max_allowed_transfers > 1)
// cm = curl_multi_init();
// curl_multi_setopt(cm, CURLMOPT_MAXCONNECTS, (long)max_allowed_transfers);
// }
// multi::~multi(){
// cleanup();
// }
ptr<transfers> multi::make_requests( // string multi::url_escape(const string &url){
const string_list& urls, // return curl_easy_escape(cm, url.c_str(), url.size());;
const http::request& params // }
){
if(cm == nullptr){
log::error(error(PRECONDITION_FAILED,
"http::multi::make_downloads(): multi client not initialized.")
);
return std::make_unique<transfers>();
}
if(urls.size() <= 0){
log::warn("No requests to make.");
return std::make_unique<transfers>();
}
ptr<transfers> ts = std::make_unique<transfers>();
for(const auto& url : urls){
transfer t;
if(t.curl){
curl_slist *list = add_headers(t.curl, params.headers);
if(params.method == method::POST){
string h;
std::for_each(
params.headers.begin(),
params.headers.end(),
[&h](const string_pair& kv){
h += kv.first + "=" + kv.second + "&";
}
);
h.pop_back();
h = url_escape(h);
curl_easy_setopt(t.curl, CURLOPT_POSTFIELDSIZE, h.size());
curl_easy_setopt(t.curl, CURLOPT_POSTFIELDS, h.c_str());
}
else if(params.method == method::GET){
curl_easy_setopt(t.curl, CURLOPT_CUSTOMREQUEST, "GET");
}
curl_easy_setopt(t.curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(t.curl, CURLOPT_WRITEDATA, (void*)&t.data);
curl_easy_setopt(t.curl, CURLOPT_WRITEFUNCTION, write_to_buffer);
curl_easy_setopt(t.curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(t.curl, CURLOPT_XFERINFODATA, &t.data);
curl_easy_setopt(t.curl, CURLOPT_XFERINFOFUNCTION, show_download_progress);
curl_easy_setopt(t.curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
curl_easy_setopt(t.curl, CURLOPT_TIMEOUT_MS, params.timeout);
}
}
return ts;
}
ptr<transfers> multi::make_downloads( // ptr<responses> multi::execute(
const string_list& urls, // ptr<transfers> transfers,
const string_list& storage_paths, // size_t timeout
const http::request& params // ){
){ // if(cm == nullptr){
if(cm == nullptr){ // log::error(error(PRECONDITION_FAILED,
log::error(error(ec::PRECONDITION_FAILED, // "http::multi::execute(): multi client not initialized")
"http::multi::make_downloads(): multi client not initialized.") // );
); // return std::make_unique<responses>();
return std::make_unique<transfers>(); // }
} // // if(transfers->empty()){
if(urls.size() != storage_paths.size()){ // // log::debug("http::multi::execute(): no transfers found");
log::error(error(ec::ASSERTION_FAILED, // // return std::make_unique<responses>();
"http::context::make_downloads(): urls.size() != storage_paths.size()" // // }
)); // // size_t transfers_left = transfers->size();
} // ptr<responses> responses = std::make_unique<http::responses>(transfers->size());
ptr<transfers> ts = std::make_unique<transfers>();
for(size_t i = 0; i < urls.size(); i++){
const string& url = urls.at(i);
const string& storage_path = storage_paths.at(i);
response r;
transfer t;
t.id = i;
if(t.curl){ // }
t.fp = fopen(storage_path.c_str(), "wb");
curl_slist *list = add_headers(t.curl, params.headers);
curl_easy_setopt(t.curl, CURLOPT_URL, url.c_str());
// curl_easy_setopt(t.curl, CURLOPT_PRIVATE, url.c_str());
curl_easy_setopt(t.curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(t.curl, CURLOPT_HEADER, 0);
curl_easy_setopt(t.curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(t.curl, CURLOPT_WRITEDATA, t.fp);
curl_easy_setopt(t.curl, CURLOPT_WRITEFUNCTION, write_to_stream);
curl_easy_setopt(t.curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(t.curl, CURLOPT_XFERINFODATA, &t.data);
curl_easy_setopt(t.curl, CURLOPT_XFERINFOFUNCTION, show_download_progress);
curl_easy_setopt(t.curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
curl_easy_setopt(t.curl, CURLOPT_TIMEOUT_MS, params.timeout);
cres = curl_multi_add_handle(cm, t.curl);
curl_slist_free_all(list);
if(cres != CURLM_OK){
log::error(ec::LIBCURL_ERR,
std::format("http::context::make_downloads(): {}", curl_multi_strerror(cres))
);
}
ts->emplace_back(std::move(t));
/* NOTE: Should the file pointer be closed here? */
}
}
return ts; // void multi::cleanup(){
} // if(cm != nullptr){
// log::println("curl_multi_cleanup");
// curl_multi_cleanup(cm);
ptr<responses> multi::execute( // }
ptr<transfers> transfers, // log::println("curl_global_cleanup");
size_t timeout // curl_global_cleanup();
){ // }
if(cm == nullptr){
log::error(error(PRECONDITION_FAILED,
"http::multi::execute(): multi client not initialized")
);
return std::make_unique<responses>();
}
if(transfers->empty()){
log::debug("http::multi::execute(): no transfers found");
return std::make_unique<responses>();
}
size_t transfers_left = transfers->size();
ptr<responses> responses = std::make_unique<http::responses>(transfers->size());
do{
int still_alive = 1;
cres = curl_multi_perform(cm, &still_alive);
while((cmessage = curl_multi_info_read(cm, &messages_left))){
if(cmessage->msg == CURLMSG_DONE){
char *url = nullptr;
transfer& t = transfers->at(transfers_left-1);
response& r = responses->at(transfers_left-1);
t.curl = cmessage->easy_handle;
curl_easy_getinfo(cmessage->easy_handle, CURLINFO_EFFECTIVE_URL, &url);
curl_easy_getinfo(cmessage->easy_handle, CURLINFO_RESPONSE_CODE, &r.code);
if((int)cmessage->data.result != CURLM_OK){
log::error(error(ec::LIBCURL_ERR,
std::format("http::context::execute({}): {} <url: {}>", (int)cmessage->data.result, curl_easy_strerror(cmessage->data.result), url))
);
}
curl_multi_remove_handle(cm, t.curl);
curl_easy_cleanup(t.curl);
if(t.fp) fclose(t.fp);
transfers->pop_back();
transfers_left -= 1;
}
else{
log::error(error(ec::LIBCURL_ERR,
std::format("http::context::execute(): {}", (int)cmessage->msg))
);
}
}
if(transfers_left)
curl_multi_wait(cm, NULL, 0, timeout, NULL);
}while(transfers_left);
return responses;
}
curl_slist* add_headers( curl_slist* add_headers(

33
src/package.cpp
View file

@ -83,12 +83,6 @@ namespace gdpm::package{
table[index][3].format().font_align(FontAlign::center); table[index][3].format().font_align(FontAlign::center);
table[index][4].format().font_align(FontAlign::center); table[index][4].format().font_align(FontAlign::center);
table[index][6].format().font_align(FontAlign::center); table[index][6].format().font_align(FontAlign::center);
// string output(p.title + GDPM_COLOR_CYAN " v" + p.version + GDPM_COLOR_RESET);
// output += GDPM_COLOR_BLUE " last updated: " + p.modify_date + GDPM_COLOR_RESET;
// output += (p.is_installed) ? GDPM_COLOR_LIGHT_CYAN " (reinstall)" : "";
// output += GDPM_COLOR_RESET;
// log::print(" {}\n", output);
} }
table.print(std::cout); table.print(std::cout);
log::println(""); log::println("");
@ -177,14 +171,13 @@ namespace gdpm::package{
)); ));
} }
/* Attempt to download ZIPs in parallel */ /* Download ZIP files using download url */
if(config.jobs > 1){ if(config.jobs > 1){
http::multi http(config.jobs); http::context http(config.jobs);
ptr<http::transfers> transfers = http.make_downloads(p_download_urls, p_storage_paths); http::responses responses = http.download_files(p_download_urls, p_storage_paths);
ptr<http::responses> responses = http.execute(std::move(transfers));
/* Check for HTTP response errors */ /* Check for HTTP response errors */
for(const auto& r : *responses){ for(const auto& r : responses){
if(r.code != http::OK){ if(r.code != http::OK){
log::error(error(ec::HTTP_RESPONSE_ERR, log::error(error(ec::HTTP_RESPONSE_ERR,
std::format("HTTP error: {}", r.code) std::format("HTTP error: {}", r.code)
@ -339,15 +332,15 @@ namespace gdpm::package{
} // for loop } // for loop
/* Get the packages not found in cache and download */ /* Get the packages not found in cache and download */
{
string_list urls; string_list urls;
for(const auto& p : p_left){ for(const auto& p : p_left){
urls.emplace_back(p.download_url); urls.emplace_back(p.download_url);
} }
http::multi http; http::context http;
ptr<http::transfers> transfers = http.make_requests(urls); http::responses responses = http.requests(urls);
ptr<http::responses> responses = http.execute(std::move(transfers));
for(const auto& response : *responses){ for(const auto& response : responses){
if(response.code == http::OK){ if(response.code == http::OK){
log::println("Done."); log::println("Done.");
}else{ }else{
@ -357,6 +350,7 @@ namespace gdpm::package{
)); ));
} }
} }
}
/* Extract all packages and update cache database */ /* Extract all packages and update cache database */
for(auto& p : p_found){ for(auto& p : p_found){
@ -434,11 +428,10 @@ namespace gdpm::package{
.font_style({FontStyle::underline, FontStyle::bold}); .font_style({FontStyle::underline, FontStyle::bold});
for(const auto& p : p_cache){ for(const auto& p : p_cache){
table.add_row({p.title, p.author, p.category, p.version, p.godot_version, p.modify_date, (p.is_installed) ? "✔️": "❌"}); table.add_row({p.title, p.author, p.category, p.version, p.godot_version, p.modify_date, (p.is_installed) ? "✔️": "❌"});
// string output(p.title + GDPM_COLOR_CYAN " v" + p.version + GDPM_COLOR_RESET); size_t index = table.size() - 1;
// output += GDPM_COLOR_BLUE " last updated: " + p.modify_date + GDPM_COLOR_RESET; table[index][3].format().font_align(FontAlign::center);
// output += (p.is_installed) ? GDPM_COLOR_LIGHT_CYAN " (reinstall)" : ""; table[index][4].format().font_align(FontAlign::center);
// output += GDPM_COLOR_RESET; table[index][6].format().font_align(FontAlign::center);
// log::print(" {}\n", output);
} }
table.print(std::cout); table.print(std::cout);
log::println(""); log::println("");

10
src/package_manager.cpp
View file

@ -85,6 +85,14 @@ namespace gdpm::package_manager{
} }
}; };
template <typename T, typename String = string>
auto set_if_used_limit(
const argparse::ArgumentParser& cmd,
T& value,
const String& arg
){
}
string_list get_values_from_parser( string_list get_values_from_parser(
const argparse::ArgumentParser& cmd, const argparse::ArgumentParser& cmd,
@ -412,6 +420,8 @@ namespace gdpm::package_manager{
// set_if_used(install_command, config.enable_cache, "disable-cache"); // set_if_used(install_command, config.enable_cache, "disable-cache");
set_if_used(install_command, params.remote_source, "remote"); set_if_used(install_command, params.remote_source, "remote");
set_if_used(install_command, config.jobs, "jobs"); set_if_used(install_command, config.jobs, "jobs");
if(install_command.is_used("jobs"))
config.jobs = std::clamp(install_command.get<int>("jobs"), GDPM_MIN_JOBS, GDPM_MAX_JOBS);
set_if_used(install_command, config.skip_prompt, "skip-prompt"); set_if_used(install_command, config.skip_prompt, "skip-prompt");
set_if_used(install_command, params.input_files, "file"); set_if_used(install_command, params.input_files, "file");
set_if_used(install_command, config.timeout, "timeout"); set_if_used(install_command, config.timeout, "timeout");

7
src/rest_api.cpp
View file

@ -246,7 +246,7 @@ namespace gdpm::rest_api{
"multi::get_assets(): urls.size() != filters.size()" "multi::get_assets(): urls.size() != filters.size()"
); );
} }
http::multi http; http::context http(4);
http::request params; http::request params;
json::documents docs; json::documents docs;
params.headers.insert(http::header("Accept", "*/*")); params.headers.insert(http::header("Accept", "*/*"));
@ -269,9 +269,8 @@ namespace gdpm::rest_api{
} }
/* Parse JSON string into objects */ /* Parse JSON string into objects */
ptr<http::transfers> transfers = http.make_requests(prepared_urls, params); http::responses responses = http.requests(prepared_urls, params);
ptr<http::responses> responses = http.execute(std::move(transfers)); for(const auto& response : responses){
for(const auto& response : *responses){
docs.emplace_back(_parse_json(response.body)); docs.emplace_back(_parse_json(response.body));
} }
return docs; return docs;