Minor bug fixes and improvements

- Update `README.md` file with minor changes - Added `max_results` parameter to config for persistence - Added `csv2`, `tabulate`, and `indicators` as included dependencies - Added downloading progress bar that *sometimes* work -Added HTTP request header handling with curl - Added more options to some commands - Moved `http` functions into `context` class TODO: Fix optional parameters not working correctly NOTE: Github does not always return `Content-length` HTTP response header
2025-12-20 03:27:02 -07:00 · 2023-06-25 20:00:58 -06:00 · 2023-06-25 20:00:58 -06:00 · 460e2054c2
commit 460e2054c2
parent a3e4c054c2
15 changed files with 408 additions and 101 deletions
--- a/src/http.cpp
+++ b/src/http.cpp
@ -3,26 +3,36 @@
 #include "utils.hpp"
 #include "log.hpp"
 #include <curl/curl.h>
+#include <curl/easy.h>
 #include <stdio.h>
 #include <chrono>


 namespace gdpm::http{

-	string url_escape(const string &url){
-		CURL *curl = nullptr;
+	context::context(){
 		curl_global_init(CURL_GLOBAL_ALL);
-		char *escaped_url = curl_easy_escape(curl, url.c_str(), url.size());
-		std::string url_copy = escaped_url;
-		curl_global_cleanup();
-		return escaped_url;
+		curl = curl_easy_init();
 	}

-	response request_get(
+
+	context::~context(){
+		curl_global_cleanup();
+		curl_easy_cleanup(curl);
+	}
+
+	CURL* const context::get_curl() const{
+		return curl;
+	}
+
+	string context::url_escape(const string &url){
+		return curl_easy_escape(curl, url.c_str(), url.size());;
+	}
+
+	response context::request_get(
 		const string& url, 
 		const http::request_params& params
 	){
-		CURL *curl = nullptr;
 		CURLcode res;
 		utils::memory_buffer buf = utils::make_buffer();
 		response r;
@ -32,17 +42,23 @@ namespace gdpm::http{
 		utils::delay();
 #endif

-		curl_global_init(CURL_GLOBAL_ALL);
-		curl = curl_easy_init();
+		// curl_global_init(CURL_GLOBAL_ALL);
+		// curl = curl_easy_init();
 		if(curl){
+			utils::memory_buffer *data;
+			curl_slist *list = _add_headers(curl, params.headers);
 			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 			// curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");
 			curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
 			curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf);
-			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl_write_to_buffer);
+			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_buffer);
+			curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+			curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
+			curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress);
 			curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
 			curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
 			res = curl_easy_perform(curl);
+			curl_slist_free_all(list);
 			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code);
 			if(res != CURLE_OK && params.verbose > 0)
 				log::error("_make_request.curl_easy_perform(): {}", curl_easy_strerror(res));
@ -51,16 +67,16 @@ namespace gdpm::http{

 		r.body = buf.addr;
 		utils::free_buffer(buf);
-		curl_global_cleanup();
+		// curl_global_cleanup();
 		return r;
 	}


-	response request_post(
+	response context::request_post(
 		const string& url, 
 		const http::request_params& params
 	){
-		CURL *curl = nullptr;
+		// CURL *curl = nullptr;
 		CURLcode res;
 		utils::memory_buffer buf = utils::make_buffer();
 		response r;
@ -81,18 +97,24 @@ namespace gdpm::http{
 		h = url_escape(h);

 		// const char *post_fields = "";
-		curl_global_init(CURL_GLOBAL_ALL);
-		curl = curl_easy_init();
+		// curl_global_init(CURL_GLOBAL_ALL);
+		// curl = curl_easy_init();
 		if(curl){
+			utils::memory_buffer *data;
+			curl_slist *list = _add_headers(curl, params.headers);
 			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 			// curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");
 			curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, h.size());
 			curl_easy_setopt(curl, CURLOPT_POSTFIELDS, h.c_str());
 			curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf);
-			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl_write_to_buffer);
+			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_buffer);
+			curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+			curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
+			curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress);
 			curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
 			curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
 			res = curl_easy_perform(curl);
+			curl_slist_free_all(list);
 			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code);
 			if(res != CURLE_OK && params.verbose > 0)
 				log::error("_make_request.curl_easy_perform(): {}", curl_easy_strerror(res));
@ -101,17 +123,17 @@ namespace gdpm::http{

 		r.body = buf.addr;
 		utils::free_buffer(buf);
-		curl_global_cleanup();
+		// curl_global_cleanup();
 		return r;
 	}


-	response download_file(
+	response context::download_file(
 		const string& url, 
 		const string& storage_path, 
 		const http::request_params& params
 	){
-		CURL *curl = nullptr;
+		// CURL *curl = nullptr;
 		CURLcode res;
 		response r;
 		FILE *fp;
@ -121,8 +143,8 @@ namespace gdpm::http{
 		utils::delay();
 #endif

-		curl_global_init(CURL_GLOBAL_ALL);
-		curl = curl_easy_init();
+		// curl_global_init(CURL_GLOBAL_ALL);
+		// curl = curl_easy_init();
 		if(curl){
 			fp = fopen(storage_path.c_str(), "wb");
 			// if(!config.username.empty() && !config.password.empty()){
@ -136,23 +158,82 @@ namespace gdpm::http{
 			// 	curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
 			// 	curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
 			// }
+			utils::memory_buffer *data;
+			curl_slist *list = _add_headers(curl, params.headers);
 			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 			// curl_easy_setopt(curl, CURLOPT_USERPWD, "user:pass");
 			curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
 			curl_easy_setopt(curl, CURLOPT_HEADER, 0);
 			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
 			curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
-			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl_write_to_stream);
+			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_stream);
+			curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+			curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
+			curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress);
 			curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
 			curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
 			res = curl_easy_perform(curl);
+			curl_slist_free_all(list);
+
+			/* Get response code, process error, save data, and close file. */
 			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code);
 			if(res != CURLE_OK && params.verbose > 0){
 				log::error("download_file.curl_easy_perform() failed: {}", curl_easy_strerror(res));
 			}
 			fclose(fp);
 		}
-		curl_global_cleanup();
+		// curl_global_cleanup();
 		return r;
 	}
+
+	long context::get_download_size(const string& url){
+		// CURL *curl = curl_easy_init();
+		CURLcode res;
+		if(curl){
+			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+			res = curl_easy_perform(curl);
+			if(!res){
+				curl_off_t cl;
+				res = curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &cl);
+				if(!res){
+					log::debug("download size: {}", cl);
+				}
+				return cl;
+			}
+		}
+		return -1;
+	}
+
+
+	long context::get_bytes_downloaded(const string& url){
+		CURLcode res;
+		if(curl){
+			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+			res = curl_easy_perform(curl);
+			if(!res){
+				curl_off_t dl;
+				res = curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD_T, &dl);
+				if(!res){
+					/* TODO: Integrate the `indicators` progress bar here. */
+				}
+			}
+		}
+		return -1;
+	}
+
+
+	curl_slist* context::_add_headers(
+		CURL *curl,
+		const headers_t& headers
+	){
+		struct curl_slist *list = NULL;
+		if(!headers.empty()){
+			for(const auto& header : headers){
+				string h = header.first + ": " + header.second;
+				list = curl_slist_append(list, h.c_str());
+			}
+			curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
+		}
+		return list;
+	}
 }
--- a/src/package.cpp
+++ b/src/package.cpp
@ -20,7 +20,7 @@ namespace gdpm::package{
 	
 	error install(
 		const config::context& config,
-		const package::title_list& package_titles, 
+		package::title_list& package_titles, 
 		const package::params& params
 	){
 		using namespace rapidjson;
@ -38,6 +38,20 @@ namespace gdpm::package{

 		*/

+		/* Append files from --file option */
+		if(!params.input_files.empty()){
+			log::print("input files");
+			for(const auto& filepath : params.input_files){
+				string contents = utils::readfile(filepath);
+				log::print("contents: {}", contents);
+				string_list input_titles = utils::split_lines(contents);
+				package_titles.insert(
+					std::end(package_titles),
+					std::begin(input_titles),
+					std::end(input_titles)
+				);
+			}
+		}
 		result_t result = cache::get_package_info_by_title(package_titles);
 		package::info_list p_found = {};
 		package::info_list p_cache = result.unwrap_unsafe();
@ -51,6 +65,7 @@ namespace gdpm::package{
 			}
 		}

+
 		/* Match queried package titles with those found in cache. */
 		log::debug("Searching for packages in cache...");
 		for(const auto& p_title : package_titles){
@ -68,10 +83,13 @@ namespace gdpm::package{

 		/* If size of package_titles == p_found, then all packages can be installed
 		from cache and there's no need to query remote API. However, this will
-		only installed the latest *local* version and will not sync with remote. */
-		if(p_found.size() == package_titles.size()){
-			log::info("Found all packages stored in local cache.");
-		}
+		only installed the latest *local* version and will not sync with remote. 
+		
+		FIXME: This needs to also check if it is installed as well.
+		*/
+		// if(p_found.size() == package_titles.size()){
+		// 	log::info("Found all packages stored in local cache.");
+		// }

 		/* Found nothing to install so there's nothing to do at this point. */
 		if(p_found.empty()){
@ -165,8 +183,9 @@ namespace gdpm::package{
 			}
 			else{
 				/* Download all the package files and place them in tmp directory. */
-				log::info("Downloading \"{}\"...", p.title);
-				http::response response = http::download_file(p.download_url, tmp_zip);
+				log::info_n("Downloading \"{}\"...", p.title);
+				http::context http;
+				http::response response = http.download_file(p.download_url, tmp_zip);
 				if(response.code == http::OK){
 					log::println("Done.");
 				}else{
@ -185,8 +204,9 @@ namespace gdpm::package{
 			p.install_path = package_dir;

 			/* Extract all the downloaded packages to their appropriate directory location. */
-			for(const auto& p : dir_pairs)
-				utils::extract_zip(p.first.c_str(), p.second.c_str());
+			for(const auto& p : dir_pairs){
+				int error_code = utils::extract_zip(p.first.c_str(), p.second.c_str());
+			}

 			/* Update the cache data with information from  */
 			log::info_n("Updating local asset data...");
@ -196,6 +216,9 @@ namespace gdpm::package{
 				log::println(GDPM_COLOR_LOG_ERROR"\n{}{}" GDPM_COLOR_RESET, prefix, error.get_message());
 				return error;
 			}
+			if(config.clean_temporary){
+				clean_temporary(config, package_titles);
+			}

 			log::println("Done.");
 				// })
@ -208,7 +231,7 @@ namespace gdpm::package{

 	error add(
 		const config::context& config,
-		const title_list& package_titles,
+		title_list& package_titles,
 		const params& params
 	){
 		/* Install packages in local project instead of package database.
@ -219,15 +242,29 @@ namespace gdpm::package{

 	error remove(
 		const config::context& config,
-		const string_list& package_titles, 
+		string_list& package_titles, 
 		const package::params& params
 	){
 		using namespace rapidjson;
 		using namespace std::filesystem;

+
+		/* Append package titles from --file option */
+		if(!params.input_files.empty()){
+			for(const auto& filepath : params.input_files){
+				string contents = utils::readfile(filepath);
+				string_list _package_titles = utils::split_lines(contents);
+				package_titles.insert(
+					std::end(package_titles),
+					std::begin(_package_titles),
+					std::end(_package_titles)
+				);
+			}
+		}
+
 		/* Find the packages to remove if they're is_installed and show them to the user */
 		result_t result = cache::get_package_info_by_title(package_titles);
-		std::vector<package::info> p_cache = result.unwrap_unsafe();
+		package::info_list p_cache = result.unwrap_unsafe();
 		if(p_cache.empty()){
 			error error(
 				constants::error::NOT_FOUND,
@ -246,7 +283,7 @@ namespace gdpm::package{
 		if(p_count == 0){
 			error error(
 				constants::error::NOT_FOUND,
-				"\nNo packages to remove."
+				"No packages to remove."
 			);
 			log::error(error);
 			return error;
@ -259,7 +296,7 @@ namespace gdpm::package{
 		log::println("");
 		
 		if(!config.skip_prompt){
-			if(!utils::prompt_user_yn("Do you want to remove these packages? (y/n)"))
+			if(!utils::prompt_user_yn("Do you want to remove these packages? (Y/n)"))
 				return error();
 		}

@ -299,6 +336,9 @@ namespace gdpm::package{
 			p.is_installed = false;
 		}
 		log::println("Done.");
+		if(config.clean_temporary){
+			clean_temporary(config, package_titles);
+		}
 		log::info_n("Updating local asset data...");
 		{
 			error error = cache::update_package_info(p_cache);
@ -392,6 +432,7 @@ namespace gdpm::package{
 	){
 		result_t r_cache = cache::get_package_info_by_title(package_titles);
 		info_list p_cache = r_cache.unwrap_unsafe();
+		http::context http;
 		
 		if(!p_cache.empty() && !config.enable_sync){
 			print_list(p_cache);
@ -401,11 +442,8 @@ namespace gdpm::package{
 		rest_api::request_params rest_api_params = rest_api::make_from_config(config);
 		for(const auto& p_title : package_titles){
 			using namespace rapidjson;
-		
-			rest_api_params.filter = http::url_escape(p_title);
-			rest_api_params.verbose = config.verbose;
-			rest_api_params.godot_version = config.info.godot_version;
-			rest_api_params.max_results = 200;
+
+			rest_api_params.filter = http.url_escape(p_title);

 			std::string request_url{constants::HostUrl};
 			request_url += rest_api::endpoints::GET_Asset;
@ -471,13 +509,17 @@ namespace gdpm::package{

 		/* Write contents of installed packages in reusable format */
 		for(const auto& path : paths ){
-			std::ofstream of(path);
 			if(std::filesystem::exists(path)){
-				constexpr const char *message = "File or directory exists!";
-				log::error(message);
-				of.close();
-				return error(constants::error::FILE_EXISTS, message);
+				if(utils::prompt_user_yn("File or directory exists. Do you want to overwrite it?")){
+
+				}
+				else {
+					constexpr const char *message = "File or directory exists!";
+					log::error(message);
+					return error(constants::error::FILE_EXISTS, message);
+				}
 			}
+			std::ofstream of(path);
 			log::println("writing contents to file");
 			of << output;
 			of.close();
--- a/src/package_manager.cpp
+++ b/src/package_manager.cpp
@ -108,7 +108,8 @@ namespace gdpm::package_manager{
 		auto verboseOpt 				= joinable(repeatable(option("-v", "--verbose").call([]{ config.verbose += 1; }))) % "show verbose output";	

 		/* Set the options */
-		auto fileOpt 			= repeatable(option("--file", "-f").set(params.args)  % "read file as input");
+		// auto fileOpt 			= repeatable(option("--file", "-f").set(params.input_files)  % "read file as input");
+		auto fileOpt 				= repeatable(option("--file", "-f") & values("input", params.input_files)) % "read file as input";
 		auto cleanOpt 			= option("--clean").set(config.clean_temporary) % "enable/disable cleaning temps";
 		auto parallelOpt 		= option("--jobs").set(config.jobs) % "set number of parallel jobs";
 		auto cacheOpt 			= option("--enable-cache").set(config.enable_cache) % "enable/disable local caching";
@ -132,7 +133,7 @@ namespace gdpm::package_manager{
 		auto removeCmd = "remove" % (
 			command("remove").set(action, action_e::remove),
 			packageValues % "packages(s) to remove from project",
-			fileOpt
+			fileOpt, skipOpt, cleanOpt
 		);
 		auto updateCmd = "update package(s)" % (
 			command("update").set(action, action_e::update),
@ -140,7 +141,8 @@ namespace gdpm::package_manager{
 		);
 		auto searchCmd = "search for package(s)" % (
 			command("search").set(action, action_e::search),
-			packageValues % ""
+			packageValues % "",
+			godotVersionOpt, fileOpt, remoteOpt, configOpt
 		);
 		auto exportCmd = "export installed package list to file" % (
 			command("export").set(action, action_e::p_export),
--- a/src/rest_api.cpp
+++ b/src/rest_api.cpp
@ -14,7 +14,6 @@
 #include <curlpp/Easy.hpp>
 #include <curlpp/Options.hpp>
 #include <curlpp/Exception.hpp>
-
 namespace gdpm::rest_api{
 	
 	request_params make_from_config(const config::context& config){
@ -22,6 +21,7 @@ namespace gdpm::rest_api{
 		request_params params 	= make_request_params();
 		params.godot_version 	= (is_latest) ? "" : config.info.godot_version;
 		params.verbose 			= config.verbose;
+		params.max_results 		= config.max_results;
 		return params;
 	}

@ -166,9 +166,10 @@ namespace gdpm::rest_api{
 		type_e type, 
 		int verbose
 	){
+		http::context http;
 		string request_url{url};
 		request_url += to_string(type);
-		http::response r = http::request_get(url);
+		http::response r = http.request_get(url);
 		if(verbose > 0)
 			log::info("URL: {}", url);
 		return _parse_json(r.body);
@ -208,8 +209,14 @@ namespace gdpm::rest_api{
 		const string& url, 
 		const request_params& c
 	){
+		http::context http;
+		http::request_params http_params;
+		http_params.headers.insert(http::header("Accept", "*/*"));
+		http_params.headers.insert(http::header("Accept-Encoding", "application/gzip"));
+		http_params.headers.insert(http::header("Content-Encoding", "application/gzip"));
+		http_params.headers.insert(http::header("Connection", "keep-alive"));
 		string request_url = _prepare_request(url, c);
-		http::response r = http::request_get(request_url);
+		http::response r = http.request_get(request_url, http_params);
 		if(c.verbose > 0)
 			log::info("get_asset().URL: {}", request_url);
 		return _parse_json(r.body, c.verbose);
@ -218,11 +225,19 @@ namespace gdpm::rest_api{
 	rapidjson::Document get_asset(
 		const string& url, 
 		int asset_id, 
-		const request_params& params
+		const rest_api::request_params& api_params
 	){
-		string request_url = utils::replace_all(_prepare_request(url, params), "{id}", std::to_string(asset_id));
-		http::response r = http::request_get(request_url.c_str());
-		if(params.verbose >= log::INFO)
+		/* Set up HTTP request */
+		http::context http;
+		http::request_params http_params;
+		http_params.headers.insert(http::header("Accept", "*/*"));
+		http_params.headers.insert(http::header("Accept-Encoding", "application/gzip"));
+		http_params.headers.insert(http::header("Content-Encoding", "application/gzip"));
+		http_params.headers.insert(http::header("Connection", "keep-alive"));
+
+		string request_url = utils::replace_all(_prepare_request(url, api_params), "{id}", std::to_string(asset_id));
+		http::response r = http.request_get(request_url.c_str(), http_params);
+		if(api_params.verbose >= log::INFO)
 			log::info("get_asset().URL: {}", request_url);
 		
 		return _parse_json(r.body);
--- a/src/utils.cpp
+++ b/src/utils.cpp
@ -3,6 +3,11 @@
 #include "config.hpp"
 #include "constants.hpp"
 #include "log.hpp"
+#include "indicators/indeterminate_progress_bar.hpp"
+#include "indicators/dynamic_progress.hpp"
+#include "indicators/progress_bar.hpp"
+#include "indicators/block_progress_bar.hpp"
+#include "csv2/reader.hpp"


 #include <asm-generic/errno-base.h>
@ -21,9 +26,56 @@
 #include <thread>
 #include <unordered_map>
 #include <zip.h>
+#include <curl/curl.h>

 namespace gdpm::utils{

+	using namespace indicators;
+	BlockProgressBar bar {
+		option::BarWidth{50},
+		// option::Start{"["},
+		// option::Fill{"="},
+		// option::Lead{">"},
+		// option::Remainder{" "},
+		// option::End{"]"},
+		option::PrefixText{"Downloading file "},
+		option::PostfixText{""},
+		option::ForegroundColor{Color::green},
+		option::FontStyles{std::vector<FontStyle>{FontStyle::bold}},
+	};
+		// option::ShowElapsedTime{true},
+		// option::ShowRemainingTime{true},
+	IndeterminateProgressBar bar_unknown {
+		option::BarWidth{50},
+		option::Start{"["},
+		option::Fill{"."},
+		option::Lead{"<==>"},
+		option::PrefixText{"Downloading file "},
+		option::End{"]"},
+		option::PostfixText{""},
+		option::ForegroundColor{Color::green},
+		option::FontStyles{std::vector<FontStyle>{FontStyle::bold}},
+	};
+
+	std::vector<std::string> split_lines(const std::string& contents){
+		using namespace csv2;
+		csv2::Reader<
+			delimiter<'\n'>,
+			quote_character<'"'>,
+			first_row_is_header<false>,
+			trim_policy::trim_whitespace
+		> csv;
+		std::vector<std::string> lines;
+		if(csv.parse(contents)){
+			for(const auto& row : csv){
+				for(const auto& cell : row){
+					lines.emplace_back(cell.read_view());
+				}
+			}
+		}
+		return lines;
+	}
+
 	
 	#if (GDPM_READFILE_IMPL == 0)
 		std::string readfile(const std::string& path){
@ -142,7 +194,8 @@ namespace gdpm::utils{
 		int i, len, fd;
 		zip_uint64_t sum;	

-		log::info_n("Extracting package contents to '{}'...", dest);
+		// log::info_n("Extracting package contents to '{}'...", dest);
+		log::info_n("Extracting package contents...");
 		if((za = zip_open(archive, 0, &err)) == nullptr){
 			zip_error_to_str(buf, sizeof(buf), err, errno);
 			log::error("{}: can't open zip archive {}: {}", prog, archive, buf);
@ -263,8 +316,104 @@ namespace gdpm::utils{
 		return o;
 	}

-	namespace json {
+	std::string convert_size(long size){
+		int digit = 0;
+		while(size > 1000){
+			size /= 1000;
+			digit += 1;
+		}
+		std::string s = std::to_string(size);
+		switch(digit){
+			case 0: return s + " B";
+			case 1: return s + " KB";
+			case 2: return s + " MB";
+			case 3: return s + " GB";
+			case 4: return s + " TB";
+			case 5: return s + " PB";
+		}
+		return std::to_string(size);
+	}

+
+	namespace curl {
+		size_t write_to_buffer(
+			char *contents, 
+			size_t size, 
+			size_t nmemb, 
+			void *userdata
+		){
+			size_t realsize = size * nmemb;
+			struct memory_buffer *m = (struct memory_buffer*)userdata;
+
+			m->addr = (char*)realloc(m->addr, m->size + realsize + 1);
+			if(m->addr == nullptr){
+				/* Out of memory */
+				fprintf(stderr, "Not enough memory (realloc returned NULL)\n");
+				return 0;
+			}
+
+			memcpy(&(m->addr[m->size]), contents, realsize);
+			m->size += realsize;
+			m->addr[m->size] = 0;
+
+			return realsize;
+		}
+
+		size_t write_to_stream(
+			char *ptr, 
+			size_t size, 
+			size_t nmemb, 
+			void *userdata
+		){
+			if(nmemb == 0)
+				return 0;
+						
+			return fwrite(ptr, size, nmemb, (FILE*)userdata);
+		}
+
+		int show_progress(
+			void *ptr,
+			curl_off_t total_download,
+			curl_off_t current_downloaded,
+			curl_off_t total_upload,
+			curl_off_t current_upload
+		){
+
+			if(current_downloaded >= total_download)
+				return 0;
+			using namespace indicators;
+			show_console_cursor(false);
+			if(total_download != 0){
+				// double percent = std::floor((current_downloaded / (total_download)) * 100);
+				bar.set_option(option::MaxProgress{total_download});
+				bar.set_progress(current_downloaded);
+				bar.set_option(option::PostfixText{
+						convert_size(current_downloaded) + " / " + 
+						convert_size(total_download)
+				});
+				if(bar.is_completed()){
+					bar.set_option(option::PrefixText{"Download completed."});
+					bar.mark_as_completed();
+				}
+			} else {
+				if(bar_unknown.is_completed()){
+					bar.set_option(option::PrefixText{"Download completed."});
+					bar.mark_as_completed();
+				} else {
+					bar.tick();
+					bar_unknown.set_option(
+						option::PostfixText(std::format("{}", convert_size(current_downloaded)))
+					);
+
+				}
+			}
+			show_console_cursor(true);
+			memory_buffer *m = (memory_buffer*)ptr;
+			return 0;
+		}
+	}
+
+	namespace json {
 		std::string from_array(
 			const std::set<std::string>& a, 
 			const std::string& prefix