Minor bug fixes and improvements

- Update `README.md` file with minor changes
- Added `max_results` parameter to config for persistence
- Added `csv2`, `tabulate`, and `indicators` as included dependencies
- Added downloading progress bar that *sometimes* work
-Added HTTP request header handling with curl
- Added more options to some commands
- Moved `http` functions into `context` class

TODO: Fix optional parameters not working correctly
NOTE: Github does not always return `Content-length` HTTP response header

.gitmodules

@@ -4,3 +4,12 @@
 [submodule "modules/doxygen-awesome-css"]
 	path = modules/doxygen-awesome-css
 	url = https://github.com/jothepro/doxygen-awesome-css/
+[submodule "submodules/indicators"]
+	path = submodules/indicators
+	url = https://github.com/p-ranav/indicators
+[submodule "modules/tabulate"]
+	path = modules/tabulate
+	url = https://github.com/p-ranav/tabulate
+[submodule "modules/csv2"]
+	path = modules/csv2
+	url = https://github.com/p-ranav/csv2

README.md

@@ -1,6 +1,6 @@
 # Godot Package Manager (GDPM)
 
-https://ody.sh/B5vPxVhNTr
+[Demo](https://ody.sh/B5vPxVhNTr)
 
 GDPM is an attempt to make a simple, front-end, command-line, package manager designed to handle assets from the Godot Game Engine's official asset library. It is written in C++ to be lightwight and fast with a few common dependencies found in most Linux distributions and can be used completely independent of Godot. It is designed to add more functionality not included with the official AssetLib with the ability to automate downloads for different platforms. So far, the package manager is capable of searching, downloading, installing, and removing packages and makes managing Godot assets across multiple projects much easier. It stores "packages" in a global directories and is capable of linking or copying global packages to local projects.
 
@@ -161,20 +161,20 @@ Other installation behavior can be changed with additional flags. Adding the `-y
 
 ```bash
 gdpm install "Flappy Godot" "GodotNetworking" -y
-gdpm install -f packages.txt --config config.json --no-sync --no-prompt --verbose
+gdpm install -f packages.txt --config config.json --no-sync --skip-prompt --verbose
 
 gdpm export path/to/packages.txt
-gdpm install -f path/to/packages.txt --no-sync --no-prompt
+gdpm install -f path/to/packages.txt --no-sync --skip-prompt
 
 # Future work for multithreading support...
-# gdpm install -f path/to/packages.txt -j$(nproc) --no-sync --no-prompt --verbose
+# gdpm install -f path/to/packages.txt -j$(nproc) --no-sync --skip-prompt --verbose
 ```
 
 Packages can be removed similiarly to installing.
 
 ```bash
 gdpm remove "Flappy Godot" "GodotNetworking" -y
-gdpm remove -f packages.txt --config config.json --no-sync --no-prompt
+gdpm remove -f packages.txt --config config.json --no-sync --skip-prompt
 ```
 
 Packages can be updated simliar installing or removing packages. However, if no argument is passed, GDPM will prompt the user to update ALL packages to latest instead. The local metadata database can be updated using the `sync` command. (Note: The `sync` command will be changed to the `fetch` command to reflect `git`'s API.)
@@ -265,6 +265,8 @@ export PATH=$PATH:path/to/bin/gdpm
 
 *   The `help` command does currently print the command/options correctly. Commands do not use the double hypen, `--command` format. Commands should be used like `gdpm command --option` instead.
 
+*   Some asset types might not install correctly due to partial downloads being interrupted. Try running `gdpm clean <package>`, then install again.
+
 ## License
 
 See the LICENSE.md file.

bin/compile.sh

@@ -80,7 +80,7 @@ function build_exe(){
 
 function build_libs(){
 	mkdir -p build
-	$CMAKE_COMMAND \\
+	$CMAKE_COMMAND \
 		--target gdpm-static \
 		--target gdpm-shared \
 		--target gdpm-http \

include/config.hpp

@@ -27,6 +27,7 @@ namespace gdpm::config{
 		string_map remote_sources;
 		size_t jobs					= 1;
 		size_t timeout				= 3000;
+		size_t max_results			= 200;
 		bool enable_sync			= true;
 		bool enable_cache			= true;
 		bool skip_prompt			= false;

include/http.hpp

@@ -3,9 +3,12 @@
 #include "constants.hpp"
 #include "types.hpp"
 #include <unordered_map>
+#include <curl/curl.h>
+#include <curl/easy.h>
 
 namespace gdpm::http{
 	using headers_t = std::unordered_map<string, string>;
+	using header = std::pair<string, string>;
 
 	// REF: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
 	enum response_code{
@@ -88,9 +91,25 @@ namespace gdpm::http{
 		int verbose = 0;
 	};
 
-	string url_escape(const string& url);
-	response request_get(const string& url, const http::request_params& params = http::request_params());
-	response request_post(const string& url, const http::request_params& params = http::request_params());
-	response download_file(const string& url, const string& storage_path, const http::request_params& params = http::request_params());
+	class context : public non_copyable{
+	public:
+		context();
+		~context();
+
+		inline CURL* const get_curl() const;
+		string url_escape(const string& url);
+		response request_get(const string& url, const http::request_params& params = http::request_params());
+		response request_post(const string& url, const http::request_params& params = http::request_params());
+		response download_file(const string& url, const string& storage_path, const http::request_params& params = http::request_params());
+		long get_download_size(const string& url);
+		long get_bytes_downloaded(const string& url);
 	
+	private:
+		CURL *curl;
+		curl_slist* _add_headers(CURL *curl, const headers_t& headers);
+	};
+
+	
+
+	extern context http;
 }

include/package.hpp

@@ -90,21 +90,21 @@ namespace gdpm::package {
 
 	To copy the package to a project instead of linking, use the `--clone` option.
 
-	`gdpm install --clone "super cool examle package"
+	`gdpm install --clone "super cool example package"
 
 	*/
-	GDPM_DLL_EXPORT error install(const config::context& config, const title_list& package_titles, const params& params = package::params());
+	GDPM_DLL_EXPORT error install(const config::context& config, title_list& package_titles, const params& params = package::params());
 	/*!
 	@brief Adds package to project locally only.
 	@param config
 	@param package_titles
 	@param params 
 	*/
-	GDPM_DLL_EXPORT error add(const config::context& config, const title_list& package_titles, const params& params = package::params());
+	GDPM_DLL_EXPORT error add(const config::context& config, title_list& package_titles, const params& params = package::params());
 	/*!
 	@brief Remove's package and contents from local database.
 	*/
-	GDPM_DLL_EXPORT error remove(const config::context& config, const title_list& package_titles, const params& params = package::params());
+	GDPM_DLL_EXPORT error remove(const config::context& config, title_list& package_titles, const params& params = package::params());
 	GDPM_DLL_EXPORT error remove_all(const config::context& config, const params& params = package::params());
 	GDPM_DLL_EXPORT error update(const config::context& config, const title_list& package_titles, const params& params = package::params());
 	GDPM_DLL_EXPORT error search(const config::context& config, const title_list& package_titles, const params& params = package::params());

include/utils.hpp

@@ -17,10 +17,12 @@
 #include <fmt/format.h>
 #include <fmt/chrono.h>
 
+
+typedef long curl_off_t;
 namespace gdpm::utils {
 
 	using namespace std::chrono_literals;
-
+	// extern indicators::DynamicProgress<indicators::ProgressBar> bars;
 	struct memory_buffer{
 		char *addr = nullptr;
 		size_t size = 0;
@@ -37,31 +39,6 @@ namespace gdpm::utils {
 		free(buf.addr);
 	}
 
-	static size_t curl_write_to_buffer(char *contents, size_t size, size_t nmemb, void *userdata){
-
-		size_t realsize = size * nmemb;
-		struct memory_buffer *m = (struct memory_buffer*)userdata;
-
-		m->addr = (char*)realloc(m->addr, m->size + realsize + 1);
-		if(m->addr == nullptr){
-			/* Out of memory */
-			fprintf(stderr, "Not enough memory (realloc returned NULL)\n");
-			return 0;
-		}
-
-		memcpy(&(m->addr[m->size]), contents, realsize);
-		m->size += realsize;
-		m->addr[m->size] = 0;
-
-		return realsize;
-	}
-
-	static size_t curl_write_to_stream(char *ptr, size_t size, size_t nmemb, void *userdata){
-		if(nmemb == 0)
-			return 0;
-		return fwrite(ptr, size, nmemb, (FILE*)userdata);
-	}
-
 	/* Use ISO 8601 for default timestamp format. */
 	static inline auto timestamp(const std::string& format = GDPM_TIMESTAMP_FORMAT){
 		time_t t = std::time(nullptr);
@@ -94,7 +71,8 @@ namespace gdpm::utils {
 		std::move(part, from.end(), std::back_inserter(from));
 		from.erase(part);
 	}
-	
+
+	std::vector<std::string> split_lines(const std::string& contents);
 	std::string readfile(const std::string& path);
 	std::string to_lower(const std::string& s);
 	std::string trim(const std::string& s);
@@ -111,8 +89,14 @@ namespace gdpm::utils {
 	void delay(std::chrono::milliseconds milliseconds = GDPM_REQUEST_DELAY);
 	std::string join(const std::vector<std::string>& target, const std::string& delimiter = ", ");
 	std::string join(const std::unordered_map<std::string, std::string>& target, const std::string& prefix = "", const std::string& delimiter = "\n");
+	std::string convert_size(long size);
 	// TODO: Add function to get size of decompressed zip
 
+	namespace curl {
+		extern size_t write_to_buffer(char *contents, size_t size, size_t nmemb, void *userdata);
+		extern size_t write_to_stream(char *ptr, size_t size, size_t nmemb, void *userdata);
+		extern int show_progress(void *ptr, curl_off_t total_download, curl_off_t current_downloaded, curl_off_t total_upload, curl_off_t current_upload);
+	}
 	namespace json {
 		std::string from_array(const std::set<std::string>& a, const std::string& prefix);
 		std::string from_object(const std::unordered_map<std::string, std::string>& m, const std::string& prefix, const std::string& spaces);

modules/csv2

@@ -0,0 +1 @@
+Subproject commit 12989a1f0517c61aa273d4514f6364be79d2a211

modules/tabulate

@@ -0,0 +1 @@
+Subproject commit b35db4cce50a4b296290b0ae827305cdeb23751e

src/http.cpp

@@ -3,26 +3,36 @@
 #include "utils.hpp"
 #include "log.hpp"
 #include <curl/curl.h>
+#include <curl/easy.h>
 #include <stdio.h>
 #include <chrono>
 
 
 namespace gdpm::http{
 
-	string url_escape(const string &url){
-		CURL *curl = nullptr;
+	context::context(){
 		curl_global_init(CURL_GLOBAL_ALL);
-		char *escaped_url = curl_easy_escape(curl, url.c_str(), url.size());
-		std::string url_copy = escaped_url;
-		curl_global_cleanup();
-		return escaped_url;
+		curl = curl_easy_init();
 	}
 
-	response request_get(
+
+	context::~context(){
+		curl_global_cleanup();
+		curl_easy_cleanup(curl);
+	}
+
+	CURL* const context::get_curl() const{
+		return curl;
+	}
+
+	string context::url_escape(const string &url){
+		return curl_easy_escape(curl, url.c_str(), url.size());;
+	}
+
+	response context::request_get(
 		const string& url, 
 		const http::request_params& params
 	){
-		CURL *curl = nullptr;
 		CURLcode res;
 		utils::memory_buffer buf = utils::make_buffer();
 		response r;
@@ -32,17 +42,23 @@ namespace gdpm::http{
 		utils::delay();
 #endif
 
-		curl_global_init(CURL_GLOBAL_ALL);
-		curl = curl_easy_init();
+		// curl_global_init(CURL_GLOBAL_ALL);
+		// curl = curl_easy_init();
 		if(curl){
+			utils::memory_buffer *data;
+			curl_slist *list = _add_headers(curl, params.headers);
 			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 			// curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");
 			curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
 			curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf);
-			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl_write_to_buffer);
+			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_buffer);
+			curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+			curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
+			curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress);
 			curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
 			curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
 			res = curl_easy_perform(curl);
+			curl_slist_free_all(list);
 			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code);
 			if(res != CURLE_OK && params.verbose > 0)
 				log::error("_make_request.curl_easy_perform(): {}", curl_easy_strerror(res));
@@ -51,16 +67,16 @@ namespace gdpm::http{
 
 		r.body = buf.addr;
 		utils::free_buffer(buf);
-		curl_global_cleanup();
+		// curl_global_cleanup();
 		return r;
 	}
 
 
-	response request_post(
+	response context::request_post(
 		const string& url, 
 		const http::request_params& params
 	){
-		CURL *curl = nullptr;
+		// CURL *curl = nullptr;
 		CURLcode res;
 		utils::memory_buffer buf = utils::make_buffer();
 		response r;
@@ -81,18 +97,24 @@ namespace gdpm::http{
 		h = url_escape(h);
 
 		// const char *post_fields = "";
-		curl_global_init(CURL_GLOBAL_ALL);
-		curl = curl_easy_init();
+		// curl_global_init(CURL_GLOBAL_ALL);
+		// curl = curl_easy_init();
 		if(curl){
+			utils::memory_buffer *data;
+			curl_slist *list = _add_headers(curl, params.headers);
 			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 			// curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");
 			curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, h.size());
 			curl_easy_setopt(curl, CURLOPT_POSTFIELDS, h.c_str());
 			curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf);
-			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl_write_to_buffer);
+			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_buffer);
+			curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+			curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
+			curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress);
 			curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
 			curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
 			res = curl_easy_perform(curl);
+			curl_slist_free_all(list);
 			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code);
 			if(res != CURLE_OK && params.verbose > 0)
 				log::error("_make_request.curl_easy_perform(): {}", curl_easy_strerror(res));
@@ -101,17 +123,17 @@ namespace gdpm::http{
 
 		r.body = buf.addr;
 		utils::free_buffer(buf);
-		curl_global_cleanup();
+		// curl_global_cleanup();
 		return r;
 	}
 
 
-	response download_file(
+	response context::download_file(
 		const string& url, 
 		const string& storage_path, 
 		const http::request_params& params
 	){
-		CURL *curl = nullptr;
+		// CURL *curl = nullptr;
 		CURLcode res;
 		response r;
 		FILE *fp;
@@ -121,8 +143,8 @@ namespace gdpm::http{
 		utils::delay();
 #endif
 
-		curl_global_init(CURL_GLOBAL_ALL);
-		curl = curl_easy_init();
+		// curl_global_init(CURL_GLOBAL_ALL);
+		// curl = curl_easy_init();
 		if(curl){
 			fp = fopen(storage_path.c_str(), "wb");
 			// if(!config.username.empty() && !config.password.empty()){
@@ -136,23 +158,82 @@ namespace gdpm::http{
 			// 	curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
 			// 	curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
 			// }
+			utils::memory_buffer *data;
+			curl_slist *list = _add_headers(curl, params.headers);
 			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 			// curl_easy_setopt(curl, CURLOPT_USERPWD, "user:pass");
 			curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
 			curl_easy_setopt(curl, CURLOPT_HEADER, 0);
 			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
 			curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
-			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl_write_to_stream);
+			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, utils::curl::write_to_stream);
+			curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+			curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
+			curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, utils::curl::show_progress);
 			curl_easy_setopt(curl, CURLOPT_USERAGENT, constants::UserAgent.c_str());
 			curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, params.timeout);
 			res = curl_easy_perform(curl);
+			curl_slist_free_all(list);
+
+			/* Get response code, process error, save data, and close file. */
 			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r.code);
 			if(res != CURLE_OK && params.verbose > 0){
 				log::error("download_file.curl_easy_perform() failed: {}", curl_easy_strerror(res));
 			}
 			fclose(fp);
 		}
-		curl_global_cleanup();
+		// curl_global_cleanup();
 		return r;
 	}
+
+	long context::get_download_size(const string& url){
+		// CURL *curl = curl_easy_init();
+		CURLcode res;
+		if(curl){
+			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+			res = curl_easy_perform(curl);
+			if(!res){
+				curl_off_t cl;
+				res = curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &cl);
+				if(!res){
+					log::debug("download size: {}", cl);
+				}
+				return cl;
+			}
+		}
+		return -1;
+	}
+
+
+	long context::get_bytes_downloaded(const string& url){
+		CURLcode res;
+		if(curl){
+			curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+			res = curl_easy_perform(curl);
+			if(!res){
+				curl_off_t dl;
+				res = curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD_T, &dl);
+				if(!res){
+					/* TODO: Integrate the `indicators` progress bar here. */
+				}
+			}
+		}
+		return -1;
+	}
+
+
+	curl_slist* context::_add_headers(
+		CURL *curl,
+		const headers_t& headers
+	){
+		struct curl_slist *list = NULL;
+		if(!headers.empty()){
+			for(const auto& header : headers){
+				string h = header.first + ": " + header.second;
+				list = curl_slist_append(list, h.c_str());
+			}
+			curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
+		}
+		return list;
+	}
 }

src/package.cpp

@@ -20,7 +20,7 @@ namespace gdpm::package{
 	
 	error install(
 		const config::context& config,
-		const package::title_list& package_titles, 
+		package::title_list& package_titles, 
 		const package::params& params
 	){
 		using namespace rapidjson;
@@ -38,6 +38,20 @@ namespace gdpm::package{
 
 		*/
 
+		/* Append files from --file option */
+		if(!params.input_files.empty()){
+			log::print("input files");
+			for(const auto& filepath : params.input_files){
+				string contents = utils::readfile(filepath);
+				log::print("contents: {}", contents);
+				string_list input_titles = utils::split_lines(contents);
+				package_titles.insert(
+					std::end(package_titles),
+					std::begin(input_titles),
+					std::end(input_titles)
+				);
+			}
+		}
 		result_t result = cache::get_package_info_by_title(package_titles);
 		package::info_list p_found = {};
 		package::info_list p_cache = result.unwrap_unsafe();
@@ -51,6 +65,7 @@ namespace gdpm::package{
 			}
 		}
 
+
 		/* Match queried package titles with those found in cache. */
 		log::debug("Searching for packages in cache...");
 		for(const auto& p_title : package_titles){
@@ -68,10 +83,13 @@ namespace gdpm::package{
 
 		/* If size of package_titles == p_found, then all packages can be installed
 		from cache and there's no need to query remote API. However, this will
-		only installed the latest *local* version and will not sync with remote. */
-		if(p_found.size() == package_titles.size()){
-			log::info("Found all packages stored in local cache.");
-		}
+		only installed the latest *local* version and will not sync with remote. 
+		
+		FIXME: This needs to also check if it is installed as well.
+		*/
+		// if(p_found.size() == package_titles.size()){
+		// 	log::info("Found all packages stored in local cache.");
+		// }
 
 		/* Found nothing to install so there's nothing to do at this point. */
 		if(p_found.empty()){
@@ -165,8 +183,9 @@ namespace gdpm::package{
 			}
 			else{
 				/* Download all the package files and place them in tmp directory. */
-				log::info("Downloading \"{}\"...", p.title);
-				http::response response = http::download_file(p.download_url, tmp_zip);
+				log::info_n("Downloading \"{}\"...", p.title);
+				http::context http;
+				http::response response = http.download_file(p.download_url, tmp_zip);
 				if(response.code == http::OK){
 					log::println("Done.");
 				}else{
@@ -185,8 +204,9 @@ namespace gdpm::package{
 			p.install_path = package_dir;
 
 			/* Extract all the downloaded packages to their appropriate directory location. */
-			for(const auto& p : dir_pairs)
-				utils::extract_zip(p.first.c_str(), p.second.c_str());
+			for(const auto& p : dir_pairs){
+				int error_code = utils::extract_zip(p.first.c_str(), p.second.c_str());
+			}
 
 			/* Update the cache data with information from  */
 			log::info_n("Updating local asset data...");
@@ -196,6 +216,9 @@ namespace gdpm::package{
 				log::println(GDPM_COLOR_LOG_ERROR"\n{}{}" GDPM_COLOR_RESET, prefix, error.get_message());
 				return error;
 			}
+			if(config.clean_temporary){
+				clean_temporary(config, package_titles);
+			}
 
 			log::println("Done.");
 				// })
@@ -208,7 +231,7 @@ namespace gdpm::package{
 
 	error add(
 		const config::context& config,
-		const title_list& package_titles,
+		title_list& package_titles,
 		const params& params
 	){
 		/* Install packages in local project instead of package database.
@@ -219,15 +242,29 @@ namespace gdpm::package{
 
 	error remove(
 		const config::context& config,
-		const string_list& package_titles, 
+		string_list& package_titles, 
 		const package::params& params
 	){
 		using namespace rapidjson;
 		using namespace std::filesystem;
 
+
+		/* Append package titles from --file option */
+		if(!params.input_files.empty()){
+			for(const auto& filepath : params.input_files){
+				string contents = utils::readfile(filepath);
+				string_list _package_titles = utils::split_lines(contents);
+				package_titles.insert(
+					std::end(package_titles),
+					std::begin(_package_titles),
+					std::end(_package_titles)
+				);
+			}
+		}
+
 		/* Find the packages to remove if they're is_installed and show them to the user */
 		result_t result = cache::get_package_info_by_title(package_titles);
-		std::vector<package::info> p_cache = result.unwrap_unsafe();
+		package::info_list p_cache = result.unwrap_unsafe();
 		if(p_cache.empty()){
 			error error(
 				constants::error::NOT_FOUND,
@@ -246,7 +283,7 @@ namespace gdpm::package{
 		if(p_count == 0){
 			error error(
 				constants::error::NOT_FOUND,
-				"\nNo packages to remove."
+				"No packages to remove."
 			);
 			log::error(error);
 			return error;
@@ -259,7 +296,7 @@ namespace gdpm::package{
 		log::println("");
 		
 		if(!config.skip_prompt){
-			if(!utils::prompt_user_yn("Do you want to remove these packages? (y/n)"))
+			if(!utils::prompt_user_yn("Do you want to remove these packages? (Y/n)"))
 				return error();
 		}
 
@@ -299,6 +336,9 @@ namespace gdpm::package{
 			p.is_installed = false;
 		}
 		log::println("Done.");
+		if(config.clean_temporary){
+			clean_temporary(config, package_titles);
+		}
 		log::info_n("Updating local asset data...");
 		{
 			error error = cache::update_package_info(p_cache);
@@ -392,6 +432,7 @@ namespace gdpm::package{
 	){
 		result_t r_cache = cache::get_package_info_by_title(package_titles);
 		info_list p_cache = r_cache.unwrap_unsafe();
+		http::context http;
 		
 		if(!p_cache.empty() && !config.enable_sync){
 			print_list(p_cache);
@@ -401,11 +442,8 @@ namespace gdpm::package{
 		rest_api::request_params rest_api_params = rest_api::make_from_config(config);
 		for(const auto& p_title : package_titles){
 			using namespace rapidjson;
-		
-			rest_api_params.filter = http::url_escape(p_title);
-			rest_api_params.verbose = config.verbose;
-			rest_api_params.godot_version = config.info.godot_version;
-			rest_api_params.max_results = 200;
+
+			rest_api_params.filter = http.url_escape(p_title);
 
 			std::string request_url{constants::HostUrl};
 			request_url += rest_api::endpoints::GET_Asset;
@@ -471,13 +509,17 @@ namespace gdpm::package{
 
 		/* Write contents of installed packages in reusable format */
 		for(const auto& path : paths ){
-			std::ofstream of(path);
 			if(std::filesystem::exists(path)){
-				constexpr const char *message = "File or directory exists!";
-				log::error(message);
-				of.close();
-				return error(constants::error::FILE_EXISTS, message);
+				if(utils::prompt_user_yn("File or directory exists. Do you want to overwrite it?")){
+
+				}
+				else {
+					constexpr const char *message = "File or directory exists!";
+					log::error(message);
+					return error(constants::error::FILE_EXISTS, message);
+				}
 			}
+			std::ofstream of(path);
 			log::println("writing contents to file");
 			of << output;
 			of.close();

src/package_manager.cpp

@@ -108,7 +108,8 @@ namespace gdpm::package_manager{
 		auto verboseOpt 				= joinable(repeatable(option("-v", "--verbose").call([]{ config.verbose += 1; }))) % "show verbose output";	
 
 		/* Set the options */
-		auto fileOpt 			= repeatable(option("--file", "-f").set(params.args)  % "read file as input");
+		// auto fileOpt 			= repeatable(option("--file", "-f").set(params.input_files)  % "read file as input");
+		auto fileOpt 				= repeatable(option("--file", "-f") & values("input", params.input_files)) % "read file as input";
 		auto cleanOpt 			= option("--clean").set(config.clean_temporary) % "enable/disable cleaning temps";
 		auto parallelOpt 		= option("--jobs").set(config.jobs) % "set number of parallel jobs";
 		auto cacheOpt 			= option("--enable-cache").set(config.enable_cache) % "enable/disable local caching";
@@ -132,7 +133,7 @@ namespace gdpm::package_manager{
 		auto removeCmd = "remove" % (
 			command("remove").set(action, action_e::remove),
 			packageValues % "packages(s) to remove from project",
-			fileOpt
+			fileOpt, skipOpt, cleanOpt
 		);
 		auto updateCmd = "update package(s)" % (
 			command("update").set(action, action_e::update),
@@ -140,7 +141,8 @@ namespace gdpm::package_manager{
 		);
 		auto searchCmd = "search for package(s)" % (
 			command("search").set(action, action_e::search),
-			packageValues % ""
+			packageValues % "",
+			godotVersionOpt, fileOpt, remoteOpt, configOpt
 		);
 		auto exportCmd = "export installed package list to file" % (
 			command("export").set(action, action_e::p_export),

src/rest_api.cpp

@@ -14,7 +14,6 @@
 #include <curlpp/Easy.hpp>
 #include <curlpp/Options.hpp>
 #include <curlpp/Exception.hpp>
-
 namespace gdpm::rest_api{
 	
 	request_params make_from_config(const config::context& config){
@@ -22,6 +21,7 @@ namespace gdpm::rest_api{
 		request_params params 	= make_request_params();
 		params.godot_version 	= (is_latest) ? "" : config.info.godot_version;
 		params.verbose 			= config.verbose;
+		params.max_results 		= config.max_results;
 		return params;
 	}
 
@@ -166,9 +166,10 @@ namespace gdpm::rest_api{
 		type_e type, 
 		int verbose
 	){
+		http::context http;
 		string request_url{url};
 		request_url += to_string(type);
-		http::response r = http::request_get(url);
+		http::response r = http.request_get(url);
 		if(verbose > 0)
 			log::info("URL: {}", url);
 		return _parse_json(r.body);
@@ -208,8 +209,14 @@ namespace gdpm::rest_api{
 		const string& url, 
 		const request_params& c
 	){
+		http::context http;
+		http::request_params http_params;
+		http_params.headers.insert(http::header("Accept", "*/*"));
+		http_params.headers.insert(http::header("Accept-Encoding", "application/gzip"));
+		http_params.headers.insert(http::header("Content-Encoding", "application/gzip"));
+		http_params.headers.insert(http::header("Connection", "keep-alive"));
 		string request_url = _prepare_request(url, c);
-		http::response r = http::request_get(request_url);
+		http::response r = http.request_get(request_url, http_params);
 		if(c.verbose > 0)
 			log::info("get_asset().URL: {}", request_url);
 		return _parse_json(r.body, c.verbose);
@@ -218,11 +225,19 @@ namespace gdpm::rest_api{
 	rapidjson::Document get_asset(
 		const string& url, 
 		int asset_id, 
-		const request_params& params
+		const rest_api::request_params& api_params
 	){
-		string request_url = utils::replace_all(_prepare_request(url, params), "{id}", std::to_string(asset_id));
-		http::response r = http::request_get(request_url.c_str());
-		if(params.verbose >= log::INFO)
+		/* Set up HTTP request */
+		http::context http;
+		http::request_params http_params;
+		http_params.headers.insert(http::header("Accept", "*/*"));
+		http_params.headers.insert(http::header("Accept-Encoding", "application/gzip"));
+		http_params.headers.insert(http::header("Content-Encoding", "application/gzip"));
+		http_params.headers.insert(http::header("Connection", "keep-alive"));
+
+		string request_url = utils::replace_all(_prepare_request(url, api_params), "{id}", std::to_string(asset_id));
+		http::response r = http.request_get(request_url.c_str(), http_params);
+		if(api_params.verbose >= log::INFO)
 			log::info("get_asset().URL: {}", request_url);
 		
 		return _parse_json(r.body);

src/utils.cpp

@@ -3,6 +3,11 @@
 #include "config.hpp"
 #include "constants.hpp"
 #include "log.hpp"
+#include "indicators/indeterminate_progress_bar.hpp"
+#include "indicators/dynamic_progress.hpp"
+#include "indicators/progress_bar.hpp"
+#include "indicators/block_progress_bar.hpp"
+#include "csv2/reader.hpp"
 
 
 #include <asm-generic/errno-base.h>
@@ -21,9 +26,56 @@
 #include <thread>
 #include <unordered_map>
 #include <zip.h>
+#include <curl/curl.h>
 
 namespace gdpm::utils{
 
+	using namespace indicators;
+	BlockProgressBar bar {
+		option::BarWidth{50},
+		// option::Start{"["},
+		// option::Fill{"="},
+		// option::Lead{">"},
+		// option::Remainder{" "},
+		// option::End{"]"},
+		option::PrefixText{"Downloading file "},
+		option::PostfixText{""},
+		option::ForegroundColor{Color::green},
+		option::FontStyles{std::vector<FontStyle>{FontStyle::bold}},
+	};
+		// option::ShowElapsedTime{true},
+		// option::ShowRemainingTime{true},
+	IndeterminateProgressBar bar_unknown {
+		option::BarWidth{50},
+		option::Start{"["},
+		option::Fill{"."},
+		option::Lead{"<==>"},
+		option::PrefixText{"Downloading file "},
+		option::End{"]"},
+		option::PostfixText{""},
+		option::ForegroundColor{Color::green},
+		option::FontStyles{std::vector<FontStyle>{FontStyle::bold}},
+	};
+
+	std::vector<std::string> split_lines(const std::string& contents){
+		using namespace csv2;
+		csv2::Reader<
+			delimiter<'\n'>,
+			quote_character<'"'>,
+			first_row_is_header<false>,
+			trim_policy::trim_whitespace
+		> csv;
+		std::vector<std::string> lines;
+		if(csv.parse(contents)){
+			for(const auto& row : csv){
+				for(const auto& cell : row){
+					lines.emplace_back(cell.read_view());
+				}
+			}
+		}
+		return lines;
+	}
+
 	
 	#if (GDPM_READFILE_IMPL == 0)
 		std::string readfile(const std::string& path){
@@ -142,7 +194,8 @@ namespace gdpm::utils{
 		int i, len, fd;
 		zip_uint64_t sum;	
 
-		log::info_n("Extracting package contents to '{}'...", dest);
+		// log::info_n("Extracting package contents to '{}'...", dest);
+		log::info_n("Extracting package contents...");
 		if((za = zip_open(archive, 0, &err)) == nullptr){
 			zip_error_to_str(buf, sizeof(buf), err, errno);
 			log::error("{}: can't open zip archive {}: {}", prog, archive, buf);
@@ -263,8 +316,104 @@ namespace gdpm::utils{
 		return o;
 	}
 
-	namespace json {
+	std::string convert_size(long size){
+		int digit = 0;
+		while(size > 1000){
+			size /= 1000;
+			digit += 1;
+		}
+		std::string s = std::to_string(size);
+		switch(digit){
+			case 0: return s + " B";
+			case 1: return s + " KB";
+			case 2: return s + " MB";
+			case 3: return s + " GB";
+			case 4: return s + " TB";
+			case 5: return s + " PB";
+		}
+		return std::to_string(size);
+	}
 
+
+	namespace curl {
+		size_t write_to_buffer(
+			char *contents, 
+			size_t size, 
+			size_t nmemb, 
+			void *userdata
+		){
+			size_t realsize = size * nmemb;
+			struct memory_buffer *m = (struct memory_buffer*)userdata;
+
+			m->addr = (char*)realloc(m->addr, m->size + realsize + 1);
+			if(m->addr == nullptr){
+				/* Out of memory */
+				fprintf(stderr, "Not enough memory (realloc returned NULL)\n");
+				return 0;
+			}
+
+			memcpy(&(m->addr[m->size]), contents, realsize);
+			m->size += realsize;
+			m->addr[m->size] = 0;
+
+			return realsize;
+		}
+
+		size_t write_to_stream(
+			char *ptr, 
+			size_t size, 
+			size_t nmemb, 
+			void *userdata
+		){
+			if(nmemb == 0)
+				return 0;
+						
+			return fwrite(ptr, size, nmemb, (FILE*)userdata);
+		}
+
+		int show_progress(
+			void *ptr,
+			curl_off_t total_download,
+			curl_off_t current_downloaded,
+			curl_off_t total_upload,
+			curl_off_t current_upload
+		){
+
+			if(current_downloaded >= total_download)
+				return 0;
+			using namespace indicators;
+			show_console_cursor(false);
+			if(total_download != 0){
+				// double percent = std::floor((current_downloaded / (total_download)) * 100);
+				bar.set_option(option::MaxProgress{total_download});
+				bar.set_progress(current_downloaded);
+				bar.set_option(option::PostfixText{
+						convert_size(current_downloaded) + " / " + 
+						convert_size(total_download)
+				});
+				if(bar.is_completed()){
+					bar.set_option(option::PrefixText{"Download completed."});
+					bar.mark_as_completed();
+				}
+			} else {
+				if(bar_unknown.is_completed()){
+					bar.set_option(option::PrefixText{"Download completed."});
+					bar.mark_as_completed();
+				} else {
+					bar.tick();
+					bar_unknown.set_option(
+						option::PostfixText(std::format("{}", convert_size(current_downloaded)))
+					);
+
+				}
+			}
+			show_console_cursor(true);
+			memory_buffer *m = (memory_buffer*)ptr;
+			return 0;
+		}
+	}
+
+	namespace json {
 		std::string from_array(
 			const std::set<std::string>& a, 
 			const std::string& prefix

submodules/indicators

@@ -0,0 +1 @@
+Subproject commit ef71abd9bc7254f7734fa84d5b1c336be2deb9f7