diff --git a/PKGBUILD b/PKGBUILD index 3cb57ce..4442ed1 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Sebastien MacDougall-Landry pkgname=libscry -pkgver=0.3 +pkgver=0.3.1 pkgrel=1 pkgdesc='A Magic: The Gathering library' url='https://github.com/EmperorPenguin18/libscry/' diff --git a/examples/test.cc b/examples/test.cc index aafcb4b..9f21411 100644 --- a/examples/test.cc +++ b/examples/test.cc @@ -86,6 +86,9 @@ int main(int argc, char **argv) cout << card->dual_sided() << endl; cout << card->json() << endl; cout << card->loyalty() << endl; + cout << card->set() << endl; + cout << card->price() << endl; + cout << card->legality()[0] << endl; cout << endl; cout << "cards_named(\"vannifar\", &img_size)" << endl; diff --git a/src/card.cc b/src/card.cc index f517e38..8e41e68 100644 --- a/src/card.cc +++ b/src/card.cc @@ -54,3 +54,35 @@ string Card::loyalty() { if (!data.HasMember("loyalty")) return ""; return data["loyalty"].GetString(); } + +string Card::set() { + return data["set_name"].GetString(); +} + +string Card::price() { + return data["prices"]["usd"].GetString(); +} + +vector Card::legality() { + vector output; + output.push_back(data["legalities"]["standard"].GetString()); + output.push_back(data["legalities"]["future"].GetString()); + output.push_back(data["legalities"]["historic"].GetString()); + output.push_back(data["legalities"]["gladiator"].GetString()); + output.push_back(data["legalities"]["pioneer"].GetString()); + output.push_back(data["legalities"]["explorer"].GetString()); + output.push_back(data["legalities"]["modern"].GetString()); + output.push_back(data["legalities"]["legacy"].GetString()); + output.push_back(data["legalities"]["pauper"].GetString()); + output.push_back(data["legalities"]["vintage"].GetString()); + output.push_back(data["legalities"]["penny"].GetString()); + output.push_back(data["legalities"]["commander"].GetString()); + output.push_back(data["legalities"]["brawl"].GetString()); + output.push_back(data["legalities"]["historicbrawl"].GetString()); + output.push_back(data["legalities"]["alchemy"].GetString()); + output.push_back(data["legalities"]["paupercommander"].GetString()); + output.push_back(data["legalities"]["duel"].GetString()); + output.push_back(data["legalities"]["oldschool"].GetString()); + output.push_back(data["legalities"]["premodern"].GetString()); + return output; +} diff --git a/src/card.h b/src/card.h index 77fb1b9..cf60047 100644 --- a/src/card.h +++ b/src/card.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include #include @@ -34,6 +35,12 @@ class Card { virtual string json(); ///Get the starting loyalty of the card virtual string loyalty(); + ///Get the set name of the card + virtual string set(); + ///Get the USD non-foil price of the most recent printing of the card + virtual string price(); + ///Get the legalities (for most formats) of the card + virtual vector legality(); private: Document data; }; diff --git a/src/data.cc b/src/data.cc index 16005b2..331f4b1 100644 --- a/src/data.cc +++ b/src/data.cc @@ -70,22 +70,9 @@ DataAccess::~DataAccess() { } void DataAccess::db_copy(int isSave) { -#ifdef DEBUG - cerr << "DB Copy:" << endl; -#endif sqlite3 *pFile; int rc = sqlite3_open(fname, &pFile); if (rc == SQLITE_OK) { - /*size_t size = 0; - char* page_size = (isSave ? (char*)sql_read(db, "PRAGMA PAGE_SIZE;", &size) : (char*)sql_read(pFile, "PRAGMA PAGE_SIZE;", &size) ); -#ifdef DEBUG - cerr << "Page size: " << (char*)page_size << endl; -#endif - char cmd[30] = "pragma page_size = "; - strcat(cmd, page_size); - strcat(cmd, ";"); - if (isSave) sql_write(pFile, cmd, NULL, size); - else sql_write(db, cmd, NULL, size);*/ sqlite3 *pFrom = (isSave ? db : pFile); sqlite3 *pTo = (isSave ? pFile : db); sqlite3_backup *pBackup = sqlite3_backup_init(pTo, "main", pFrom, "main"); @@ -104,7 +91,7 @@ void DataAccess::db_copy(int isSave) { byte* DataAccess::sql_read(sqlite3 *pDb, const char* cmd, size_t* size) { #ifdef DEBUG - cerr << "Sql call: " << cmd << endl; + fprintf(stderr, "Sql call: %s\n", cmd); #endif sqlite3_stmt *stmt; int rc = sqlite3_prepare_v2(pDb, cmd, -1, &stmt, NULL); @@ -125,9 +112,10 @@ byte* DataAccess::sql_read(sqlite3 *pDb, const char* cmd, size_t* size) { } *size = sqlite3_column_bytes(stmt, 0); #ifdef DEBUG - cerr << "Database returned (250 chars): "; - for (size_t i = 0; i < min((size_t)250, *size); i++) cerr << (char)rawoutput[i]; - cerr << endl; + fprintf(stderr, "Database returned (250 chars): "); + if (*size < 250) for (size_t i = 0; i < *size; i++) fprintf(stderr, "%c", (char)rawoutput[i]); + else for (size_t i = 0; i < 250; i++) fprintf(stderr, "%c", (char)rawoutput[i]); + fprintf(stderr, "\n"); #endif byte* output = (byte*)malloc(sizeof(byte)*(*size) + 1); if (!output) { @@ -142,7 +130,7 @@ byte* DataAccess::sql_read(sqlite3 *pDb, const char* cmd, size_t* size) { void DataAccess::sql_write(sqlite3 *pDb, const char* cmd, byte* data, const size_t& size) { #ifdef DEBUG - cerr << "Sql call: " << cmd << endl; + fprintf(stderr, "Sql call: %s\n", cmd); #endif sqlite3_stmt *stmt; int rc = sqlite3_prepare_v2(pDb, cmd, -1, &stmt, NULL); @@ -151,9 +139,10 @@ void DataAccess::sql_write(sqlite3 *pDb, const char* cmd, byte* data, const size exit(1); } #ifdef DEBUG - cerr << "Data input (250 chars): "; - for (size_t i = 0; i < min((size_t)250, size); i++) cerr << (char)data[i]; - cerr << endl; + fprintf(stderr, "Data input (250 chars): "); + if (size < 250) for (size_t i = 0; i < size; i++) fprintf(stderr, "%c", (char)data[i]); + else for (size_t i = 0; i < 250; i++) fprintf(stderr, "%c", (char)data[i]); + fprintf(stderr, "\n"); #endif rc = sqlite3_bind_blob(stmt, 1, data, size, SQLITE_STATIC); rc = sqlite3_step(stmt); @@ -164,14 +153,14 @@ void DataAccess::sql_write(sqlite3 *pDb, const char* cmd, byte* data, const size } #ifdef DEBUG if (rc == SQLITE_ROW) - cerr << "Write returned: " << (char*)sqlite3_column_blob(stmt, 0) << endl; + fprintf(stderr, "Write returned: %s\n", (char*)sqlite3_column_blob(stmt, 0)); #endif sqlite3_finalize(stmt); } void DataAccess::sql_exec(sqlite3 *pDb, const char* cmd) { #ifdef DEBUG - cerr << "Sql call: " << cmd << endl; + fprintf(stderr, "Sql call: %s\n", cmd); #endif int rc = sqlite3_exec(pDb, cmd, NULL, NULL, NULL); if (rc != SQLITE_OK) { @@ -221,9 +210,6 @@ int DataAccess::datecheck(const char* table, const char* search) { } void DataAccess::db_exec(const char* table) { -#ifdef DEBUG - cerr << "Creating table: " << table << endl; -#endif string cmd = "CREATE TABLE IF NOT EXISTS " + string(table) + "(Key TEXT NOT NULL, Updated DATETIME NOT NULL, Value BLOB NOT NULL, PRIMARY KEY(Key));"; sql_exec(db, cmd.c_str()); } diff --git a/src/data.h b/src/data.h index 2d650f6..70b79df 100644 --- a/src/data.h +++ b/src/data.h @@ -11,11 +11,6 @@ #include #include -#ifdef DEBUG -#include -#include -#endif - using namespace std; ///This class is used to access the database diff --git a/src/list.cc b/src/list.cc index 5dae06a..ee73cd8 100644 --- a/src/list.cc +++ b/src/list.cc @@ -7,28 +7,11 @@ using namespace std; using namespace rapidjson; -List::List(const char * rawjson) { - construct(rawjson); -} - -List::List(vector rawjsons) { - string str = ""; - for (int i = 0; i < rawjsons.size(); i++) { - str += rawjsons[i] + '\n'; -#ifdef DEBUG - cerr << "Last ten chars: " << rawjsons[i].substr(rawjsons[i].length()-10, 10) << endl; - cerr << "Size: " << rawjsons[i].size() << endl << "Capacity: " << rawjsons[i].capacity() << endl; -#endif - construct(rawjsons[i].c_str()); - } - data.Parse(str.c_str()); -} - -void List::construct(const char * rawjson) { +void List::construct(char* rawjson) { data.Parse(rawjson); if (strcmp(data["object"].GetString(), "error") == 0) throw "Invalid List"; #ifdef DEBUG - if (data.IsObject()) cerr << "JSON is valid" << endl; + if (data.IsObject()) fprintf(stderr, "JSON is valid\n"); #endif for (int i = 0; i < data["data"].Size(); i++) { StringBuffer buffer; @@ -45,12 +28,20 @@ void List::construct(const char * rawjson) { smatch sm2; regex_search(url, sm2, q); smatch sm3; regex_search(url, sm3, page); #ifdef DEBUG - cerr << "Regex 1: " << sm1[0] << endl << "Regex 2: " << sm2[0] << endl << "Regex 3: " << sm3[0] << endl; + fprintf(stderr, "Regex 1: %s\nRegex 2: %s\nRegex 3: %s\n", sm1[0].str().c_str(), sm2[0].str().c_str(), sm3[0].str().c_str()); #endif nextpage = string(sm1[0]) + string(sm2[0]) + string(sm3[0]).substr(0, 5); } else nextpage = ""; } +List::List(vector rawjson) { + for (int i = 0; i < rawjson.size(); i++) construct(rawjson[i]); +} + +List::List(char* rawjson) { + construct(rawjson); +} + List::~List() { while (!content.empty()) { delete content.back(); diff --git a/src/list.h b/src/list.h index 3056a3c..940b007 100644 --- a/src/list.h +++ b/src/list.h @@ -10,15 +10,11 @@ #include #include "card.h" -#ifdef DEBUG -#include -#endif - ///This class is used to represent a list of cards (returned from a search for example) class List { public: - List(const char*); - List(vector); + List(char*); + List(vector); ~List(); ///Returns a vector with all the cards on this page of the list. For cards on all pages see allcards(). @@ -31,6 +27,6 @@ class List { Document data; vector content; string nextpage; - void construct(const char*); + virtual void construct(char*); }; diff --git a/src/scry-private.cc b/src/scry-private.cc index f7dcd60..6283710 100644 --- a/src/scry-private.cc +++ b/src/scry-private.cc @@ -8,7 +8,7 @@ using namespace std; vector Scry::explode(const char* str, const char& ch) { #ifdef DEBUG - cerr << "Exploding: " << str << endl; + fprintf(stderr, "Exploding: %s\n", str); #endif cstring_t next; next.len = 0; next.max = 0; next.str = NULL; vector result; @@ -17,14 +17,20 @@ vector Scry::explode(const char* str, const char& ch) { if (next.str) { string_cat(&next, '\0'); #ifdef DEBUG - cerr << "Line " << result.size() << ": " << next.str << endl; + fprintf(stderr, "Line %d: %s\n", result.size(), next.str); #endif result.push_back(next.str); next.len = 0; next.max = 0; next.str = NULL; } } else string_cat(&next, str[i]); } - if (next.str) result.push_back(next.str); + if (next.str) { + string_cat(&next, '\0'); +#ifdef DEBUG + fprintf(stderr, "Line %d: %s\n", result.size(), next.str); +#endif + result.push_back(next.str); + } return result; } @@ -38,7 +44,7 @@ string Scry::implode(const vector& strs, const char& ch) { void Scry::replace(char* str, char c, const char* s) { #ifdef DEBUG - cerr << "Replacing: " << str << endl; + fprintf(stderr, "Replacing: %s\n", str); #endif size_t length = strlen(str); for (size_t i = 0; i < length; i++) { @@ -48,10 +54,6 @@ void Scry::replace(char* str, char c, const char* s) { char prev[length_s] = ""; strcat(prev, s); length += length_s-1; - if (!str) { - fprintf(stderr, "not enough memory: realloc returned null"); - exit(1); - } str[i] = prev[0]; for (size_t j = i; j < length; j+=length_s-1) { for (size_t k = 1; k < length_s; k++) { @@ -64,13 +66,13 @@ void Scry::replace(char* str, char c, const char* s) { } } #ifdef DEBUG - cerr << "Replaced with: " << str << endl; + fprintf(stderr, "Replaced with: %s\n", str); #endif } void Scry::firstupper(char* str) { #ifdef DEBUG - cerr << "First upper input: " << str << endl; + fprintf(stderr, "First upper input: %s\n", str); #endif for(int i = 0; i < strlen(str); i++) { if (i == 0 || str[i-1] == ' ' || str[i-1] == '-') { @@ -80,12 +82,12 @@ void Scry::firstupper(char* str) { } } #ifdef DEBUG - cerr << "First upper output: " << str << endl; + fprintf(stderr, "First upper output: %s\n", str); #endif } char* Scry::urlformat(const string& str) { - char* output = (char*)malloc(sizeof(char)*str.length()*3+1); + char* output = (char*)calloc(str.length()*3, sizeof(char)*str.length()*3+1); if (!output) { fprintf(stderr, "not enough memory: malloc returned null"); exit(1); @@ -96,22 +98,21 @@ char* Scry::urlformat(const string& str) { replace(output, '<', "%3C"); replace(output, '>', "%3E"); #ifdef DEBUG - cerr << "URL formatted to: " << output << endl; + fprintf(stderr, "URL formatted to: %s\n", output); #endif return output; } char* Scry::nameformat(const string& str) { - char* output = (char*)malloc(sizeof(char)*str.length()*2+1); + char* output = (char*)calloc(str.length()*2, sizeof(char)*str.length()*2+1); if (!output) { fprintf(stderr, "not enough memory: malloc returned null"); exit(1); } strcpy(output, str.c_str()); replace(output, '\'', "''"); - firstupper(output); #ifdef DEBUG - cerr << "Name formatted to: " << output << endl; + fprintf(stderr, "Name formatted to: %s\n", output); #endif return output; } @@ -124,20 +125,21 @@ char* Scry::cachecard(List* list) { for (int i = 0; i < cards.size(); i++) { char* name = nameformat(cards[i]->name()); string_cat(&output, name); - string_cat(&output, "\n"); + string_cat(&output, '\n'); names.push_back(name); string json = cards[i]->json(); - char* cstr = new char[json.length()+1]; + char* cstr = (char*)calloc(json.length()+1, sizeof(char)*json.length()+1); strcpy(cstr, json.c_str()); data.push_back(cstr); } da->db_exec("Cards", names, data); output.len--; + string_cat(&output, '\0'); return output.str; } -List * Scry::allcards(List * list) { - List * newlist; +List* Scry::allcards(List* list) { + List* newlist; if (list->nextPage() != "") { Document doc; doc.Parse(list->json().c_str()); unsigned int pages = static_cast( @@ -146,20 +148,28 @@ List * Scry::allcards(List * list) { ) ); #ifdef DEBUG - cerr << "# of pages: " << to_string(pages) << endl; + fprintf(stderr, "# of pages: %d\n", pages); #endif - vector urls; + char** urls = (char**)malloc(sizeof(char*)*(pages-1)); int i; - for (i = 2; i <= pages; i++) urls.push_back(list->nextPage() + to_string(i)); - vector one; one.push_back(list->json()); - vector two = wa->api_call(urls); - two.insert(two.begin(), one.begin(), one.end()); - newlist = new List(two); + for (i = 1; i < pages; i++) { + urls[i-1] = (char*)calloc(list->nextPage().size()+4, list->nextPage().size()+4); + sprintf(urls[i-1], "%s%d", list->nextPage().c_str(), i+1); + } + vector results = wa->api_call(urls, pages-1); + for (int i = 0; i < pages-1; i++) free(urls[i]); + free(urls); + char str[list->json().size()+1] = ""; + strcat(str, list->json().c_str()); + results.insert(results.begin(), str); + newlist = new List(results); lists.push_back(newlist); while (newlist->nextPage() != "") { - string extrapage = wa->api_call(newlist->nextPage() + to_string(i)); - two.push_back(extrapage); - newlist = new List(two); + char url[newlist->nextPage().size()+4] = ""; + sprintf(url, "%s%d", newlist->nextPage().c_str(), i); + char* extrapage = wa->api_call(url); + results.push_back(extrapage); + newlist = new List(results); lists.push_back(newlist); i++; } @@ -170,12 +180,12 @@ List * Scry::allcards(List * list) { void Scry::string_cat(cstring_t* dest, const char* src) { size_t newlen = strlen(src); #ifdef DEBUG - cerr << "String cat: " << dest->len << ", " << dest->max << ", " << newlen << endl; + fprintf(stderr, "String cat: %d, %d, %d\n", dest->len, dest->max, newlen); #endif if (dest->max < newlen+dest->len) { dest->max = max(dest->max * 2, newlen+dest->len+1); #ifdef DEBUG - cerr << "Realloc to size: " << dest->max << endl; + fprintf(stderr, "Realloc to size: %d\n", dest->max); #endif dest->str = (char*)realloc(dest->str, dest->max); if (!dest->str) { @@ -189,12 +199,12 @@ void Scry::string_cat(cstring_t* dest, const char* src) { void Scry::string_cat(cstring_t* dest, char c) { #ifdef DEBUG - cerr << "String cat: " << dest->len << ", " << dest->max << ", 1" << endl; + fprintf(stderr, "String cat: %d, %d, 1\n", dest->len, dest->max); #endif if (dest->max < 1+dest->len) { dest->max = max(dest->max * 2, dest->len+2); #ifdef DEBUG - cerr << "Realloc to size: " << dest->max << endl; + fprintf(stderr, "Realloc to size: %d\n", dest->max); #endif dest->str = (char*)realloc(dest->str, dest->max); if (!dest->str) { diff --git a/src/scry-public.cc b/src/scry-public.cc index 7d59e44..8e68c3f 100644 --- a/src/scry-public.cc +++ b/src/scry-public.cc @@ -20,7 +20,7 @@ Scry::Scry() { signal(SIGSEGV, print_stacktrace); signal(SIGABRT, print_stacktrace); #endif - vector temp; + vector temp; temp.push_back("api.scryfall.com"); wa = new WebAccess(temp, 50, 20); da = new DataAccess("libscry"); @@ -72,12 +72,13 @@ void print_stacktrace(int signum) { #endif List* Scry::cards_search(string in) { - string query = string(urlformat(in)); - string url = "https://api.scryfall.com/cards/search?q=" + query; + char* query = urlformat(in); + char url[strlen(query)+50] = ""; + strcat(url, "https://api.scryfall.com/cards/search?q="); + strcat(url, query); #ifdef DEBUG - cerr << "URL: " << url << endl; + fprintf(stderr, "URL: %s\n", url); #endif - size_t size = 0; List* list; try { list = new List(wa->api_call(url)); @@ -85,13 +86,14 @@ List* Scry::cards_search(string in) { return nullptr; } #ifdef DEBUG - cerr << "First card: " << list->cards()[0]->name() << endl; + fprintf(stderr, "First card: %s\n", list->cards()[0]->name().c_str()); #endif lists.push_back(list); List* full_list = allcards(list); #ifdef DEBUG - cerr << "Full first card: " << full_list->cards()[0]->name() << endl; + fprintf(stderr, "Full first card: %s\n", full_list->cards()[0]->name().c_str()); #endif + free(query); return full_list; } @@ -116,7 +118,7 @@ List* Scry::cards_search_cache(string in) { data.len--; string_cat(&data, "],\"has_more\":false}"); #ifdef DEBUG - cerr << "New list (last 50 chars): " << &(data.str[strlen(data.str)-50]) << endl; + fprintf(stderr, "New list (last 50 chars): %s\n", data.str+strlen(data.str)-50); #endif list = new List(data.str); lists.push_back(list); @@ -128,8 +130,10 @@ List* Scry::cards_search_cache(string in) { } Card* Scry::cards_named(string in) { - string query = string(urlformat(in)); - string url = "https://api.scryfall.com/cards/named?fuzzy=" + query; + char* query = urlformat(in); + char url[strlen(query)+50] = ""; + strcat(url, "https://api.scryfall.com/cards/named?fuzzy="); + strcat(url, query); Card* card; try { card = new Card(wa->api_call(url)); @@ -137,12 +141,14 @@ Card* Scry::cards_named(string in) { return nullptr; } cards.push_back(card); + free(query); return card; } Card* Scry::cards_named_cache(string in) { Card* card; char* name = nameformat(in); + firstupper(name); if (da->datecheck("Cards", name) == 1) { card = cards_named(in); @@ -161,15 +167,20 @@ Card* Scry::cards_named_cache(string in) { } byte* Scry::cards_named(string in, size_t *size) { - string query = string(urlformat(in)); - string url = "https://api.scryfall.com/cards/named?fuzzy=" + query + "&format=image&version=border_crop"; + char* query = urlformat(in); + char url[strlen(query)+100] = ""; + strcat(url, "https://api.scryfall.com/cards/named?fuzzy="); + strcat(url, query); + strcat(url, "&format=image&version=border_crop"); byte* image = wa->api_call(url, size); + free(query); return image; } byte* Scry::cards_named_cache(string in, size_t *size) { byte* image; char* name = nameformat(in); + firstupper(name); if (da->datecheck("Images", name) == 1) { image = cards_named(in, size); @@ -183,13 +194,16 @@ byte* Scry::cards_named_cache(string in, size_t *size) { } vector Scry::cards_autocomplete(string in) { - string query = string(urlformat(in)); - string url = "https://api.scryfall.com/cards/autocomplete?q=" + query; + char* query = urlformat(in); + char url[strlen(query)+50] = ""; + strcat(url, "https://api.scryfall.com/cards/autocomplete?q="); + strcat(url, query); Document doc; doc.Parse(wa->api_call(url)); const Value& a = doc["data"]; vector output; for (auto& v : a.GetArray()) output.push_back(v.GetString()); + free(query); return output; } @@ -212,7 +226,7 @@ vector Scry::cards_autocomplete_cache(string in) { } Card* Scry::cards_random() { - string url = "https://api.scryfall.com/cards/random"; + char url[50] = "https://api.scryfall.com/cards/random"; Card* card = new Card(wa->api_call(url)); cards.push_back(card); return card; diff --git a/src/scry.h b/src/scry.h index a20b9f8..6e6555b 100644 --- a/src/scry.h +++ b/src/scry.h @@ -3,7 +3,6 @@ //https://github.com/EmperorPenguin18/libscry/blob/main/LICENSE #pragma once -#include #include "web.h" #include "data.h" #include "list.h" diff --git a/src/web.cc b/src/web.cc index a131501..4e8c504 100644 --- a/src/web.cc +++ b/src/web.cc @@ -5,40 +5,39 @@ #include "web.h" using namespace std; -using namespace std::chrono; WebAccess::WebAccess() { - delay = duration>(50); + delay = 50; conn_per_thread = 10; construct(); } -WebAccess::WebAccess(vector approved_urls) : approved_urls(approved_urls) { - delay = duration>(50); +WebAccess::WebAccess(vector approved_urls) : approved_urls(approved_urls) { + delay = 50; conn_per_thread = 10; construct(); } -WebAccess::WebAccess(vector approved_urls, long delay) : approved_urls(approved_urls), delay(delay) { +WebAccess::WebAccess(vector approved_urls, double delay) : approved_urls(approved_urls), delay(delay) { conn_per_thread = 10; construct(); } -WebAccess::WebAccess(vector approved_urls, long delay, size_t conn_per_thread) : approved_urls(approved_urls), delay(delay), conn_per_thread(conn_per_thread) { +WebAccess::WebAccess(vector approved_urls, double delay, size_t conn_per_thread) : approved_urls(approved_urls), delay(delay), conn_per_thread(conn_per_thread) { construct(); } -WebAccess::WebAccess(long delay) : delay(delay) { +WebAccess::WebAccess(double delay) : delay(delay) { conn_per_thread = 10; construct(); } -WebAccess::WebAccess(long delay, size_t conn_per_thread) : delay(delay), conn_per_thread(conn_per_thread) { +WebAccess::WebAccess(double delay, size_t conn_per_thread) : delay(delay), conn_per_thread(conn_per_thread) { construct(); } void WebAccess::construct() { - curl_lib = dlopen("libcurl.so", RTLD_LAZY | RTLD_DEEPBIND); + curl_lib = dlopen("libcurl.so", RTLD_LAZY); if (!curl_lib) { fprintf(stderr, "%s\n", dlerror()); exit(EXIT_FAILURE); @@ -70,8 +69,8 @@ WebAccess::~WebAccess() { size_t WebAccess::cb(void *data, size_t size, size_t nmemb, void *userp) { size_t realsize = size * nmemb; - struct memory *mem = (struct memory *)userp; - byte *ptr = (byte *)realloc(mem->response, *(mem->size) + realsize + 1); + memory* mem = (memory*)userp; + byte* ptr = (byte*)realloc(mem->response, *(mem->size) + realsize + 1); if (!ptr) { /* out of memory */ fprintf(stderr, "not enough memory (realloc returned NULL)\n"); @@ -80,13 +79,12 @@ size_t WebAccess::cb(void *data, size_t size, size_t nmemb, void *userp) { mem->response = ptr; memcpy(&(mem->response[*(mem->size)]), data, realsize); *(mem->size) += realsize; - //mem->response[mem->size] = (byte)0; return realsize; } -CURL * WebAccess::add_transfer(string url, struct memory* chunk, int num) { +CURL* WebAccess::add_transfer(const char* url, memory* chunk, int num) { #ifdef DEBUG - cerr << "Adding url: " << url << endl; + fprintf(stderr, "Adding url: %s\n", url); #endif *(chunk->size) = 0; CURL *eh = curl_easy_init(); @@ -94,27 +92,56 @@ CURL * WebAccess::add_transfer(string url, struct memory* chunk, int num) { curl_easy_setopt(eh, CURLOPT_HTTPGET, 1); curl_easy_setopt(eh, CURLOPT_USERAGENT, "libcurl-agent/1.0"); curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, cb); - curl_easy_setopt(eh, CURLOPT_WRITEDATA, (void *)chunk); - curl_easy_setopt(eh, CURLOPT_URL, url.c_str()); + curl_easy_setopt(eh, CURLOPT_WRITEDATA, (void*)chunk); + curl_easy_setopt(eh, CURLOPT_URL, url); +#ifdef DEBUG curl_easy_setopt(eh, CURLOPT_PRIVATE, num); +#endif return eh; } -void WebAccess::checkurl(string url) { - regex first(".*//"); - regex last("/.*"); - url = regex_replace(url, first, ""); - url = regex_replace(url, last, ""); - if ( std::find(approved_urls.begin(), approved_urls.end(), url) == approved_urls.end() ) { - fprintf(stderr, "Attempted to use unapproved url: %s\n", url.c_str()); +char* WebAccess::strremove(char *str, const char *sub) { + char *p, *q, *r; + if (*sub && (q = r = strstr(str, sub)) != NULL) { + size_t len = strlen(sub); + while ((r = strstr(p = r + len, sub)) != NULL) { + memmove(q, p, r - p); + q += r - p; + } + memmove(q, p, strlen(p) + 1); + } + return str; +} + +void WebAccess::checkurl(const char* url) { +#ifdef DEBUG + mtx.lock(); + fprintf(stderr, "Checking URL: %s\n", url); + mtx.unlock(); +#endif + char temp[strlen(url)+1] = ""; + strcat(temp, url); + strremove(temp, "https://"); + char temp2[strlen(temp)+1] = ""; + strncat(temp2, temp, strchr(temp, '/')-temp); +#ifdef DEBUG + mtx.lock(); + fprintf(stderr, "Relevant section: %s\n", temp2); + mtx.unlock(); +#endif + size_t i; + for (i = 0; i < approved_urls.size(); i++) if (strcmp(approved_urls[i], temp2) == 0) break; + if (i == approved_urls.size()) { + fprintf(stderr, "Attempted to use unapproved url: %s\n", temp2); exit(EXIT_FAILURE); } } -byte* WebAccess::api_call(string url, size_t* size) { +byte* WebAccess::api_call(const char* url, size_t* size) { checkurl(url); - struct memory chunk = {0}; + memory chunk; + chunk.response = (byte*)malloc(sizeof(byte)); chunk.size = size; CURL *eh = add_transfer(url, &chunk, 0); @@ -124,85 +151,80 @@ byte* WebAccess::api_call(string url, size_t* size) { exit(res); } #ifdef DEBUG - cerr << "First 250 chars of response: "; - for (size_t i = 0; i < min((size_t)250, *(chunk.size)); i++) cerr << (char)chunk.response[i]; - cerr << endl; + fprintf(stderr, "First 250 chars of response: "); + if (*(chunk.size) < 250) for (size_t i = 0; i < *(chunk.size); i++) fprintf(stderr, "%c", (char)chunk.response[i]); + else for (size_t i = 0; i < 250; i++) fprintf(stderr, "%c", (char)chunk.response[i]); + fprintf(stderr, "\n"); #endif curl_easy_cleanup(eh); return chunk.response; } -char* WebAccess::api_call(string url) { - size_t size = 0; +char* WebAccess::api_call(const char* url) { + size_t size; char* output = (char*)api_call(url, &size); output[size] = '\0'; return output; } -vector WebAccess::start_multi(vector urls) { +CURL* WebAccess::add_transfer_multi(const char* url, struct memory* chunk, size_t num) { + checkurl(url); + chunk->response = (byte*)malloc(sizeof(byte)); + chunk->size = (size_t*)malloc(sizeof(size_t)); + return add_transfer(url, chunk, num); +} + +WebAccess::memory* WebAccess::start_multi(char** urls, size_t conns) { #ifdef DEBUG mtx.lock(); - cerr << "All urls: " << endl; - for (int i = 0; i < urls.size(); i++) cerr << urls[i] << endl; + fprintf(stderr, "All urls: \n"); + for (int i = 0; i < conns; i++) fprintf(stderr, "%s\n", urls[i]); mtx.unlock(); #endif CURLM *cm; CURLMsg *msg; - unsigned int transfers = 0; + size_t transfers = 0; int msgs_left = -1; int still_alive = 1; - unsigned int conns = min(urls.size(), conn_per_thread); - vector output(conns); - struct memory chunks[conns]; - size_t sizes[conns]; + memory* chunks = (memory*)malloc(sizeof(memory)*conns); cm = curl_multi_init(); curl_multi_setopt(cm, CURLMOPT_MAXCONNECTS, conn_per_thread); - for (transfers = 0; transfers < conns; transfers++) { - checkurl(urls[transfers]); - chunks[transfers] = {0}; - sizes[transfers] = 0; - chunks[transfers].size = &sizes[transfers]; - curl_multi_add_handle(cm, add_transfer(urls[transfers], &chunks[transfers], transfers)); + size_t start = (conns > conn_per_thread) ? conn_per_thread : conns; + for (transfers = 0; transfers < start; transfers++) { + curl_multi_add_handle(cm, add_transfer_multi(urls[transfers], &chunks[transfers], transfers)); } #ifdef DEBUG mtx.lock(); - cerr << "Handles added" << endl; + fprintf(stderr, "Handles added\n"); mtx.unlock(); #endif do { - duration> time_span = duration_cast>>(steady_clock::now() - prev_time); + double time_span = (((double)(clock()-prev_time))/CLOCKS_PER_SEC)*1000; if (time_span > delay) { curl_multi_perform(cm, &still_alive); mtx.lock(); - prev_time = steady_clock::now(); + prev_time = clock(); mtx.unlock(); } while ((msg = curl_multi_info_read(cm, &msgs_left))) { #ifdef DEBUG mtx.lock(); - cerr << "Msg ready" << endl; + fprintf(stderr, "Msg ready\n"); mtx.unlock(); #endif if (msg->msg == CURLMSG_DONE) { - int num; CURL *e = msg->easy_handle; - curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &num); -#ifdef DEBUG - mtx.lock(); - cerr << "Transfer num: " << to_string(num) << endl; - mtx.unlock(); -#endif - chunks[num].response[*(chunks[num].size)] = (byte)'\0'; - output[num].reserve(strlen((char*)chunks[num].response)); - output[num].assign((char*)chunks[num].response); #ifdef DEBUG + int num; + curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &num); + chunks[num].response[*(chunks[num].size)] = (byte)'\0'; mtx.lock(); - cerr << "First 250 chars of response: " << output[num].substr(0, 250) << endl; + fprintf(stderr, "Transfer num: %d\nSize: %d\nFirst 250 chars of response: %.250s\n", num, *(chunks[num].size), (char*)chunks[num].response); mtx.unlock(); #endif curl_multi_remove_handle(cm, e); @@ -211,52 +233,70 @@ vector WebAccess::start_multi(vector urls) { else { fprintf(stderr, "E: CURLMsg (%d)\n", msg->msg); } + if (transfers < conns) { + curl_multi_add_handle(cm, add_transfer_multi(urls[transfers], &chunks[transfers], transfers)); + transfers++; + } } if (still_alive) curl_multi_wait(cm, NULL, 0, 1000, NULL); - } while (still_alive || (transfers < urls.size())); + } while (still_alive || (transfers < conns)); curl_multi_cleanup(cm); - return output; + return chunks; } -vector WebAccess::api_call(vector urls) { - unsigned available_threads = thread::hardware_concurrency()-1; //Not actual max, but keeps things reasonable +vector WebAccess::api_call(char** urls, size_t size) { + unsigned available_threads = thread::hardware_concurrency()-1; unsigned used_threads = static_cast(ceil( - static_cast(urls.size()) / static_cast(conn_per_thread) + static_cast(size) / static_cast(conn_per_thread) )); #ifdef DEBUG - cerr << "Available threads: " << to_string(available_threads) << endl - << "Used threads: " << to_string(used_threads) << endl; + fprintf(stderr, "Available threads: %d\nUsed threads: %d\n", available_threads, used_threads); #endif + vector output; if ( (available_threads > 0) && (available_threads >= used_threads) ) { - vector output; - vector>> threads(used_threads); + vector> threads(used_threads); + size_t sizes[used_threads]; for (int i = 0; i < used_threads; i++) { - vector data( min(conn_per_thread, urls.size()-i*conn_per_thread) ); - for (int j = 0; j < data.size(); j++) data.at(j) = urls.at(i*conn_per_thread+j); + sizes[i] = 0; + for (int j = 0; j < conn_per_thread; j++) { + if (i*conn_per_thread+j < size) { + sizes[i]++; + } + } #ifdef DEBUG mtx.lock(); - cerr << "Thread " << to_string(i) << " is getting: " << endl; - for (int j = 0; j < data.size(); j++) cerr << data.at(j) << endl; + fprintf(stderr, "Thread %d is getting: \n", i); + for (int j = 0; j < sizes[i]; j++) fprintf(stderr, "%s\n", *(urls+(i*conn_per_thread)+j)); mtx.unlock(); #endif - threads.at(i) = async(&WebAccess::start_multi, this, data); + threads.at(i) = async(&WebAccess::start_multi, this, urls+(i*conn_per_thread), sizes[i]); } for (int i = 0; i < used_threads; i++) { - vector newdata = threads[i].get(); - for (int j = 0; j < newdata.size(); j++) { + memory* newdata = threads[i].get(); + for (int j = 0; j < sizes[i]; j++) { + newdata[j].response[*(newdata[j].size)] = (byte)'\0'; #ifdef DEBUG mtx.lock(); - cerr << "Adding to output: " << newdata[j].substr(0, 10) << endl; + fprintf(stderr, "Adding to output: %.10s\n", (char*)newdata[j].response); mtx.unlock(); #endif - output.push_back(newdata[j]); + output.push_back((char*)newdata[j].response); } } return output; } - start_multi(urls); - return urls; + memory* newdata = start_multi(urls, size); + for (int j = 0; j < size; j++) { + newdata[j].response[*(newdata[j].size)] = (byte)'\0'; +#ifdef DEBUG + mtx.lock(); + fprintf(stderr, "Adding to output: %.10s\n", (char*)newdata[j].response); + mtx.unlock(); +#endif + output.push_back((char*)newdata[j].response); + } + return output; } diff --git a/src/web.h b/src/web.h index cefb984..56348cb 100644 --- a/src/web.h +++ b/src/web.h @@ -3,43 +3,31 @@ //https://github.com/EmperorPenguin18/libscry/blob/main/LICENSE #pragma once -#include #include #include -#include -#include -#include -#include #include #include #include -#include -#include #include #include -#ifdef DEBUG -#include -#endif - using namespace std; -using namespace std::chrono; class WebAccess { public: WebAccess(); - WebAccess(vector); - WebAccess(vector, long); - WebAccess(vector, long, size_t); - WebAccess(long); - WebAccess(long, size_t); + WebAccess(vector); + WebAccess(vector, double); + WebAccess(vector, double, size_t); + WebAccess(double); + WebAccess(double, size_t); ~WebAccess(); - virtual byte* api_call(string, size_t*); - virtual char* api_call(string); - virtual vector api_call(vector); + virtual byte* api_call(const char*, size_t*); + virtual char* api_call(const char*); + virtual vector api_call(char**, size_t); private: - void construct(); + virtual void construct(); void* curl_lib; typedef CURL* (*cgi_handle)(int); cgi_handle curl_global_init; @@ -80,13 +68,15 @@ class WebAccess { size_t* size; }; - vector approved_urls; - void checkurl(string); + vector approved_urls; + virtual char* strremove(char*, const char*); + virtual void checkurl(const char*); - duration> delay; - steady_clock::time_point prev_time; + double delay; + clock_t prev_time; size_t conn_per_thread; - CURL* add_transfer(string, struct memory*, int); + virtual CURL* add_transfer(const char*, struct memory*, int); mutex mtx; - vector start_multi(vector); + virtual CURL* add_transfer_multi(const char*, struct memory*, size_t); + virtual struct memory* start_multi(char**, size_t); };