diff --git a/.gitignore b/.gitignore index d1b90137..5dd24e6c 100644 --- a/.gitignore +++ b/.gitignore @@ -134,4 +134,6 @@ env_variables.sh lib/**/*.html lib/**/*.css -debug.log \ No newline at end of file +debug.log + +backups/ \ No newline at end of file diff --git a/data_base.sqlite b/data_base.sqlite index 64dbecef..b1b9a4a8 100644 Binary files a/data_base.sqlite and b/data_base.sqlite differ diff --git a/lib/wekafiles/packages/packages/installedPackageCache.ser b/lib/wekafiles/packages/packages/installedPackageCache.ser deleted file mode 100644 index 9954e28f..00000000 Binary files a/lib/wekafiles/packages/packages/installedPackageCache.ser and /dev/null differ diff --git a/migrate.py b/migrate.py new file mode 100644 index 00000000..dcd7dc3e --- /dev/null +++ b/migrate.py @@ -0,0 +1,214 @@ +# Migrate last database mantaining the users and experiments +import sqlite3 +from shutil import copyfile +from datetime import datetime +import os + +current_db_path = './ubumlaas/data.sqlite' +last_db_path = './data_base.sqlite' + +current_db = sqlite3.connect(current_db_path) +last_db = sqlite3.connect(last_db_path) + + +def get_table_names(db): + cursor = db.cursor() + result = cursor.execute( + "SELECT name FROM sqlite_master \ + WHERE type='table'")\ + .fetchall() + cursor.close() + return set(map(lambda x: x[0], result)) + + +def get_index_of_primary_key(db, table): + info = get_table_info(db, table) + indexes = [] + pk_names = [] + loop_index = 0 + for _, name, _, _, _, pk in info: + if pk == 1: + indexes.append(loop_index) + pk_names.append(name) + loop_index += 1 + + return pk_names, indexes + + +def remove_tables(db, tables): + cursor = db.cursor() + for t in tables: + cursor.execute("DROP TABLE "+t) + cursor.close() + + +def get_table_info(db, table): + return db.cursor().execute("PRAGMA table_info('" + table + "')").fetchall() + + +def add_tables(db, tables): + cursor = db.cursor() + for t in tables: + sql_command = "CREATE TABLE " + t + " ( " + loop_index = 0 + for _, name, _type, _null, _default, pk in tables[t]: + appendix = "" + if pk == 1: + appendix += "PRIMARY KEY" + elif _null == 1: + appendix += "NOT NULL" + if _default is not None: + appendix += " DEFAULT "+_default + sql_command += name + " " + _type + " " + appendix + + loop_index += 1 + if(loop_index < len(tables[t])): + sql_command += "," + sql_command += ");" + cursor.execute(sql_command) + cursor.close() + + +def get_column_information(table, column): + table_info = get_table_info(last_db, table) + foreign_key_info = last_db.cursor().execute("PRAGMA foreign_key_list('" + table + "')").fetchall() + _, _, _type, _null, _default, _ = next(filter(lambda x: x[1] == column, table_info)) + try: + foreign_key_data = next(filter(lambda x: x[3] == column, foreign_key_info)) + except StopIteration: + foreign_key_data = None + column_information = " " + _type + if _null != 0: + column_information += " NOT NULL" + if _default is not None: + column_information += " DEFAULT "+_default + if foreign_key_data is not None: + column_information += " REFERENCES `"+foreign_key_data[2]+"`(`"+foreign_key_data[4]+"`)" + return column_information + + +def add_columns(db, table, columns): + cursor = db.cursor() + for c in columns: + column_information = get_column_information(table, c) + cursor.execute("ALTER TABLE "+table+" ADD COLUMN "+c+" "+column_information) + cursor.close() + + +def drop_columns(db, table, columns): + cursor = db.cursor() + for c in columns: + cursor.execute("ALTER TABLE "+table+" DROP "+c) + cursor.close() + + +def change_columns_type(db, table, columns): + cursor = db.cursor() + for c in columns: + cursor.execute("ALTER TABLE "+table+" ALTER COLUMN "+c+" "+columns[c]) + cursor.close() + + +def update_all_columns(): + # In this call last_db and current_db must have the same tables + last_tables = get_table_names(last_db) + for table in last_tables: + columns_last = {k: v for k, v in map(lambda x: (x[1], x[2]), get_table_info(last_db, table))} + columns_current = {k: v for k, v in map(lambda x: (x[1], x[2]), get_table_info(current_db, table))} + columns_to_add = {} + # columns_to_remove = [] + # columns_to_modify = {} + for name in columns_last: + current_column_type = columns_current.get(name, None) + if current_column_type is None: + columns_to_add[name] = columns_last[name] + """elif current_column_type != columns_last[name]: + columns_to_modify[name] = columns_last[name]""" + """for name in columns_current: + last_column_type = columns_last.get(name, None) + if last_column_type is None: + columns_to_remove.append(name)""" + add_columns(current_db, table, columns_to_add) + # change_columns_type(current_db, table, columns_to_modify) + # drop_columns(current_db, table, columns_to_remove) + + +def get_line(db, table, condition): + cursor = db.cursor() + result = cursor.execute("SELECT * FROM "+ table +" WHERE "+condition).fetchone() + cursor.close() + return result + + +def insert(db, table, values): + cursor = db.cursor() + query = "INSERT INTO " + table + " VALUES(" + query += "?" + for v in values[1:]: + query += ",?" + query += ")" + cursor.execute(query, values) + cursor.close() + + +def update(db, table, keys, values, where): + cursor = db.cursor() + query = "UPDATE " + table + " SET " + query += str(keys[0]) + " = ?" + for k, v in zip(keys[1:], values[1:]): + query += "," + str(k) + " = ?" + query += " WHERE "+where + cursor.execute(query, values) + cursor.close() + + +def insert_and_update_information(): + last_tables = get_table_names(last_db) + cursor_last = last_db.cursor() + cursor_current = current_db.cursor() + for table in last_tables: + columns_last = [k for k in map(lambda x: x[1], get_table_info(last_db, table))] + content_last = cursor_last.execute("SELECT * FROM "+table).fetchall() + pk_names, indexes = get_index_of_primary_key(last_db, table) + for line in content_last: + key = [] + for i in indexes: + key.append(line[i]) + condition = str(pk_names[0])+" = '"+str(key[0])+"'" + for j in range(0, len(pk_names[1:])): + condition += " AND " + str(pk_names[j+1])+" = '"+str(key[j+1])+"'" + row = get_line(current_db, table, condition) + if row is None: + insert(current_db, table, line) + elif tuple(row) != tuple(line): + update(current_db, table, columns_last, line, condition) + + cursor_current.close() + cursor_last.close() + + +def backup(): + if not os.path.exists("./backups/"): + os.makedirs("./backups/") + copyfile(current_db_path, "./backups/db-"+datetime.now().strftime("%d-%m-%Y--%H-%M-%S")+".sqlite") + + +if __name__ == "__main__": + backup() + current_tables = get_table_names(current_db) + last_tables = get_table_names(last_db) + tables_to_remove = current_tables.difference(last_tables) + tables_to_add = last_tables.difference(current_tables) + + remove_tables(current_db, tables_to_remove) + tables_information = dict() + for t in tables_to_add: + tables_information[t] = get_table_info(last_db, t) + add_tables(current_db, tables_information) + + update_all_columns() + insert_and_update_information() + +current_db.commit() +current_db.close() +last_db.close() \ No newline at end of file diff --git a/ubumlaas/experiments/views/utils.py b/ubumlaas/experiments/views/utils.py index acdf9a52..2298aae1 100644 --- a/ubumlaas/experiments/views/utils.py +++ b/ubumlaas/experiments/views/utils.py @@ -41,7 +41,8 @@ def download_model(id): @views.experiments.route("/experiment/base_estimator_getter", methods=["POST"]) def base_estimator_getter(): alg_name = request.form.get("alg_name", None) - algorithm = get_similar_algorithms(alg_name) + exp_typ = request.form.get("exp_typ", None) + algorithm = get_similar_algorithms(alg_name, exp_typ) _ret = dict() _ret["algorithms"] = [] diff --git a/ubumlaas/models.py b/ubumlaas/models.py index 93c83d31..3fb0766a 100644 --- a/ubumlaas/models.py +++ b/ubumlaas/models.py @@ -1,7 +1,7 @@ import variables as v from werkzeug.security import generate_password_hash, check_password_hash from datetime import datetime -from sqlalchemy import and_, or_, text +from sqlalchemy import and_, or_, text, desc, asc from flask_login import UserMixin @@ -53,27 +53,51 @@ def get_experiments(idu): def get_algorithms_type(): sql = text('SELECT DISTINCT alg_typ FROM algorithms') result = v.db.engine.execute(sql) - types = [(row[0], row[0]) for row in result] + types = [(row[0], row[0]) for row in result if row[0] != "Mixed"] return types -def get_similar_algorithms(alg_name): +# def get_similar_algorithms(alg_name): +# """Get algorithm that can be base estimator + +# Arguments: +# alg_name {str} -- Name of the algorithm + +# Returns: +# list of lists -- all similar algorithms +# """ +# alg = get_algorithm_by_name(alg_name) +# if alg.lib != "meka": +# cond = Algorithm.lib == alg.lib +# else: +# cond = Algorithm.lib == "weka" +# if alg.alg_typ == "MultiClassification": +# cond = and_(cond, or_(Algorithm.alg_typ == "Classification", +# Algorithm.alg_typ == "Mixed")) +# else: +# cond = and_(cond, or_(Algorithm.alg_typ == "Regression", +# Algorithm.alg_typ == "Mixed")) +# if alg.alg_typ != "Mixed" and alg.lib != "meka": +# cond = and_(cond, Algorithm.alg_typ == alg.alg_typ) +# elif alg.lib != "meka": +# cond = and_(cond, or_(Algorithm.alg_typ == "Classification", +# Algorithm.alg_typ == "Regression")) +# algorithms = Algorithm.query.filter(cond).all() +# return algorithms + + +def get_similar_algorithms(alg_name, exp_typ): alg = get_algorithm_by_name(alg_name) - if alg.lib != "meka": - cond = Algorithm.lib == alg.lib + if alg.lib == "meka": + cond = and_(Algorithm.lib == "weka", + or_(Algorithm.alg_typ == "Classification", + Algorithm.alg_typ == "Mixed")) else: - cond = Algorithm.lib == "weka" - if alg.alg_typ == "MultiClassification": - cond = and_(cond, or_(Algorithm.alg_typ == "Classification", - Algorithm.alg_typ == "Mixed")) - else: - cond = and_(cond, or_(Algorithm.alg_typ == "Regression", - Algorithm.alg_typ == "Mixed")) - if alg.alg_typ != "Mixed" and alg.lib != "meka": - cond = and_(cond, Algorithm.alg_typ == alg.alg_typ) - elif alg.lib != "meka": - cond = and_(cond, or_(Algorithm.alg_typ == "Classification", - Algorithm.alg_typ == "Regression")) + cond = Algorithm.lib == alg.lib + subcond = Algorithm.alg_typ == exp_typ + if exp_typ in ["Classification", "Regression"]: + subcond = or_(subcond, Algorithm.alg_typ == "Mixed") + cond = and_(cond, subcond) algorithms = Algorithm.query.filter(cond).all() return algorithms @@ -87,7 +111,10 @@ def get_algorithms(alg_typ): Returns: algorithm list -- all algorithm of that type. """ - return Algorithm.query.filter(Algorithm.alg_typ == alg_typ).all() + cond = Algorithm.alg_typ == alg_typ + if alg_typ in ["Classification", "Regression"]: + cond = or_(cond, Algorithm.alg_typ == "Mixed") + return Algorithm.query.filter(cond).order_by(asc(Algorithm.web_name)).all() def get_algorithm_by_name(name): diff --git a/ubumlaas/static/js/config_alg.js b/ubumlaas/static/js/config_alg.js index 0bdf284a..124bc68c 100644 --- a/ubumlaas/static/js/config_alg.js +++ b/ubumlaas/static/js/config_alg.js @@ -95,7 +95,7 @@ function generateForm(alg_config, place_in_id, level_to_modify=0, filter=false){ function create_ensemble_block(basename, parameter){ let content; content = $("", {id: basename+"_value"}); - let petition = "alg_name="+$("#alg_name").val(); + let petition = "alg_name="+$("#alg_name").val()+"&exp_typ="+alg_typ.val(); let _options = give_me_base_estimators(petition); _options.forEach(function (e) { content.append($(""));