diff --git a/.gitignore b/.gitignore
index d1b90137..5dd24e6c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -134,4 +134,6 @@ env_variables.sh
lib/**/*.html
lib/**/*.css
-debug.log
\ No newline at end of file
+debug.log
+
+backups/
\ No newline at end of file
diff --git a/data_base.sqlite b/data_base.sqlite
index 64dbecef..b1b9a4a8 100644
Binary files a/data_base.sqlite and b/data_base.sqlite differ
diff --git a/lib/wekafiles/packages/packages/installedPackageCache.ser b/lib/wekafiles/packages/packages/installedPackageCache.ser
deleted file mode 100644
index 9954e28f..00000000
Binary files a/lib/wekafiles/packages/packages/installedPackageCache.ser and /dev/null differ
diff --git a/migrate.py b/migrate.py
new file mode 100644
index 00000000..dcd7dc3e
--- /dev/null
+++ b/migrate.py
@@ -0,0 +1,214 @@
+# Migrate last database mantaining the users and experiments
+import sqlite3
+from shutil import copyfile
+from datetime import datetime
+import os
+
+current_db_path = './ubumlaas/data.sqlite'
+last_db_path = './data_base.sqlite'
+
+current_db = sqlite3.connect(current_db_path)
+last_db = sqlite3.connect(last_db_path)
+
+
+def get_table_names(db):
+ cursor = db.cursor()
+ result = cursor.execute(
+ "SELECT name FROM sqlite_master \
+ WHERE type='table'")\
+ .fetchall()
+ cursor.close()
+ return set(map(lambda x: x[0], result))
+
+
+def get_index_of_primary_key(db, table):
+ info = get_table_info(db, table)
+ indexes = []
+ pk_names = []
+ loop_index = 0
+ for _, name, _, _, _, pk in info:
+ if pk == 1:
+ indexes.append(loop_index)
+ pk_names.append(name)
+ loop_index += 1
+
+ return pk_names, indexes
+
+
+def remove_tables(db, tables):
+ cursor = db.cursor()
+ for t in tables:
+ cursor.execute("DROP TABLE "+t)
+ cursor.close()
+
+
+def get_table_info(db, table):
+ return db.cursor().execute("PRAGMA table_info('" + table + "')").fetchall()
+
+
+def add_tables(db, tables):
+ cursor = db.cursor()
+ for t in tables:
+ sql_command = "CREATE TABLE " + t + " ( "
+ loop_index = 0
+ for _, name, _type, _null, _default, pk in tables[t]:
+ appendix = ""
+ if pk == 1:
+ appendix += "PRIMARY KEY"
+ elif _null == 1:
+ appendix += "NOT NULL"
+ if _default is not None:
+ appendix += " DEFAULT "+_default
+ sql_command += name + " " + _type + " " + appendix
+
+ loop_index += 1
+ if(loop_index < len(tables[t])):
+ sql_command += ","
+ sql_command += ");"
+ cursor.execute(sql_command)
+ cursor.close()
+
+
+def get_column_information(table, column):
+ table_info = get_table_info(last_db, table)
+ foreign_key_info = last_db.cursor().execute("PRAGMA foreign_key_list('" + table + "')").fetchall()
+ _, _, _type, _null, _default, _ = next(filter(lambda x: x[1] == column, table_info))
+ try:
+ foreign_key_data = next(filter(lambda x: x[3] == column, foreign_key_info))
+ except StopIteration:
+ foreign_key_data = None
+ column_information = " " + _type
+ if _null != 0:
+ column_information += " NOT NULL"
+ if _default is not None:
+ column_information += " DEFAULT "+_default
+ if foreign_key_data is not None:
+ column_information += " REFERENCES `"+foreign_key_data[2]+"`(`"+foreign_key_data[4]+"`)"
+ return column_information
+
+
+def add_columns(db, table, columns):
+ cursor = db.cursor()
+ for c in columns:
+ column_information = get_column_information(table, c)
+ cursor.execute("ALTER TABLE "+table+" ADD COLUMN "+c+" "+column_information)
+ cursor.close()
+
+
+def drop_columns(db, table, columns):
+ cursor = db.cursor()
+ for c in columns:
+ cursor.execute("ALTER TABLE "+table+" DROP "+c)
+ cursor.close()
+
+
+def change_columns_type(db, table, columns):
+ cursor = db.cursor()
+ for c in columns:
+ cursor.execute("ALTER TABLE "+table+" ALTER COLUMN "+c+" "+columns[c])
+ cursor.close()
+
+
+def update_all_columns():
+ # In this call last_db and current_db must have the same tables
+ last_tables = get_table_names(last_db)
+ for table in last_tables:
+ columns_last = {k: v for k, v in map(lambda x: (x[1], x[2]), get_table_info(last_db, table))}
+ columns_current = {k: v for k, v in map(lambda x: (x[1], x[2]), get_table_info(current_db, table))}
+ columns_to_add = {}
+ # columns_to_remove = []
+ # columns_to_modify = {}
+ for name in columns_last:
+ current_column_type = columns_current.get(name, None)
+ if current_column_type is None:
+ columns_to_add[name] = columns_last[name]
+ """elif current_column_type != columns_last[name]:
+ columns_to_modify[name] = columns_last[name]"""
+ """for name in columns_current:
+ last_column_type = columns_last.get(name, None)
+ if last_column_type is None:
+ columns_to_remove.append(name)"""
+ add_columns(current_db, table, columns_to_add)
+ # change_columns_type(current_db, table, columns_to_modify)
+ # drop_columns(current_db, table, columns_to_remove)
+
+
+def get_line(db, table, condition):
+ cursor = db.cursor()
+ result = cursor.execute("SELECT * FROM "+ table +" WHERE "+condition).fetchone()
+ cursor.close()
+ return result
+
+
+def insert(db, table, values):
+ cursor = db.cursor()
+ query = "INSERT INTO " + table + " VALUES("
+ query += "?"
+ for v in values[1:]:
+ query += ",?"
+ query += ")"
+ cursor.execute(query, values)
+ cursor.close()
+
+
+def update(db, table, keys, values, where):
+ cursor = db.cursor()
+ query = "UPDATE " + table + " SET "
+ query += str(keys[0]) + " = ?"
+ for k, v in zip(keys[1:], values[1:]):
+ query += "," + str(k) + " = ?"
+ query += " WHERE "+where
+ cursor.execute(query, values)
+ cursor.close()
+
+
+def insert_and_update_information():
+ last_tables = get_table_names(last_db)
+ cursor_last = last_db.cursor()
+ cursor_current = current_db.cursor()
+ for table in last_tables:
+ columns_last = [k for k in map(lambda x: x[1], get_table_info(last_db, table))]
+ content_last = cursor_last.execute("SELECT * FROM "+table).fetchall()
+ pk_names, indexes = get_index_of_primary_key(last_db, table)
+ for line in content_last:
+ key = []
+ for i in indexes:
+ key.append(line[i])
+ condition = str(pk_names[0])+" = '"+str(key[0])+"'"
+ for j in range(0, len(pk_names[1:])):
+ condition += " AND " + str(pk_names[j+1])+" = '"+str(key[j+1])+"'"
+ row = get_line(current_db, table, condition)
+ if row is None:
+ insert(current_db, table, line)
+ elif tuple(row) != tuple(line):
+ update(current_db, table, columns_last, line, condition)
+
+ cursor_current.close()
+ cursor_last.close()
+
+
+def backup():
+ if not os.path.exists("./backups/"):
+ os.makedirs("./backups/")
+ copyfile(current_db_path, "./backups/db-"+datetime.now().strftime("%d-%m-%Y--%H-%M-%S")+".sqlite")
+
+
+if __name__ == "__main__":
+ backup()
+ current_tables = get_table_names(current_db)
+ last_tables = get_table_names(last_db)
+ tables_to_remove = current_tables.difference(last_tables)
+ tables_to_add = last_tables.difference(current_tables)
+
+ remove_tables(current_db, tables_to_remove)
+ tables_information = dict()
+ for t in tables_to_add:
+ tables_information[t] = get_table_info(last_db, t)
+ add_tables(current_db, tables_information)
+
+ update_all_columns()
+ insert_and_update_information()
+
+current_db.commit()
+current_db.close()
+last_db.close()
\ No newline at end of file
diff --git a/ubumlaas/experiments/views/utils.py b/ubumlaas/experiments/views/utils.py
index acdf9a52..2298aae1 100644
--- a/ubumlaas/experiments/views/utils.py
+++ b/ubumlaas/experiments/views/utils.py
@@ -41,7 +41,8 @@ def download_model(id):
@views.experiments.route("/experiment/base_estimator_getter", methods=["POST"])
def base_estimator_getter():
alg_name = request.form.get("alg_name", None)
- algorithm = get_similar_algorithms(alg_name)
+ exp_typ = request.form.get("exp_typ", None)
+ algorithm = get_similar_algorithms(alg_name, exp_typ)
_ret = dict()
_ret["algorithms"] = []
diff --git a/ubumlaas/models.py b/ubumlaas/models.py
index 93c83d31..3fb0766a 100644
--- a/ubumlaas/models.py
+++ b/ubumlaas/models.py
@@ -1,7 +1,7 @@
import variables as v
from werkzeug.security import generate_password_hash, check_password_hash
from datetime import datetime
-from sqlalchemy import and_, or_, text
+from sqlalchemy import and_, or_, text, desc, asc
from flask_login import UserMixin
@@ -53,27 +53,51 @@ def get_experiments(idu):
def get_algorithms_type():
sql = text('SELECT DISTINCT alg_typ FROM algorithms')
result = v.db.engine.execute(sql)
- types = [(row[0], row[0]) for row in result]
+ types = [(row[0], row[0]) for row in result if row[0] != "Mixed"]
return types
-def get_similar_algorithms(alg_name):
+# def get_similar_algorithms(alg_name):
+# """Get algorithm that can be base estimator
+
+# Arguments:
+# alg_name {str} -- Name of the algorithm
+
+# Returns:
+# list of lists -- all similar algorithms
+# """
+# alg = get_algorithm_by_name(alg_name)
+# if alg.lib != "meka":
+# cond = Algorithm.lib == alg.lib
+# else:
+# cond = Algorithm.lib == "weka"
+# if alg.alg_typ == "MultiClassification":
+# cond = and_(cond, or_(Algorithm.alg_typ == "Classification",
+# Algorithm.alg_typ == "Mixed"))
+# else:
+# cond = and_(cond, or_(Algorithm.alg_typ == "Regression",
+# Algorithm.alg_typ == "Mixed"))
+# if alg.alg_typ != "Mixed" and alg.lib != "meka":
+# cond = and_(cond, Algorithm.alg_typ == alg.alg_typ)
+# elif alg.lib != "meka":
+# cond = and_(cond, or_(Algorithm.alg_typ == "Classification",
+# Algorithm.alg_typ == "Regression"))
+# algorithms = Algorithm.query.filter(cond).all()
+# return algorithms
+
+
+def get_similar_algorithms(alg_name, exp_typ):
alg = get_algorithm_by_name(alg_name)
- if alg.lib != "meka":
- cond = Algorithm.lib == alg.lib
+ if alg.lib == "meka":
+ cond = and_(Algorithm.lib == "weka",
+ or_(Algorithm.alg_typ == "Classification",
+ Algorithm.alg_typ == "Mixed"))
else:
- cond = Algorithm.lib == "weka"
- if alg.alg_typ == "MultiClassification":
- cond = and_(cond, or_(Algorithm.alg_typ == "Classification",
- Algorithm.alg_typ == "Mixed"))
- else:
- cond = and_(cond, or_(Algorithm.alg_typ == "Regression",
- Algorithm.alg_typ == "Mixed"))
- if alg.alg_typ != "Mixed" and alg.lib != "meka":
- cond = and_(cond, Algorithm.alg_typ == alg.alg_typ)
- elif alg.lib != "meka":
- cond = and_(cond, or_(Algorithm.alg_typ == "Classification",
- Algorithm.alg_typ == "Regression"))
+ cond = Algorithm.lib == alg.lib
+ subcond = Algorithm.alg_typ == exp_typ
+ if exp_typ in ["Classification", "Regression"]:
+ subcond = or_(subcond, Algorithm.alg_typ == "Mixed")
+ cond = and_(cond, subcond)
algorithms = Algorithm.query.filter(cond).all()
return algorithms
@@ -87,7 +111,10 @@ def get_algorithms(alg_typ):
Returns:
algorithm list -- all algorithm of that type.
"""
- return Algorithm.query.filter(Algorithm.alg_typ == alg_typ).all()
+ cond = Algorithm.alg_typ == alg_typ
+ if alg_typ in ["Classification", "Regression"]:
+ cond = or_(cond, Algorithm.alg_typ == "Mixed")
+ return Algorithm.query.filter(cond).order_by(asc(Algorithm.web_name)).all()
def get_algorithm_by_name(name):
diff --git a/ubumlaas/static/js/config_alg.js b/ubumlaas/static/js/config_alg.js
index 0bdf284a..124bc68c 100644
--- a/ubumlaas/static/js/config_alg.js
+++ b/ubumlaas/static/js/config_alg.js
@@ -95,7 +95,7 @@ function generateForm(alg_config, place_in_id, level_to_modify=0, filter=false){
function create_ensemble_block(basename, parameter){
let content;
content = $("", {id: basename+"_value"});
- let petition = "alg_name="+$("#alg_name").val();
+ let petition = "alg_name="+$("#alg_name").val()+"&exp_typ="+alg_typ.val();
let _options = give_me_base_estimators(petition);
_options.forEach(function (e) {
content.append($(""));