Add verbosity and classifier_by_lines

.gitignore

@@ -108,3 +108,6 @@ ENV/
 
 # MacOS files
 *.DS_Store
+
+# Models
+Classification/

README.md

@@ -22,6 +22,7 @@ To use this tool:
 1) *python3 \<path-of-clustering/learner.py\> --help*;  
 2) *python3 \<path-of-clustering/updater.py\> --help*;  
 3) *python3 \<path-of-clustering/classifier.py\> --help*.
+4) *python3 \<path-of-clustering/classifier_by_lines.py\> --help*.
 
 - Clustering of JavaScript samples into *k* (configurable) families.   
 To use this tool: *python3 \<path-of-clustering/cluster.py\> --help*.

clustering/classifier.py

@@ -61,6 +61,7 @@ def test_model(names, labels, attributes, model, print_res=True, print_res_verbo
     # to predict the target values
 
     if print_res:
+        utility.get_proba('malicious', labels_predicted_proba_test)
         utility.get_classification_results(names, labels_predicted_test)
 
     if print_res_verbose:
@@ -71,7 +72,7 @@ def test_model(names, labels, attributes, model, print_res=True, print_res_verbo
     if print_score:
         utility.get_score(labels, labels_predicted_test)
 
-    return labels_predicted_test
+    return labels_predicted_test, labels_predicted_proba_test
 
 
 def parsing_commands():

clustering/classifier_by_lines.py

@@ -0,0 +1,101 @@
+#!/usr/bin/python
+
+# Standard
+import argparse
+import sys
+
+# JaSt lib
+import utility
+import classifier
+import static_analysis
+from is_js import is_js_file
+
+# Debug
+# from pdb import set_trace as st
+
+def parsing_commands():
+    """
+        Creation of an ArgumentParser object, holding all the information necessary to parse
+        the command line into Python data types.
+
+        -------
+        Returns:
+        - ArgumentParser such as:
+          * js_dirs=arg_obj['d'],
+          * labels_d=arg_obj['l'],
+          * js_files=arg_obj['f'],
+          * labels_f=arg_obj['lf'],
+          * model=arg_obj['m'],
+          * threshold=arg_obj['th'],
+          * tolerance=arg_obj['t'][0],
+          * n=arg_obj['n'][0].
+          A more thorough description can be obtained:
+            >$ python3 <path-of-clustering/classifier.py> -help
+    """
+
+    parser = argparse.ArgumentParser(description='Given a list of directory or file paths,\
+    detects the malicious JS inputs.')
+
+    parser.add_argument('--d', metavar='DIR', type=str, nargs='+',
+                        help='directories containing the JS files to be analyzed')
+    parser.add_argument('--l', metavar='LABEL', type=str, nargs='+',
+                        choices=['benign', 'malicious', '?'],
+                        help='labels of the JS directories to evaluate the model from')
+    parser.add_argument('--f', metavar='FILE', type=str, nargs='+', help='files to be analyzed')
+    parser.add_argument('--lf', metavar='LABEL', type=str, nargs='+',
+                        choices=['benign', 'malicious', '?'],
+                        help='labels of the JS files to evaluate the model from')
+    parser.add_argument('--m', metavar='MODEL', type=str, nargs=1,
+                        help='path of the model used to classify the new JS inputs '
+                             + '(see >$ python3 <path-of-clustering/learner.py> -help) '
+                             + 'to build a model)')
+    parser.add_argument('--th', metavar='THRESHOLD', type=float, nargs=1, default=[0.29],
+                        help='threshold over which all samples are considered malicious')
+    utility.parsing_commands(parser)
+
+    return vars(parser.parse_args())
+
+OPTS = parsing_commands()
+JAVASCRIPT = OPTS['f'][0]
+MODEL = OPTS['m'][0]
+TMPFILENAME = '/tmp/.tmp.js'
+
+def get_malicious_score(js):
+    """
+        Classification of the sub-javascript
+    """
+    names, attributes, labels = static_analysis.main_analysis \
+    (js_files=[js], js_dirs=OPTS['d'], labels_files=OPTS['lf'], labels_dirs=OPTS['l'], \
+        n=OPTS['n'][0], tolerance=OPTS['t'][0], dict_not_hash=OPTS['dnh'][0])
+    if not names:
+        return 0
+    _, labels_predicted_proba = classifier.test_model(names, labels, attributes, \
+        model=MODEL, print_res=False, print_score=False)
+    malicious_proba = int(labels_predicted_proba[0][1]*100)
+    return malicious_proba
+
+def main():
+    """
+        Main function
+    """
+    js = open(JAVASCRIPT, 'r')
+    copy = open(TMPFILENAME, 'w+')
+    line = js.readline()
+    n = 1
+    begin_line = n
+    while line:
+        copy.write(str(line))
+        copy.close()
+        if is_js_file(TMPFILENAME) == 0:
+            score = get_malicious_score(TMPFILENAME)
+            print('Line {} to {}: {}%'.format(begin_line, n, score))
+            begin_line = n + 1
+            copy = open(TMPFILENAME, 'w+')
+        else:
+            copy = open(TMPFILENAME, 'a+')
+        line = js.readline()
+        n += 1
+    js.close()
+
+if __name__ == '__main__':
+    main()

clustering/utility.py

@@ -143,6 +143,23 @@ def get_score(labels, labels_predicted):
             logging.exception(error_message)
 
 
+def get_proba(label, labels_predicted_proba):
+    """
+        Print in stdout the probability of the classification.
+
+        -------
+        Parameters:
+        - label: string
+            Contains the proba's name.
+        - labels_predicted_proba: matrix
+            Contains in the first column the probability of the samples being benign,
+            and malicious in the second one.
+
+    """
+    label_id = ['benign', 'malicious'].index(label)
+    print("{}: {} %".format(label, int(labels_predicted_proba[0][label_id]*100)))
+
+
 def get_nb_trees_specific_label(model, attributes, labels, labels_predicted, threshold):
     """
         Get the number of trees which gave the same prediction as the one of the whole forest.