Merge pull request #5918 from BOINC/dpa_buda2

Web and server: Add web-based job submission for BUDA

html/inc/db_conn.inc

@@ -116,8 +116,8 @@ class DbConn {
 
     function enum_general($classname, $query) {
         $result = $this->do_query($query);
-        if (!$result) return null;
-        $x = array();
+        if (!$result) return [];
+        $x = [];
         while ($obj = $result->fetch_object($classname)) {
             $x[] = $obj;
         }
@@ -133,7 +133,7 @@ class DbConn {
             $where_clause = "where $where_clause";
         }
         $query = "select $fields from DBNAME.$table $where_clause $order_clause";
-        return $this->enum_general($classname,$query);
+        return $this->enum_general($classname, $query);
     }
 
     function enum($table, $classname, $where_clause=null, $order_clause=null) {

html/inc/dir_hier.inc

@@ -120,22 +120,45 @@ function check_download_file($path, $dl_path) {
     }
 }
 
-// Stage the given file, which is assumed to not be in download dir already.
-//
-function stage_file_basic($dir, $fname) {
+function get_hier_info() {
     static $dl_dir = null;
     static $fanout = null;
     if (!$dl_dir) {
         $conf = get_config();
         $dl_dir = parse_config($conf, "<download_dir>");
         $fanout = parse_config($conf, "<uldl_dir_fanout>");
     }
+    return [$dl_dir, $fanout];
+}
+
+function download_hier_path($fname) {
+    [$dl_dir, $fanout] = get_hier_info();
+    return dir_hier_path($fname, $dl_dir, $fanout);
+}
+
+// Stage (move, not copy) the given file,
+// which is assumed to not be in download dir already.
+//
+function stage_file_basic($dir, $fname) {
+    [$dl_dir, $fanout] = get_hier_info();
     $old_path = "$dir/$fname";
-    $new_path = dir_hier_path($fname, $dl_dir, $fanout);
+    $new_path = download_hier_path($fname);
     $md5 = md5_file($old_path);
     $size = filesize($old_path);
     file_put_contents("$new_path.md5", "$md5 $size\n");
     rename($old_path, $new_path);
 }
 
+// copy the given file (with given md5/size)
+// to the download dir w/ given phys name; and create .md5 file
+// If phys name is already there, do nothing.
+//
+function stage_file_aux($path, $md5, $size, $phys_name) {
+    [$dl_dir, $fanout] = get_hier_info();
+    $phys_path = dir_hier_path($phys_name, $dl_dir, $fanout);
+    if (file_exists($phys_path)) return;
+    copy($path, $phys_path);
+    file_put_contents("$phys_path.md5", "$md5 $size\n");
+}
+
 ?>

html/inc/result.inc

@@ -723,7 +723,7 @@ function show_result($result, $show_outfile_links=false) {
     }
     if ($show_outfile_links && $result->outcome == 1) {
         $fanout = parse_config(get_config(), "<uldl_dir_fanout>");
-        $names = get_outfile_names($result);
+        $names = get_outfile_phys_names($result);
         $i = 0;
         $x = "";
         foreach ($names as $name) {

html/inc/sandbox.inc

@@ -19,14 +19,23 @@
 // Utility functions for user file sandbox feature
 //
 // In this system:
-// - sandbox files live in the download hierarchy,
-//   with names of the form sb_userid_md5
-// - each user has a "sandbox dir" project/sandbox/userid.
-//   The files in this have user-visible names, and contents of the form
-//   sb file_size file_md5
+// - each user (job submitter) has a 'sandbox' where they can store files
+//      on the BOINC server, via a web interface.
+//      These files are mutable; you can modify a file w/ a given name.
+// - files are stored in a dir project/sandbox/<userid>
+// - When a file is uploaded, its size and MD5 are computed and stored
+//      in an 'info file' in a parallel dir, project/sandbox/<userid>/.md5
+//
+// Sandbox files can be used for web-based job submissions systems
+// like BUDA and autodock on BOINC Central.
+// Typically they are used as job input files or app files,
+// in which case they are downloadable.
+// When a file is used in this way,
+// it must be copied to the download hierarchy,
+// and assigned a physical name that includes its MD5.
+// The name depends on the role of the file.
 
 require_once("../inc/util.inc");
-require_once("../inc/submit_db.inc");
 require_once("../inc/dir_hier.inc");
 
 // Return path of sandbox directory for the given user.
@@ -44,75 +53,53 @@ function sandbox_dir($user) {
     if (!is_dir($d)) {
         mkdir($d);
     }
+    if (!is_dir("$d/.md5")) {
+        mkdir("$d/.md5");
+    }
     return $d;
 }
 
-function sandbox_write_link_file($path, $size, $md5) {
-    file_put_contents($path, "sb $size $md5");
-}
-
-// check if a link with the given md5 exists
+// parse a sandbox file's info file.
+// If missing, create it.
 //
-function sandbox_lf_exists($user, $md5) {
-    $exist = false;
-    $elf = "";
+function sandbox_parse_info_file($user, $name) {
     $dir = sandbox_dir($user);
-    $files = sandbox_file_names($user);
-    foreach ($files as $file) {
-        $path = "$dir/$file";
-        [$err, $file_size, $file_md5] = sandbox_parse_link_file($path);
-        if (!$err){
-            if (strcmp($md5, $file_md5) == 0) {
-                $exist = true;
-                $elf = $file;
-                break;
-            }
-        }
+    $info_path = "$dir/.md5/$name";
+    $info = parse_info_file($info_path);
+    if ($info) {
+        return $info;
     }
-    return array($exist, $elf);
-}
-
-// parse a link file and return (error, size, md5)
-//
-function sandbox_parse_link_file($path) {
-    if (!file_exists($path)) { return array(true, null, null); }
-    $x = file_get_contents($path);
-    $n = sscanf($x, "%s %d %s", $s, $size, $md5);
-    if ($n != 3) return array(true, null, null);
-    if ($s != 'sb') return array(true, null, null);
-    return array(false, $size, $md5);
-}
-
-$fanout = parse_config(get_config(), "<uldl_dir_fanout>");
-
-// return the physical name of the file
-//
-function sandbox_file_name($user, $md5) {
-    return "sb_".$user->id."_".$md5;
-}
-
-// return the path of the file in the download directory
-//
-function sandbox_physical_path($user, $md5) {
-    global $fanout;
-    $f = sandbox_file_name($user, $md5);
-    return dir_hier_path($f, parse_config(get_config(), "<download_dir>"), $fanout);
+    [$size, $md5] = get_file_info("$dir/$name");
+    write_info_file($info_path, $md5, $size);
+    return [$md5, $size];
 }
 
 // return list of files in sandbox
 //
 function sandbox_file_names($user) {
-    $d = opendir(sandbox_dir($user));
+    $files = scandir(sandbox_dir($user));
     $names = array();
-    while (($f = readdir($d)) !== false) {
-        if ($f == '.') continue;
-        if ($f == '..') continue;
+    foreach ($files as $f) {
+        if ($f[0] == '.') continue;
         $names[] = $f;
     }
     natsort($names);
     return $names;
 }
 
+// return list of files matching given pattern,
+// in the format used for form_select() and form_select_multiple()
+//
+function sandbox_select_items($user, $pattern=null) {
+    $sbfiles = sandbox_file_names($user);
+    $sbitems = [];
+    foreach ($sbfiles as $f) {
+        if ($pattern && !preg_match($pattern, $f)) continue;
+        $sbitems[] = [$f, $f];
+    }
+    return $sbitems;
+}
+
 // return a <select> for files in sandbox
 //
 function sandbox_file_select(
@@ -131,66 +118,13 @@ function sandbox_file_select(
     return $x;
 }
 
-// convert sandbox (link) name to physical path
+// copy file and info file from sandbox to $dir
+// (which must have a subdir .md5/)
 //
-function sandbox_log_to_phys($user, $log_name) {
-    $dir = sandbox_dir($user);
-    [$err, $file_size, $file_md5] = sandbox_parse_link_file("$dir/$log_name");
-    if ($err) return null;
-    return sandbox_physical_path($user, $file_md5);
-}
-
-// convert sandbox name to physical name
-//
-function sandbox_name_to_phys_name($user, $log_name) {
-    $dir = sandbox_dir($user);
-    [$err, $file_size, $file_md5] = sandbox_parse_link_file("$dir/$log_name");
-    if ($err) return null;
-    return sandbox_file_name($user, $file_md5);
-}
-
-// check if a file is still being used by a unfinished batch
-//
-// TODO: this is a kludge.
-// Should we use the job_file and batch_file_assoc tables instead?
-//
-function sandbox_file_in_use($user, $file){
-    $ufiles = array();
-
-    $pbatches = BoincBatch::enum(
-        sprintf('user_id=%d and state!=%d and state!=%d',
-            $user->id, BATCH_STATE_COMPLETE, BATCH_STATE_ABORTED
-        )
-    );
-    if (!$pbatches) return false;
-
-    foreach ($pbatches as $batch){
-        $wus = BoincWorkUnit::enum("batch = $batch->id limit 1" );
-        if ($wus == null){
-            continue;
-        }
-        foreach($wus as $wu){
-            $x = "<in>".$wu->xml_doc."</in>";
-            $x = simplexml_load_string($x);
-            global $fanout;
-            foreach($x->workunit->file_ref as $fr){
-                $pname = (string)$fr->file_name;
-                $ufiles[] = $pname;
-            }
-        }
-    }
-    $dir = sandbox_dir($user);
-    $path = $dir."/".$file;
-    list($err, $size, $md5) = sandbox_parse_link_file($path);
-    if (!$err){
-        $f = sandbox_file_name($user, $md5);
-        foreach($ufiles as $uf) {
-            if (strcmp($f,$uf) == 0){
-                return true;
-            }
-        }
-    }
-    return false;
+function copy_sandbox_file($user, $fname, $dir) {
+    $sbdir = sandbox_dir($user);
+    copy("$sbdir/$fname", "$dir/$fname");
+    copy("$sbdir/.md5/$fname", "$dir/.md5/$fname");
 }
 
 ?>

html/inc/submit_util.inc

@@ -164,7 +164,9 @@ function wus_nsent($wus) {
     return $n;
 }
 
-function get_outfile_names($result) {
+// get the physical names of a result's output files.
+/
+function get_outfile_phys_names($result) {
     $names = [];
     $xml = "<a>".$result->xml_doc_out."</a>";
     $r = simplexml_load_string($xml);
@@ -261,7 +263,7 @@ function batch_output_file_size($batchid) {
     foreach ($wus as $wu) {
         if (!$wu->canonical_resultid) continue;
         $result = BoincResult::lookup_id($wu->canonical_resultid);
-        $names = get_outfile_names($result);
+        $names = get_outfile_phys_names($result);
         foreach ($names as $name) {
             $path = dir_hier_path($name, $upload_dir, $fanout);
             if (is_file($path)) {
@@ -288,4 +290,40 @@ function boinc_get_wu_output_files_url($user, $wu_id) {
     return "get_output.php?cmd=workunit_files&wu_id=$wu_id&auth_str=$auth_str";
 }
 
+////////////////// FILE INFO FILES //////////////
+
+// these are used:
+// 1) in user file sandbox
+// 2) in BUDA app variant dirs
+// in each case a file dir/foo has an info file dir/.md5/foo
+// containing its md5 and size
+// (same format as .md5 files in download hierarchy)
+
+// get the MD5 and size of a file
+//
+function get_file_info($path) {
+    $md5 = md5_file($path);
+    $s = stat($path);
+    $size = $s['size'];
+    return [$md5, $size];
+}
+
+// write a "info file" containing MD5 and size
+//
+function write_info_file($path, $md5, $size) {
+    file_put_contents($path, "$md5 $size");
+}
+
+// parse info file and return [md5, size]
+//
+function parse_info_file($path) {
+    if (!file_exists($path)) return null;
+    $x = file_get_contents($path);
+    $n = sscanf($x, "%s %d", $md5, $size);
+    if ($n != 2 || strlen($md5)!=32) {
+        return null;
+    }
+    return [$md5, $size];
+}
+
 ?>

html/inc/util.inc

@@ -1133,10 +1133,8 @@ function credit_to_gflop_hours($c) {
     return $c/(200/24);
 }
 
-function do_download($path,$name="") {
-    if (strcmp($name,"") == 0) {
-        $name=basename($path);
-    }
+function do_download($path) {
+    $name=basename($path);
     header('Content-Description: File Transfer');
     header('Content-Type: application/octet-stream');
     header('Content-Disposition: attachment; filename='.$name);

html/inc/util_basic.inc

@@ -56,14 +56,6 @@ function sched_stopped() {
     return file_exists("$d/stop_sched");
 }
 
-function show_page($x, $y) {
-    echo "
-        <title>$x</title>
-        <h1>$x</h1>
-        $y
-    ";
-}
-
 function xml_error($num=-1, $msg=null, $file=null, $line=null) {
     global $xml_outer_tag;
     if (!$msg) {
@@ -205,4 +197,11 @@ function dtime() {
     return microtime(true);
 }
 
+// is $x a valid file (or dir) name?
+//
+function is_valid_filename($x) {
+    if (strstr($x, '/')) return false;
+    return true;
+}
+
 ?>