numSOS Examples Directory

Some numSOS examples are in the Source under bin:

=== mem_used ===

#!/usr/bin/env python from future import print_function from sosdb import Sos from numsos.DataSource import SosDataSource from numsos.DataSink import CsvDataSink, SosDataSink from numsos.Transform import Transform from sosdb import DataSet from numsos.ArgParse import ArgParse import numpy as np import datetime as dt import time import argparse import sys

def get_times_from_args(args): if args.begin: start = int(args.begin.strftime("%s")) else: start = 0 if args.end: end = int(args.end.strftime("%s")) else: end = 0 if args.period: end = time.time() start = end - args.period return (start, end)

class Xfrm(Transform): def get_job_info(self, job_id): src = SosDataSource() src.config(cont=self.source.cont) src.select([ 'job_id', 'job_name', 'job_user' ], from_ = [ 'jobinfo' ], where = [ [ 'job_id', Sos.COND_EQ, job_id ] ], order_by = 'job_id') res = src.get_results(limit=1) return res.array('job_name')[0], res.array('job_user')[0]

def mem_stats(self, values):
    self.dup()
    self.mean([ 'MemFree' ], group_name='component_id')
    job_size = self.pop().get_series_size()

    data = self.top()

    # Compute (MemTotal - MemFree) / MemTotal
    memUsedRatio = (data['MemTotal'] - data['MemFree']) / data['MemTotal']
    memUsedRatio *= 100.0
    memUsedRatio.rename(0, 'Mem_Used_Pct')

    # compute the standard deviation
    self.push(memUsedRatio)
    self.std([ 'Mem_Used_Pct' ])
    result = self.pop()

    # compute the mean
    self.push(memUsedRatio)
    self.mean([ 'Mem_Used_Pct' ])
    result <<= self.pop()

    # compute the min memory used
    memUsedRatio <<= data['component_id']
    self.push(memUsedRatio)
    self.min([ 'Mem_Used_Pct' ], group_name='component_id')
    self.minrow('Mem_Used_Pct_min')
    self.top().rename('component_id', 'Min_Mem_Nid')
    result <<= self.pop()

    # compute the max memory used
    self.push(memUsedRatio)
    self.max([ 'Mem_Used_Pct' ], group_name='component_id')
    self.maxrow('Mem_Used_Pct_max')
    self.top().rename('component_id', 'Max_Mem_Nid')
    result <<= self.pop()

    job_id = int(values[0])
    job_name, job_user = self.get_job_info(job_id)

    print("{0:12} {1:12} {2:12} {3:12} "
          "{4:12} {5:12.2f} {6:12} {7:12.8f} {8:12.8f} {9:12.8f} ".format(
              int(values[0]),
              job_name, job_user,
              job_size,
              int(result.array('Min_Mem_Nid')[0]),
              result.array('Mem_Used_Pct_min')[0],
              int(result.array('Max_Mem_Nid')[0]),
              result.array('Mem_Used_Pct_max')[0],
              result.array('Mem_Used_Pct_mean')[0],
              result.array('Mem_Used_Pct_std')[0]))

if name == "main": parser = ArgParse(description="Compute memory summary statistics for a job") parser.add_argument( "--schema", required=False, default='meminfo', help="The meminfo schema name.") args = parser.parse_args() (start, end) = get_times_from_args(args) where = [] if start > 0: where.append([ 'timestamp', Sos.COND_GE, start ]) if end > 0: where.append([ 'timestamp', Sos.COND_LE, end ])

cont = Sos.Container(args.path)
src = SosDataSource()
src.config(cont=cont)
src.select([ 'timestamp', 'job_id', 'component_id', 'MemTotal', 'MemFree' ],
           from_    = [ args.schema ],
           where    = where,
           order_by = 'timestamp')
print("{0:12} {1:12} {2:12} {3:12} {4:12} {5:12} {6:12} {7:12} {8:12} {9:12}".format(
    "Job ID", "Job Name", "User",
    "Job Size",
    "Min Used NID", "Min Used %",
    "Max Used NID", "Max Used %",
    "Mean %", "Std"))
print("{0:12} {1:12} {2:12} {3:12} {4:12} {5:12} {6:12} {7:12} {8:12} {9:12}".format(
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-")))

xfrm = Xfrm(src, None, limit=1024 * 1024)
res = xfrm.begin()
while res:
    res = xfrm.next()
    if res is not None:
        # concatenate TOP and TOP~1
        xfrm.concat()

xfrm.for_each([ 'job_id' ], xfrm.mem_stats)

print("{0:12} {1:12} {2:12} {3:12} {4:12} {5:12} {6:12} {7:12} {8:12} {9:12}".format(
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-"),
    "-".ljust(12, "-")))