module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import os
import csv
import json

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def process(i):
    """
    Run collective tuning pipeline

    Input:  {
              state     - init state
              (state1)  - state where to record characteristics - if empty, use 'state'
              desc      - state description
            }

    Output: {
              cm_return - if =0, success
              state     - final state
              state1    - state1
            }
    """

    state=i.get('state',{})
    desc=i.get('desc',{})

    state1=state
    if 'state1' in i: state1=i.get('state1',{})

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('********************* Performing statistical analysis of characteristics ...')
    cm_kernel.print_for_con('')

    for q in desc:
        v=desc[q]

        if v.get('characteristic','')=='yes' and v.get('stat_characteristic','')=='yes':

           rg=cm_kernel.get_value_by_flattened_key({'cm_array':state, 'cm_key':q})
           if rg['cm_return']>0: return rg
           vv=rg['cm_value']

           t=v.get('type','')
           if vv!=None and vv!='' and (t=='integer' or t=='float'):

              cm_kernel.print_for_con('Processing '+q+' ...')

              if t=='integer': 
                 dv=int(vv)
#                 dt="%d"
              elif t=='float': 
                 dv=float(vv)
#                 dt="%f"

              q1='##cm_stat_characteristics'+q[1:]
              rg=cm_kernel.get_value_by_flattened_key({'cm_array':state1, 'cm_key':q1})
              if rg['cm_return']>0: return rg
              var=rg['cm_value']

              if var==None: var={}
              if 'number' not in var: var['number']='0'
              if 'min' not in var: var['min']='-1'
              if 'max' not in var: var['max']='-1'
              if 'all' not in var: var['all']=[]
              if 'delta' not in var: var['delta']='0'
              if 'delta_percent' not in var: var['delta_percent']='0'
              if 'nortest_lillie' not in var: var['nortest_lillie']={}
              if 'nortest_cvm' not in var: var['nortest_cvm']={}
              if 'nortest_pearson' not in var: var['nortest_pearson']={}

              dmin=float(var['min'])
              dmax=float(var['max'])
              ddelta=float(var['delta'])
              ddeltap=float(var['delta_percent'])

              # FGG: should move to more sophisticated statistical analysis from scipy or R
              # to detect if gaussian distribution or if there are spikes (usually mean that
              # there is some characterizing property missing that should be added to cM)

              if dmin==-1: dmin=dv
              elif dv<dmin: dmin=dv

              if dmax==-1: dmax=dv
              elif dv>dmax: dmax=dv

              ddelta=dmax-dmin

              if dmin!=0: ddeltap=ddelta/dmin

              var['min']=str(dmin)
              var['max']=str(dmax)
              var['all'].append(vv)
              var['delta']=str(ddelta)
              var['delta_percent']="%.4f" % ddeltap
              var['number']=str(len(var['all']))

              # Check if can use R
              if cm_kernel.ini['dcfg'].get('cm_use_r','')=='yes':
                 if len(var['all'])>7:
                    cm_kernel.print_for_con('')
                    cm_kernel.print_for_con('Starting R script for normality test ...')

                    r=cm_kernel.gen_uid({})
                    if r['cm_return']>0: return r
                    cm_r_filename='tmp-r-filename-'+r['cm_uid']+'.dat'

                    fx=file(cm_r_filename,'w')
                    for a in var['all']:
                        fx.write(str(a)+'\n')
                    fx.close()

                    os.putenv('CM_R_FILENAME', cm_r_filename)

                    m=os.path.join(ini['path'], ini['cfg']['cm_r_normtest'])

                    s='R < '+m+' --vanilla'

                    os.system(s)

                    os.remove(cm_r_filename)

                    # Reading files
                    if os.path.isfile(cm_r_filename+'.1.out'):
                       f=open(cm_r_filename+'.1.out', 'rb')
                       c=csv.DictReader(f, delimiter=',')
                       for a in c:
                           for b in a:
                               var['nortest_lillie'][b]="%.4f" % float(a[b])
                       f.close()
                       os.remove(cm_r_filename+'.1.out')
                    if os.path.isfile(cm_r_filename+'.2.out'):
                       f=open(cm_r_filename+'.2.out', 'rb')
                       c=csv.DictReader(f, delimiter=',')
                       for a in c:
                           for b in a:
                               var['nortest_cvm'][b]="%.4f" % float(a[b])
                       f.close()
                       os.remove(cm_r_filename+'.2.out')
                    if os.path.isfile(cm_r_filename+'.3.out'):
                       f=open(cm_r_filename+'.3.out', 'rb')
                       c=csv.DictReader(f, delimiter=',')
                       for a in c:
                           for b in a:
                               var['nortest_pearson'][b]="%.4f" % float(a[b])
                       f.close()
                       os.remove(cm_r_filename+'.3.out')

              rx=cm_kernel.set_value_by_flattened_key({'cm_array':state1, 'cm_key':q1, 'cm_value':var})
              if rx['cm_return']>0: return rx

    return {'cm_return':0, 'state':state, 'state1':state1}