module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import os
import csv
import math

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def build(i):

    """
    Build model

    Input:  {
              data                 
              ct_dimensions_input
              ct_dimensions_output
              desc
              model_name                   - (earth, svm)
              (record_data_to_file_prefix) - if !='', use this filename prefix instead of randomly generated
            }

    Output: {
              cm_return       - return code >0 if error
              file_with_model - file with model
            }
    
    """

    data=i.get('data',{})
    desc=i.get('desc',{})

    model_name=i.get('model_name','')
    if model_name=='': 
       return {'cm_return':1, 'cm_error':'model name is not specified'}
    
    dim1=i.get('ct_dimensions_input',[])
    dim2=i.get('ct_dimensions_output',[])
    
    dim=[]
    for d in dim1: dim.append(d)
    for d in dim2: dim.append(d)

    # Generate input tmp csv file name
    r=cm_kernel.gen_cm_tmp_file({})
    fitmp=r['cm_path1']+'.csv'

    # Generate output tmp file name
    fotmp=i.get('record_data_to_file_prefix','')
    if fotmp=='':
       r=cm_kernel.gen_cm_tmp_file({})
       fotmp=r['cm_path1']

    fotmp1=fotmp+'.r.model.obj'

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Generating tmp CSV file for learning (input='+fitmp+', output='+fotmp1+') ...')

    # Convert data to csv 
    ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.space'],
        'cm_action':'convert_ct_multi_array_to_csv',
        'cm_multi_array_s': data,
        'ct_dimensions': dim,
        'ct_dim_desc': desc,
        'ct_output_file': fitmp,
        'csv_no_header':'yes',
        'csv_separator':';',
        'csv_decimal_mark':'.'
       }
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    # Prepare calling model
    model_code=os.path.join(ini['path'], ini['cfg']['model_code_build']).replace(cm_kernel.convert_str_to_special('model_name'), model_name)
    
    cmd='r --vanilla --args '+fitmp+' '+fotmp+' < '+model_code
    os.system(cmd)

    if os.path.isfile(fitmp): os.remove(fitmp)

    return {'cm_return':0, 'file_with_model':fotmp1}

# ============================================================================
def predict(i):

    """
    Predict using model

    Input:  {
              model_file 
              data                 
              ct_dimensions_input
              (ct_dimensions_output)  - for comparison
              desc                    - cM data description
              model_name              - (earth, svm)
              (max_variation_percent) - for comparison, report points where variation is more than this number (default=0.2)
            }

    Output: {
              cm_return - return code >0 if error
              (rmse)    - if comparison, root mean square error for predictions vs original
              (max_var) - list of points with variation more than max_variation_percent          
            }
    
    """

    mf=i.get('model_file','')
    if mf=='':
       return {'cm_return':1, 'cm_error':'"model_file" is not defined'}

    model_name=i.get('model_name','')
    if model_name=='': 
       return {'cm_return':1, 'cm_error':'model name is not specified'}

    data=i.get('data',{})
    desc=i.get('desc',{})
    
    dim1=i.get('ct_dimensions_input',[])
    dim2=i.get('ct_dimensions_output',[])
    
    dim=[]
    for d in dim1: dim.append(d)
#    for d in dim2: dim.append(d)

    # Generate input tmp csv file name
    r=cm_kernel.gen_cm_tmp_file({})
    fitmp=r['cm_path1']+'.csv'

    # Generate output tmp file name
    r=cm_kernel.gen_cm_tmp_file({})
    fotmp=r['cm_path1']+'.csv'

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Generating tmp CSV file for learning (input='+fitmp+', output='+fotmp+') ...')

    # Convert data to csv 
    ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.space'],
        'cm_action':'convert_ct_multi_array_to_csv',
        'cm_multi_array_s': data,
        'ct_dimensions': dim,
        'ct_dim_desc': desc,
        'ct_output_file': fitmp,
        'csv_no_header':'yes',
        'csv_separator':';',
        'csv_decimal_mark':'.'
       }
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    # Prepare calling model
    model_code=os.path.join(ini['path'], ini['cfg']['model_code_predict']).replace(cm_kernel.convert_str_to_special('model_name'), model_name)
    
    cmd='r --vanilla --args '+mf+' '+fitmp+' '+fotmp+' < '+model_code
    os.system(cmd)

    # Trying to read csv
    if not os.path.isfile(fotmp):
       return {'cm_return':1, 'cm_error':'file with predictions was not created'}

    # Read predictions
    y=[]

    f=open(fotmp, 'rb')
    c=csv.DictReader(f, delimiter=',')
    for a in c:
        k=a.keys()
        if len(k)>0:
           y.append(a[k[1]])
    f.close()
    
    # Update original data array
    ky=i['ct_dimensions_output'][0]
    d0=data[ky]

    rr={'cm_return':0}

    if len(d0)>0:
       # If original data is present - substitute and check variation
       if len(d0)!=len(y):
          return {'cm_return':1, 'cm_error':'length of array with original data and predictions is not matching ('+str(len(data[ky]))+' vs '+str(len(y))+')'}

       tp=desc.get(ky,{}).get('type','')

       mvp=i.get('max_variation_percent','0.2')
       dmvp=float(mvp)

       var=[]
       
       s=0.0
       l=range(0, len(data[ky]))
       for q in l:
           v0=d0[q]
           v1=y[q]

           if tp=='float' or tp=='integer':
              if tp=='float':
                 dv0=float(v0)
                 dv1=float(v1)
              else:
                 dv0=int(v0)
                 dv1=int(v1)
              s+=(dv0-dv1)*(dv0-dv1)
              diff=abs(dv1-dv0)/dv0
              x1=''
              if diff>dmvp: 
                 x1=' ***'
                 var.append(q)
              print "%7s" % data[dim1[0]][q], "%7.3f" % dv0, "%7.3f" % dv1, "%7.3f" % s, "%5.3f" % diff,x1
           else:
              if v0!=v1:
                 s+=1

           d0[q]=y[q]

       rmse=math.sqrt(s/len(l))

       rr['rmse']=str(rmse)
       rr['max_var']=var

       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Model RMSE='+str(rmse))

    else:
       data[ky]=y

    # check saving to json (values as strings)
    fx=i.get('ct_output_file_s_json','')
    if fx!='':
       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Recording to json file '+fx+' ...')

       r=cm_kernel.save_array_to_file_as_json({'cm_filename':fx, 'cm_array':{'data':data, 'desc':desc}})
       if r['cm_return']>0: return r

    return rr