module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import copy
import os
import json
import random

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def build(i):

    """
    Build model

    Input:  {
              model_module_uoa             - model's module
              (model_data_uoa)             - model's data (if need to record)
              model_name                   - (earth, svm)
              data                 
              ct_dimensions_input
              ct_dimensions_output
              desc
              (record_data_to_file_prefix) - if !='', use this filename prefix instead of randomly generated
            }

    Output: {
              cm_return        - return code >0 if error
            }
    
    """

    mmu=i.get('model_module_uoa','')
    if mmu=='':
       return {'cm_return':1, 'cm_error':'model_module_uoa is not specified in math.model.r/build'}

    i['cm_run_module_uoa']=mmu
    return cm_kernel.access(i)

# ============================================================================
def predict(i):

    """
    Predict using some model

    Input:  {
              model_module_uoa        - model's module
              (model_data_uoa)        - model's data (if need to record)
              model_name              - (earth, svm)
              data                 
              ct_dimensions_input
              (ct_dimensions_output)  - for comparison
              desc                    - cM data description
              (max_variation_percent) - for comparison, report points where variation is more than this number (default=0.2)
            }

    Output: {
              cm_return - return code >0 if error
              (rmse)    - if comparison, root mean square error for predictions vs original
              (max_var) - list of points with variation more than max_variation_percent          
            }
    
    """

    mmu=i.get('model_module_uoa','')
    if mmu=='':
       return {'cm_return':1, 'cm_error':'model_module_uoa is not specified in math.model.r/build'}

    i['cm_run_module_uoa']=mmu
    return cm_kernel.access(i)

# ============================================================================
def fit(i):

    """
    Fit data using existing models (combine build and predict)

    Input:  {
              data                 
              ct_dimensions_input
              (ct_dimensions_output)  - for comparison
              desc                    - cM data description
              (max_variation_percent) - for comparison, report points where variation is more than this number (default=0.2)
            }

    Output: {
              cm_return - return code >0 if error
              results   - list with info about models including
                            model_file_size
                            model_rmse
            }
    
    """

    ii_orig=copy.deepcopy(i)

    tp=i['model_type']

    models=ini['cfg']['models']

    fm='tmp'

    results=[]

    for mm in models:
        if mm['type']==tp:
           mmx=mm['name']
           mmu=mm['model_module_uoa']
           mmn=mm['model_name']

           cm_kernel.print_for_con('****************************************************')
           cm_kernel.print_for_con('Fitting model '+mmx+' ...')

           ii=copy.deepcopy(ii_orig)

           # Build model
           ii['cm_run_module_uoa']=mmu
           ii['model_name']=mmn
           ii['record_data_to_file_prefix']=fm
           ii['cm_action']='build'

           r=cm_kernel.access(ii)
           if r['cm_return']>0: return r

           fm1=r['file_with_model']

           if os.path.isfile(fm1):
              ms=os.path.getsize(fm1)
              mm['model_file_size']=str(ms)

              # Predict
              ii['cm_run_module_uoa']=mmu
              ii['model_name']=mmn
              ii['model_file']=fm1
              ii['cm_action']='predict'

              r=cm_kernel.access(ii)
              if r['cm_return']>0: return r

              rmse=r['rmse']
              mm['model_rmse']=rmse

              results.append(mm)

    print json.dumps(results, indent=2)

    return {'cm_return':0, 'results':results}

# ============================================================================
def detect_representative_points(i):

    """
    Detect representative points to build the same model

    Input:  {
              model_type              - "regression" or "classification"
              data                 
              ct_dimensions_input
              (ct_dimensions_output)  - for comparison
              desc                    - cM data description
              (max_variation_percent) - for comparison, report points where variation is more than this number (default=0.2)
            }

    Output: {
              cm_return - return code >0 if error
              results   - list with info about models including
                            model_file_size
                            model_rmse
            }
    
    """

    mmu=i.get('model_module_uoa','')
    if mmu=='':
       return {'cm_return':1, 'cm_error':'model_module_uoa is not specified in math.model.r/build'}

    mmn=i.get('model_name','')
    if mmn=='': 
       return {'cm_return':1, 'cm_error':'model name is not specified'}

    data=i['data']

    dim1=i.get('ct_dimensions_input',[])
    dim2=i.get('ct_dimensions_output',[])

    ap=i.get('additional_params',{})

    rmse_t=float(i.get('rmse_threshold','0.2'))
    
    npoints=len(data[dim1[0]])

    fm='tmp'

    ii_orig=copy.deepcopy(i)

    results=[]

    rmse0=0
    sing0={}

#    for q in range(-1, npoints-5):
    for q in range(-1, npoints*2):
        ii=copy.deepcopy(ii_orig)
        data=ii['data']

        if q==-1:
           cm_kernel.print_for_con('****************************************************')
           cm_kernel.print_for_con('Building original model ...')
        else:
           cm_kernel.print_for_con('****************************************************')

#           q1=npoints-q-1
           q1=random.randint(0,len(data[dim1[0]])-1)

           cm_kernel.print_for_con('Step '+str(q+1)+' of '+str(npoints*2)+' ...')

           for d in dim1:
               del(data[d][q1])
           for d in dim2:
               del(data[d][q1])

        ii['data']=data

        # Build model
        ii['cm_run_module_uoa']=mmu
        ii['model_name']=mmn
        ii['record_data_to_file_prefix']=fm
        ii['cm_action']='build'
        ii['additional_params']=ap

        r=cm_kernel.access(ii)
        if r['cm_return']==0:
           sing=r.get('model',{}).get('singularities',{})

           fm1=r['file_with_model']
           if os.path.isfile(fm1):
              ms=os.path.getsize(fm1)

              # Predict
              ii['cm_run_module_uoa']=mmu
              ii['model_name']=mmn
              ii['model_file']=fm1
              ii['cm_action']='predict'

              r=cm_kernel.access(ii)
              if r['cm_return']>0: return r

              rmse=r['rmse']
              rmse1=float(rmse)

              if q==-1:
                 rmse0=float(rmse)
                 sing0=copy.deepcopy(sing)
              else:
                 delta=(rmse1-rmse0)/rmse0
                 print rmse0, rmse1, delta, len(sing), len(sing0)
                 results.append(delta)

                 if abs(delta)<rmse_t and len(sing)==len(sing0): 
                    data=ii_orig['data']

                    ii_orig['data']=data

                    for d in dim1:
                        del(data[d][q1])
                    for d in dim2:
                        del(data[d][q1])

                    print ''
                    print 'POINT REMOVED!'

    print json.dumps(results, indent=2)

    return {'cm_return':0, 'results':results}