module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import copy
import time
import json
import random

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def scenario(i):
    """
    Build and run program, save to repository

    Input:  {
              caller_module_uid    - caller module UID
              state                - pipeline state
              exploration_scenario - exploraiton scenario parameters
              summary_point        - summary dimensions describing aggregated point
              aggregated_entry     - all aggregated points
              space                - if aggregated, current space
              original_input       - to get other additional forms per scenario
              cm_data_desc         - data description (from the original form)
              cm_form_explore      - form explore (form the original form)
              number_of_iterations - max number of iterations
            }

    Output: {
              cm_return   - if =0, success
            }
    """

    # Exploration scenarion parameters
    es=i.get('exploration_scenario',{})
    sr=es.get('save_results_in_repository','')
    aggr=es.get('aggregate_in_one_entry','')
    wrepo=es.get('work_repo_uoa','')
    lwrepo=es.get('work_repo_uoa','')
    pmd=i.get('pipeline_module_desc',{})

    desc=i.get('cm_data_desc',{})
    desc_copy=copy.deepcopy(desc) # description will be dynamically changing
                                  # this is the original one

    state_orig=i.get('state',{})
    state=copy.deepcopy(state_orig) # not to mix up original state
                                    # otherwise some strange aggregations of datasets, flags, etc
    ae=i.get('aggregated_entry',{})

    # Get path of aggregated LOCAL entry
    dae={}
    if aggr=='yes' and len(ae)>0:
       r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                           'cm_action':'load',
                           'cm_data_uoa':ae['cm_data_uoa'],
                           'cm_repo_uoa':lwrepo})
       if r['cm_return']>0:
          if r['cm_return']!=16: return r

          # Create entry
          r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                              'cm_action':'add',
                              'cm_data_uoa':ae['cm_data_uoa'],
                              'cm_array':{},
                              'cm_repo_uoa':lwrepo})
          if r['cm_return']>0: return r
       else:
          dae=r['cm_data_obj']['cfg']

    space=i.get('space',{})

    # Check additional scenario parameters
    ioi=i.get('original_input',{})

    frm1='#form_'+ini['cm_module_uid']+'_1'
    frm2='#form_'+ini['cm_module_uid']+'_2'

    a1={}
    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                        'cm_action':'detect_form_params',
                        'cm_array':ioi, 
                        'cm_prefix':frm1})
    if r['cm_return']>0: return r
    cm_form_array1=r['cm_array']

    r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1, 
                                         'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
    if r['cm_return']>0: return r
    a1=r['cm_array']

    ags=a1.get('aggregate_statistics','')
    rcn=a1.get('reuse_calibration_number','')

    # Detect data in the forms
    a2={}
    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                        'cm_action':'detect_form_params',
                        'cm_array':ioi, 
                        'cm_prefix':frm2})
    if r['cm_return']>0: return r
    cm_form_array2=r['cm_array']

    r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array2, 
                                         'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
    if r['cm_return']>0: return r
    a2=r['cm_array']

    # Get pipeline module and description
    mu=i['caller_module_uid']
    r=cm_kernel.get_data_description({'cm_module_uoa':mu})
    if r['cm_return']>0: return r
    pu=r['cm_data_obj']['cfg']['cm_p_uids']

    # Check if already have first point and if aggregate statistics, take statistical characteristics from it:
    repeat=''
    if len(space.get('points',[]))>0:
       sp=space['points'][0].get('pipeline_output',{}).get('state',{})

       # Take repetition number 
       if rcn=='yes':
          repeat=sp.get(pu['run_program'], {}).get('input',{}).get('run_vars',{}).get('CT_REPEAT_MAIN','')

       if ags=='yes':
          sc=sp.get('cm_stat_characteristics',{})
          if len(sc)>0:
             state['cm_stat_characteristics']=copy.deepcopy(sc)

    if a1.get('force_repeat_number','')!='':
       repeat=a1['force_repeat_number']

    # Start exploration
    cm_explore_state=[]
    iter=0
    start_time=time.time()

    nis=i.get('number_of_iterations','')
    ni=0
    if nis!='': ni=int(nis)

    # Set seed if needed
    if a1.get('random_seed','')!='':
       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Setting random seed to '+str(a1['random_seed']))
       cm_kernel.print_for_con('')
       random.seed(a1['random_seed'])

    finish=False
    while not finish:
       iter+=1
       if ni>0 and iter>ni: break

       #Prepare initial coarse grain choices
       cm_kernel.print_for_con('************************************************************************************')
       x=''
       if ni>0: x=' out of '+str(ni)
       cm_kernel.print_for_con('Fine-grain iteration:    '+str(iter)+x)

       t=time.time()-start_time
       ts="%.1f" % t
       cm_kernel.print_for_con('Time elapsed: '+ts+' sec.')

       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Preparing choices (fine-grain point in multi-dimensional space):')

       if rcn=='yes' and repeat!='':
          state[pu['run_program']]['input']['calibrate']='no'
          if 'run_vars' not in state[pu['run_program']]['input']: 
             state[pu['run_program']]['input']['run_vars']={}
          state[pu['run_program']]['input']['run_vars']['CT_REPEAT_MAIN']=repeat

       jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-choice'],
           'cm_action':'prepare_choices',
           'cm_array':state,
           'cm_data_desc':desc,
           'cm_explore':i.get('cm_form_explore',{}),
           'cm_explore_state':cm_explore_state,
           'cm_explore_level':'1'}
       r=cm_kernel.access(jj)
       if r['cm_return']>0: return r

       if a1.get('continue_indefinitely','')!='yes' and \
          r['cm_exploration_finished']=='yes':
          if r['cm_last_iteration']=='yes': finish=True
          else: break

       cm_explore_state=r['cm_explore_state']

       # Run pipeline
       ii={}
       ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['ctuning.pipeline.build_and_run_program']
       ii['cm_action']='run_stat'
       ii['state']=state
       ii_copy=copy.deepcopy(ii)
       # we may move next 2 after deepcopy to avoid creating too big data ...
       ii['desc']=desc # needed for stats ...
       ii['additional_params']=a1
       r=cm_kernel.access(ii)
       if r['cm_return']>0: return r

       statex=r['state']

       if repeat=='':
          repeat=statex.get(pu['run_program'], {}).get('input',{}).get('run_vars',{}).get('CT_REPEAT_MAIN','')

       # Append points
       if ags=='yes' and len(space.get('points',[]))>0:
          space['points'][0]['pipeline_input']=ii_copy
          space['points'][0]['pipeline_output']=r
       else:
          if 'points' not in space: space['points']=[]
          space['points'].append({'pipeline_input': ii_copy,
                                  'pipeline_output': r})

       # Record in repository
       if sr=='yes':
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Recording point in repository ...')

          jj={'cm_run_module_uoa':ini['cm_module_uid'],
              'cm_action':'add',
              'cm_array':space}

          if aggr=='yes' and len(ae)>0:
             cm_kernel.print_for_con('')
             cm_kernel.print_for_con('Recording point in aggregated entry '+ae['cm_data_uoa']+' ...')
             jj['cm_action']='update'
             jj.update(ae)

          jj['cm_repo_uoa']=wrepo

          jj_copy=copy.deepcopy(jj)

          r=cm_kernel.access(jj)
          if r['cm_return']>0: return r

          jj=jj_copy
          jj['cm_repo_uoa']=lwrepo

          r=cm_kernel.access(jj)
          if r['cm_return']>0: return r

          if aggr=='yes' and len(ae)==0:
             ae['cm_data_uoa']=r['cm_uid']
             ae['cm_module_uoa']=ini['cm_module_uid']
             ae['cm_repo_uoa']=wrepo

    return {'cm_return':0}

# ============================================================================
def prepare_input_params(i):
    """
    Prepare input parameters for this scenario

    Input:  {
              cm_form_array - array with forms
              form_exists   - yes or no
              cm_array1     - array from form 1 (coarse-grain points)
              cm_array2     - array from form 2 (additional parameters) just in case
              caller_cfg    - caller module cfg
            }

    Output: {
              cm_return   - if =0, success
            }
    """

    # Get array with forms
    af=i.get('cm_form_array',{})
    frm1='#form_'+ini['cm_module_uid']+'_1'
    frm2='#form_'+ini['cm_module_uid']+'_2'
    form_exists=i.get('form_exists','')
    xa1=i.get('cm_array1',{})
    xa2=i.get('cm_array2',{})
    cc=i.get('caller_cfg',{})

    # Detect data in the forms
    a1={}
    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                        'cm_action':'detect_form_params',
                        'cm_array':af, 
                        'cm_prefix':frm1})
    if r['cm_return']>0: return r
    cm_form_array1=r['cm_array']
    cm_form_commands1=r['cm_commands']
    cm_form_explore1=r['cm_explore']
    cm_data_desc1=ini['cfg']['scenario_desc']

    # Detect data in the forms
    a2={}
    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                        'cm_action':'detect_form_params',
                        'cm_array':af, 
                        'cm_prefix':frm2})
    if r['cm_return']>0: return r
    cm_form_array2=r['cm_array']
    cm_form_commands2=r['cm_commands']
    cm_form_explore2=r['cm_explore']

    if form_exists=='yes':
       r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1, 
                                            'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
       if r['cm_return']>0: return r
       a1=r['cm_array']

       r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array2, 
                                            'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
       if r['cm_return']>0: return r
       a2=r['cm_array']

    ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
        'cm_action':'visualize_data',
        'cm_array':a1,
        'cm_data_desc':cm_data_desc1,
        'cm_form_commands':cm_form_commands1,
        'cm_form_explore':cm_form_explore1,
        'cm_separator':'#',
        'cm_separator_form':frm1+'#',
        'cm_forms_exists':form_exists,
        'cm_support_raw_edit':'no',
        'cm_mode':'add',
        'hide_add_new_object':'yes',
        'cm_explore':'yes'}
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r
    cm_kernel.print_for_web(r['cm_string'])

    return {'cm_return':0}

# ============================================================================
def web_view_header(i):
    """
    Prepare header

    Input:  {
              header_desc - header description
            }

    Output: {
              cm_return   - if =0, success
              cm_html     - header html
              cm_length   - header length
            }
    """

    # Get web style
    if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
    else:
       return {'cm_return':1, 'cm_error':'web style is not defined'}

    local_pipeline_desc=ini['cfg'].get('pipeline_point_viewer_desc',{})

    x1=''
    if 'table_bgcolor_line5' in web: x1=' bgcolor="'+web['table_bgcolor_line5']+'" ' 
    x2=''
    if 'table_bgcolor_line6' in web: x2=' bgcolor="'+web['table_bgcolor_line6']+'" '

    vsx=ini['cfg']['pipeline_point_viewer']
    vs=vsx.get('main',[])
    ks=vsx.get('key_prefix','')

    desc=i.get('header_desc',{})

    h=''

    col=True; 
    for kx in vs:
        k=ks+kx
        ky='#'+kx
        v=desc.get(ky,{}).get('desc_text','')
        if v=='': v=local_pipeline_desc.get(ky,{}).get('desc_text','')

        if col: col=False ; x=x1
        else:   col=True  ; x=x2
        
        h+='<td align="center"'+x+'><small><b>'+v+':</b></small></td>'

    return {'cm_return':0, 'cm_html':h, 'cm_length':str(len(vs))}

# ============================================================================
def web_view_data(i):
    """
    Prepare header

    Input:  {
              cm_data        - data object from pipeline
              header_desc    - header description
              cur_line_style - { 
                                 cur    - cur (style1 or style2)
                                 style1
                                 style2
                               }
              point          - point to visualize
            }

    Output: {
              cm_return   - if =0, success
              cm_html     - header html
            }
    """

    # Get web style
    if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
    else:
       return {'cm_return':1, 'cm_error':'web style is not defined'}

    desc=i.get('header_desc',{})

    # Current style
    cls=i.get('cur_line_style',{})
    y=cls.get('cur','')
    y1=cls.get('style1','')
    y2=cls.get('style2','')

    # Prepare some vars to change colors for each second line
    x1=''
    if 'table_bgcolor_line3' in web: x1=' bgcolor="'+web['table_bgcolor_line3']+'" ' 
    x2=''
    if 'table_bgcolor_line4' in web: x2=' bgcolor="'+web['table_bgcolor_line4']+'" '
    x3=''
    if 'table_bgcolor_line5' in web: x3=' bgcolor="'+web['table_bgcolor_line5']+'" ' 
    x4=''
    if 'table_bgcolor_line6' in web: x4=' bgcolor="'+web['table_bgcolor_line6']+'" '

    vsx=ini['cfg']['pipeline_point_viewer']
    vs=vsx.get('main',[])
    vsf=vsx.get('fail',[])
    vss=vsx.get('strange',[])
    vsff=vsx.get('full_fail',[])
    vnt=vsx.get('normality_test',[])

    ks=vsx.get('key_prefix','')

    d=i.get('cm_data',{})
    points=d.get('cm_data_obj',{}).get('cfg',{}).get('points',[])

    h=''

    col=True; 
    for kx in vs:

        k=ks+kx

        if col: 
           col=False 
           if y==y1: x=x1
           else: x=x3
        else: 
           col=True 
           if y==y1: x=x2
           else: x=x4

        p=points[int(i.get('point','0'))]
        rg=cm_kernel.get_value_by_flattened_key({'cm_array':p, 'cm_key':k})
        if rg['cm_return']>0: return rg
        v=rg['cm_value']
        if v==None: v=''

        d1=desc.get('#'+kx,{})
        d1['cm_view_as_text']='yes'
        ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
            'cm_action':'convert_field_to_html',
            'cm_value':str(v),
            'cm_key':'',
            'cm_mode':'view',
            'cm_data_desc':d1,
            'cm_disable_coloring':'yes'}
        r1=cm_kernel.access(ii)
        if r1['cm_return']>0: return r1
        vh=r1['cm_value_html']
        vt=r1['cm_value_text']

        h+='<td '+x+' align="right" valign="top"><small>'
        if kx in vsf:
           if v=='' or int(v)!=0:
              h+='<span style="color:#9F0000"><B>Fail ('+str(v)+')</B></span>'
           else:
              h+=str(v)
        elif kx in vss:
           if v=='' or int(v)!=0:
              h+='<span style="color:#9F0000">Strange ('+str(v)+')</span>'
           else:
              h+=str(v)
        elif kx in vsff:
           if v=='yes':
              h+='<span style="color:#9F0000"><b>Fail</b></span>'
           else:
              h+='no'
        elif kx in vnt:
           if v!='' and float(v)<0.05:
              h+='<span style="color:#9F0000"><B>'+str(v)+'</B></span>'
           else:
              h+=str(v)
        else:
           h+=vh
        h+='<BR>\n'
        h+='</small></td>'

    return {'cm_return':0, 'cm_html':h}