module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import copy
import time
import json
import random
import math

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def scenario(i):
    """
    Extract program MILEPOST static properties (features), save to repository

    Input:  {
              caller_module_uid    - caller module UID
              state                - pipeline state
              exploration_scenario - exploraiton scenario parameters
              summary_point        - summary dimensions describing aggregated point
              aggregated_entry     - all aggregated points
              space                - if aggregated, current space
              original_input       - to get other additional forms per scenario
              cm_data_desc         - data description (from the original form)
              cm_form_explore      - form explore (form the original form)
              number_of_iterations - max number of iterations
            }

    Output: {
              cm_return   - if =0, success
            }
    """

    # Exploration scenarion parameters
    es=i.get('exploration_scenario',{})
    sr=es.get('save_results_in_repository','')
    aggr=es.get('aggregate_in_one_entry','')
    wrepo=es.get('work_repo_uoa','')
    lwrepo=es.get('work_repo_uoa','')
    pmd=i.get('pipeline_module_desc',{})

    desc=i.get('cm_data_desc',{})
    desc_copy=copy.deepcopy(desc) # description will be dynamically changing
                                  # this is the original one

    state_orig=i.get('state',{})
    state=copy.deepcopy(state_orig) # not to mix up original state
                                    # otherwise some strange aggregations of datasets, flags, etc
    ae=i.get('aggregated_entry',{})

    space=i.get('space',{})

    # Check additional scenario parameters
    ioi=i.get('original_input',{})

    frm1='#form_'+ini['cm_module_uid']+'_1'
    frm2='#form_'+ini['cm_module_uid']+'_2'

    a1={}
    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                        'cm_action':'detect_form_params',
                        'cm_array':ioi, 
                        'cm_prefix':frm1})
    if r['cm_return']>0: return r
    cm_form_array1=r['cm_array']

    r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1, 
                                         'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
    if r['cm_return']>0: return r
    a1=r['cm_array']

    ags=a1.get('aggregate_statistics','')
    mgbf=a1.get('milepost_gcc_base_flag','')
    mgaf=a1.get('milepost_gcc_additional_flags','')

    # Get pipeline module and description
    mu=i['caller_module_uid']
    r=cm_kernel.get_data_description({'cm_module_uoa':mu})
    if r['cm_return']>0: return r
    pu=r['cm_data_obj']['cfg']['cm_p_uids']

    # Start exploration
    cm_explore_state=[]

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Preparing choices (fine-grain point in multi-dimensional space):')

    # Load program to get most time consuming functions (if there)
    hot_funcs=[]

    prog_uoa=state[pu['init']]['input']['program_uoa']
    rx=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
                       'cm_data_uoa':prog_uoa,
                       'cm_action':'load'})
    if rx['cm_return']>0: return rx
    prog_uid=rx['cm_uid']

    dprog=rx['cm_data_obj']['cfg']

    rcmd=dprog.get('run_cmds',{})
    for q in rcmd:
        q1=rcmd[q].get('hot_functions',[])
        for q2 in q1:
            func=q2.get('name','')
            if func!='':
               hot_funcs.append(func)
   
    if len(hot_funcs)>0:
       state[pu['use_ctuning_milepost']]['input']['hot_functions']=hot_funcs

    # Fix various parameters
    state[pu['run_program']]['enable']='no'
    state[pu['use_ctuning_milepost']]['enable']='yes'
    state[pu['use_ctuning_milepost']]['input']['mode']='extract_program_static_properties'

    milepost_compiler_code_uoa=state[pu['use_ctuning_milepost']]['input'].get('milepost_compiler_code_uoa','')
    state[pu['init']]['input']['compiler_code_uoa']=milepost_compiler_code_uoa

    kp='##'+pu['prepare_compiler_flags']+'#input#all_compiler_flags#' # compiler description prefix in descriptions

    state[pu['build_program']]['enable']='yes'
    del(state[pu['prepare_compiler_flags']]['input']['all_compiler_flags'])
    state[pu['prepare_compiler_flags']]['input']['all_compiler_flags']={}
    state[pu['prepare_compiler_flags']]['input']['all_compiler_flags']['base_flag']=mgbf
    state[pu['build_program']]['input']['compiler_flags']=mgaf

    jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-choice'],
        'cm_action':'prepare_choices',
        'cm_array':state,
        'cm_data_desc':desc,
        'cm_explore':i.get('cm_form_explore',{}),
        'cm_explore_state':cm_explore_state,
        'cm_explore_level':'1'}
    r=cm_kernel.access(jj)
    if r['cm_return']>0: return r

    cm_explore_state=r['cm_explore_state']

    # Clean flags (not needed so not to confuse)
    del(state[pu['prepare_compiler_flags']]['input']['all_compiler_flags'])
    state[pu['prepare_compiler_flags']]['input']['all_compiler_flags']={}
    state[pu['prepare_compiler_flags']]['input']['all_compiler_flags']['base_flag']=mgbf

    # Run pipeline
    ii={}
    ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['ctuning.pipeline.build_and_run_program']
    ii['cm_action']='run_stat'
    ii['state']=state
    ii_copy=copy.deepcopy(ii)
    # we may move next 2 after deepcopy to avoid creating too big data ...
    ii['desc']=desc # needed for stats ...
    ii['additional_params']=a1
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    # Append points
    if ags=='yes' and len(space.get('points',[]))>0:
       space['points'][0]['pipeline_input']=ii_copy
       space['points'][0]['pipeline_output']=r
    else:
       space['points'].append({'pipeline_input': ii_copy,
                               'pipeline_output': r})

    # Record in repository
    if sr=='yes':
       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Recording point in repository ...')

       jj={'cm_run_module_uoa':ini['cm_module_uid'],
           'cm_action':'add',
           'cm_array':space}

       if aggr=='yes' and len(ae)>0:
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Recording point in aggregated entry '+ae['cm_data_uoa']+' ...')
          jj['cm_action']='update'
          jj.update(ae)
       else:
          # Use program UID
          jj['cm_data_uid']=prog_uid

       jj['cm_repo_uoa']=wrepo

       r=cm_kernel.access(jj)
       if r['cm_return']>0: return r

    return {'cm_return':0}

# ============================================================================
def prepare_input_params(i):
    """
    Prepare input parameters for this scenario

    Input:  {
              cm_form_array - array with forms
              form_exists   - yes or no
              cm_array1     - array from form 1 (coarse-grain points)
              cm_array2     - array from form 2 (additional parameters) just in case
              caller_cfg    - caller module cfg
            }

    Output: {
              cm_return   - if =0, success
            }
    """

    # Get array with forms
    af=i.get('cm_form_array',{})
    frm1='#form_'+ini['cm_module_uid']+'_1'
    frm2='#form_'+ini['cm_module_uid']+'_2'
    form_exists=i.get('form_exists','')
    xa1=i.get('cm_array1',{})
    xa2=i.get('cm_array2',{})
    cc=i.get('caller_cfg',{})

    # Detect data in the forms
    a1={}
    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                        'cm_action':'detect_form_params',
                        'cm_array':af, 
                        'cm_prefix':frm1})
    if r['cm_return']>0: return r
    cm_form_array1=r['cm_array']
    cm_form_commands1=r['cm_commands']
    cm_form_explore1=r['cm_explore']
    cm_data_desc1=ini['cfg']['scenario_desc']

    # Detect data in the forms
    a2={}
    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                        'cm_action':'detect_form_params',
                        'cm_array':af, 
                        'cm_prefix':frm2})
    if r['cm_return']>0: return r
    cm_form_array2=r['cm_array']
    cm_form_commands2=r['cm_commands']
    cm_form_explore2=r['cm_explore']

    if form_exists=='yes':
       r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1, 
                                            'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
       if r['cm_return']>0: return r
       a1=r['cm_array']

       r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array2, 
                                            'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
       if r['cm_return']>0: return r
       a2=r['cm_array']

    ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
        'cm_action':'visualize_data',
        'cm_array':a1,
        'cm_data_desc':cm_data_desc1,
        'cm_form_commands':cm_form_commands1,
        'cm_form_explore':cm_form_explore1,
        'cm_separator':'#',
        'cm_separator_form':frm1+'#',
        'cm_forms_exists':form_exists,
        'cm_support_raw_edit':'no',
        'cm_mode':'add',
        'hide_add_new_object':'yes',
        'cm_explore':'yes'}
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r
    cm_kernel.print_for_web(r['cm_string'])

    return {'cm_return':0}

# ============================================================================
def web_view_header(i):
    """
    Prepare header

    Input:  {
              header_desc - header description
            }

    Output: {
              cm_return   - if =0, success
              cm_html     - header html
              cm_length   - header length
            }
    """

    # Get web style
    if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
    else:
       return {'cm_return':1, 'cm_error':'web style is not defined'}

    local_pipeline_desc=ini['cfg'].get('pipeline_point_viewer_desc',{})

    x1=''
    if 'table_bgcolor_line5' in web: x1=' bgcolor="'+web['table_bgcolor_line5']+'" ' 
    x2=''
    if 'table_bgcolor_line6' in web: x2=' bgcolor="'+web['table_bgcolor_line6']+'" '

    vsx=ini['cfg']['pipeline_point_viewer']
    vs=vsx.get('main',[])
    ks=vsx.get('key_prefix','')

    desc=i.get('header_desc',{})

    h=''

    col=True; 

    for kx in vs:
        if kx=='': break

        k=ks+kx
        ky='#'+kx
        v=desc.get(ky,{}).get('desc_text','')
        if v=='': v=local_pipeline_desc.get(ky,{}).get('desc_text','')

        if col: col=False ; x=x1
        else:   col=True  ; x=x2
        
        h+='<td align="center"'+x+'><small><b>'+v+':</b></small></td>'

    if col: col=False ; x=x1
    else:   col=True  ; x=x2
    h+='<td align="center"'+x+'><small><b>Function:</b></small></td>'

    if col: col=False ; x=x1
    else:   col=True  ; x=x2
    h+='<td align="center"'+x+'><small><b>Extracted after this compiler pass:</b></small></td>'

    # Print features
    ff=ini['cfg']['milepost_features_description']
    for qx in ff:
        kx=qx.keys()[0]
        v=qx[kx]

        vd=v.get('desc','')
        vn=v.get('normalized','')

        if col: col=False ; x=x1
        else:   col=True  ; x=x2
        
        h+='<td align="center"'+x+'><small><b>Program feature '+kx+':<br><br><i>'+vd+'</i>:</b></small></td>'

    return {'cm_return':0, 'cm_html':h, 'cm_length':str(len(vs))}

# ============================================================================
def web_view_data(i):
    """
    Prepare header

    Input:  {
              cm_data        - data object from pipeline
              header_desc    - header description
              cur_line_style - { 
                                 cur    - cur (style1 or style2)
                                 style1
                                 style2
                               }
              point          - point to visualize
            }

    Output: {
              cm_return   - if =0, success
              cm_html     - header html
            }
    """

    # Get web style
    if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
    else:
       return {'cm_return':1, 'cm_error':'web style is not defined'}

    desc=i.get('header_desc',{})

    # Current style
    cls=i.get('cur_line_style',{})
    y=cls.get('cur','')
    y1=cls.get('style1','')
    y2=cls.get('style2','')

    d=i.get('cm_data',{})
    d1=d.get('cm_data_obj',{}).get('cfg',{})
    points=d1.get('points',[])

    # Prepare some vars to change colors for each second line
    x1=''
    if 'table_bgcolor_line3' in web: x1=' bgcolor="'+web['table_bgcolor_line3']+'" ' 
    x2=''
    if 'table_bgcolor_line4' in web: x2=' bgcolor="'+web['table_bgcolor_line4']+'" '
    x3=''
    if 'table_bgcolor_line5' in web: x3=' bgcolor="'+web['table_bgcolor_line5']+'" ' 
    x4=''
    if 'table_bgcolor_line6' in web: x4=' bgcolor="'+web['table_bgcolor_line6']+'" '

    vsx=ini['cfg']['pipeline_point_viewer']
    vs=vsx.get('main',[])
    vsf=vsx.get('fail',[])
    vss=vsx.get('strange',[])
    vsff=vsx.get('full_fail',[])
    vnt=vsx.get('normality_test',[])

    ks=vsx.get('key_prefix','')

    d=i.get('cm_data',{})
    d1=d.get('cm_data_obj',{}).get('cfg',{})
    points=d1.get('points',[])

    # Get pipeline module and description
    mu=d1.get('scenario_module_uoa','')
    r=cm_kernel.get_data_description({'cm_module_uoa':mu})
    if r['cm_return']>0: return r
    pu=r['cm_data_obj']['cfg']['cm_p_uids']

    h=''

    col=True; 
    for kx in vs:

        if kx=='': break

        k=ks+kx

        if col: 
           col=False 
           if y==y1: x=x1
           else: x=x3
        else: 
           col=True 
           if y==y1: x=x2
           else: x=x4

        p=points[int(i.get('point','0'))]
        rg=cm_kernel.get_value_by_flattened_key({'cm_array':p, 'cm_key':k})
        if rg['cm_return']>0: return rg
        v=rg['cm_value']
        if v==None: v=''

        d1=desc.get('#'+kx,{})
        d1['cm_view_as_text']='yes'
        ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
            'cm_action':'convert_field_to_html',
            'cm_value':str(v),
            'cm_key':'',
            'cm_mode':'view',
            'cm_data_desc':d1,
            'cm_disable_coloring':'yes'}
        r1=cm_kernel.access(ii)
        if r1['cm_return']>0: return r1
        vh=r1['cm_value_html']
        vt=r1['cm_value_text']

        h+='<td '+x+' align="right" valign="top"><small>'
        if kx in vsf:
           if v=='' or int(v)!=0:
              h+='<span style="color:#9F0000"><B>Fail ('+str(v)+')</B></span>'
           else:
              h+=str(v)
        elif kx in vss:
           if v=='' or int(v)!=0:
              h+='<span style="color:#9F0000">Strange ('+str(v)+')</span>'
           else:
              h+=str(v)
        elif kx in vsff:
           if v=='yes':
              h+='<span style="color:#9F0000"><b>Fail</b></span>'
           else:
              h+='no'
        elif kx in vnt:
           if v!='' and float(v)<0.05:
              h+='<span style="color:#9F0000"><B>'+str(v)+'</B></span>'
           else:
              h+=str(v)
        else:
           h+=vh
        h+='<BR>\n'
        h+='</small></td>'

    if col: 
       col=False 
       if y==y1: x=x1
       else: x=x3
    else: 
       col=True 
       if y==y1: x=x2
       else: x=x4
    h+='<td align="center"'+x+'><small><b>'
    mf=p.get('pipeline_output',{}).get('state',{}).get(pu['finish_ctuning_milepost'],{}).get('output',{}).get('program_structure',{})
    for q in mf:
        h+=str(q)+'<br>'
    h+='</b></small></td>'

    if col: 
       col=False 
       if y==y1: x=x1
       else: x=x3
    else: 
       col=True 
       if y==y1: x=x2
       else: x=x4
    h+='<td align="center"'+x+'><small>'
    mf=p.get('pipeline_output',{}).get('state',{}).get(pu['finish_ctuning_milepost'],{}).get('output',{}).get('program_structure',{})
    for q in mf:
        qq=mf[q].get('program_static_properties',{})
        qq1=qq.keys()
        if len(qq1)>0:
           kx=qq1[0]
           v=qq[kx]

        h+=str(kx)+'<br>'
    h+='</small></td>'

    ff=ini['cfg']['milepost_features_description']
    for qx in ff:
        kx=qx.keys()[0]
        v=qx[kx]

        if col: 
           col=False 
           if y==y1: x=x1
           else: x=x3
        else: 
           col=True 
           if y==y1: x=x2
           else: x=x4
        h+='<td align="center"'+x+'><small>'
        mf=p.get('pipeline_output',{}).get('state',{}).get(pu['finish_ctuning_milepost'],{}).get('output',{}).get('program_structure',{})

        for q in mf:
            qq=mf[q].get('program_static_properties',{})

            v1=''
            v2=''

            qq1=qq.keys()
            if len(qq1)>0:
               kx1=qq1[0]
               v1=qq[kx1]

               zqx=kx[2:]

               v2=v1.get(zqx,'')

            h+=str(v2)+'<br>'
        h+='</small></td>'

    return {'cm_return':0, 'cm_html':h}

# ============================================================================
def calculate_similarity(i):
    """
    Calculate similarity 
      FGG: for now simple euclidean distance - it's just a demo of the concept, 
           since it has many limitations so we need to collaboratively extend it
           (using PCA, SVM, etc)

    Input:  {
               program1_milepost_features - MILEPOST features for program one
               program2_milepost_features - MILEPOST features for program two
            }

    Output: {
              cm_return   - if =0, success
              distance    - Euclidean distance (can be more than 1)
            }
    """

    prog1=i['program1_milepost_features']
    prog2=i['program2_milepost_features']

    # Check which feature to use for normalization
    fd=ini['cfg']['milepost_features_description']

    ftn=''
    for q in fd:
        q1=q.keys()[0]
        v=q[q1]

        if v.get('normalizator','')=='yes':
           ftn=q1[2:]
           break

    # Check that in vectors
    ftnp1=float(prog1[ftn])
    ftnp2=float(prog2[ftn])

    distance=0.0
    
    # Calculating equlivdian distance (1st algorithm). FGG: we should add K-nearest neighbour later ...
    for q in fd:
        q1=q.keys()[0]
        v=q[q1]

        q2=q1[2:]

        ftp1=float(prog1[q2])
        ftp2=float(prog2[q2])

        if v.get('use_for_euclidean_distance','')=='yes':
           if v.get('normalized','')!='yes':
              ftp1/=ftnp1
              ftp2/=ftnp2

           distance+=(ftp1-ftp2)*(ftp1-ftp2)

    distance=math.sqrt(distance)

    return {'cm_return':0, 'distance':str(distance)}