module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import copy
import time
import json
import random

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def record(i):
    """
    Process and record semi-raw data during benchmarking

    Input:  {
              cm_array    - semi-raw benchmarking info that will be converted 
                            to cM format and recorded to cM repository
              cm_repo_uoa - working repository
            }

    Output: {
              cm_return   - if =0, success
            }
    """

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Processing and recording semi-raw data during benchmarking ...')
 
    repo=i.get('cm_repo_uoa','')

    entry={} # This entry will be prepared and saved in the repository
    entry['points']=[] # Universal multi-dimensional points in space (for visualization and analysis)

    ca=i.get('cm_array',{})
    if len(ca)>0:
       cbs=ca['common_benchmark_setup']
       be=ca['benchmark_execution']

       # Checking board *********************************************************************
       ss=cbs.get('system_string','')
       if ss!='':
          system_uoa=''

          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Searching for system "'+ss+'" in the repository ...')
 
          ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'],
              'cm_action':'search',
              'key_0':'##cm_properties#system_string',
              'value_0':ss,
              'use_flat_array':'yes',
              'module_selection':[ini['cfg']['cm_modules']['ctuning.system']],
             }
          if repo!='': ii['repo_selection']=[repo]
          r=cm_kernel.access(ii)
          if r['cm_return']>0: return r

          all=r['all']

          if len(all)==0:
             cm_kernel.print_for_con('  System not found in the repository, adding new one ...')

             # Prepare to add
             jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.system'],
                 'cm_action':'add',
                 'cm_array':{'cm_properties':{'system_string':ss}}, 
                 'cm_display_as_alias':ss
                }
             if repo!='': jj['cm_repo_uoa']=repo

             r=cm_kernel.access(jj)
             if r['cm_return']>0: return r

             system_uoa=r['cm_uid']

             cm_kernel.print_for_con('  System entry added: '+system_uoa)
          else:
             system_uoa=all[0]['cm_data_uoa']

             cm_kernel.print_for_con('  System entry found: '+system_uoa)

       # Prepare common info in the entry *********************************************************************
       entry['scenario_module_uoa']=ini['cm_module_uid']
       entry['pipeline_module_uoa']=ini['cfg']['cm_modules']['ctuning.pipeline.build_and_run_program']

       # Will be used to find exactly the right entry during new explorations/search
       entry['summary_point']={}
       entry['summary_point']['target_processor_uoa']=cbs['target_processor_uoa']
       entry['summary_point']['tuning_objective']=cbs['tuning_objective']
       entry['summary_point']['notes']=cbs['notes']
       entry['summary_point']['ctuning_system_uoa']=system_uoa
       entry['summary_point']['benchmark_execution_internal_id']=be['internal_id']
       entry['summary_point']['benchmark_execution_reason']=be['reason']
       entry['summary_point']['common_benchmark_setup_svn_repo_revision']=cbs['svn_repo_revision']

       # Will be used to visualize and prune coarse-grain results in the cM
       entry['summary_point_relaxed']={}
       entry['summary_point_relaxed'].update(entry['summary_point'])

       entry['summary_point_relaxed']['system_string']=cbs['system_string']

       # Check if already added
       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Checking if entry already added ...')

       ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.pipeline.build_and_run_program'],
           'cm_action':'find_aggregated_entry',
           'record_module_uoa':ini['cm_module_uid'],
           'summary_point':entry['summary_point']}
       if repo!='': ii['work_repo_uoa']=repo
       r=cm_kernel.access(ii)
       if r['cm_return']>0: return r

       ae=r['aggregated_entries']

       if len(ae)>0:
          # Go through results *********************************************************************
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Entry already exists ...')

          r={}
          r.update(ae[0])
          r['cm_uid']=r['cm_data_uid']
          r['cm_return']=0
          return r
       else:
          # Go through results *********************************************************************
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Processing individual benchmarks ...')

          results=ca.get('results',{})
          if len(results)>0:
             for result in results:
                 bench=result['benchmark_name']
                 
                 bench_uoa=''

                 cm_kernel.print_for_con('')
                 cm_kernel.print_for_con('Searching for benchmark "'+bench+'" in the repository ...')
        
                 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'],
                     'cm_action':'search',
                     'key_0':'##cm_properties#benchmark_string',
                     'value_0':bench,
                     'use_flat_array':'yes',
                     'module_selection':[ini['cfg']['cm_modules']['code.source']],
                     'timeout':'1000'
                    }
                 if repo!='': ii['repo_selection']=[repo]
                 r=cm_kernel.access(ii)
                 if r['cm_return']>0: return r

                 all=r['all']
                 
                 if len(all)==0:
                    cm_kernel.print_for_con('  Benchmark not found in the repository, adding new one ...')

                    # Prepare to add
                    jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
                        'cm_action':'add',
                        'cm_array':{'cm_properties':{'benchmark_string':bench}}, 
                        'cm_display_as_alias':bench
                       }
                    if repo!='': jj['cm_repo_uoa']=repo

                    r=cm_kernel.access(jj)
                    if r['cm_return']>0: return r

                    bench_uoa=r['cm_uid']

                    cm_kernel.print_for_con('  Benchmark entry added: '+bench_uoa)
                 else:
                    bench_uoa=all[0]['cm_data_uoa']

                    cm_kernel.print_for_con('  Benchmark entry found: '+bench_uoa)

                 # Adding points to entry ************************************************************
                 point={}

                 # Raw results from benchmarks
                 point['raw']=result

                 # Results and other data converted to cTuning pipeline (already has description and connections)

                 # Load pipeline description
                 r=cm_kernel.get_data_description({'cm_module_uoa':ini['cfg']['cm_modules']['ctuning.pipeline.build_and_run_program']})
                 if r['cm_return']>0: return r
                 pu=r['cm_data_obj']['cfg']['cm_p_uids']

#                 point['pipeline_input']={}
#                 point['pipeline_input']['state']={}
#                 point['pipeline_input']['state'][pu['init']]={'input':entry['summary_point']}

                 point['pipeline_output']={}
                 point['pipeline_output']['cm_return']=0
                 point['pipeline_output']['fail']=False
                 point['pipeline_output']['state']={}
                 point['pipeline_output']['state'][pu['init']]={'input':entry['summary_point']}

                 point['pipeline_output']['state'][pu['init_params']]={'output':{'global_source_code_name':result['benchmark_name'],
                                                                                 'program_uoa':bench_uoa}}

                 point['pipeline_output']['state'][pu['build_program']]={'output':{'exit_code':result['build_exit_code'], 
                                                                                  'run_time_by_module':result['compilation_time']}}

                 point['pipeline_output']['state'][pu['run_program']]={'output':{'exit_code':result['run_exit_code'], 
                                                                                 'benchmark_score':result['benchmark_score']}}

                 point['pipeline_output']['state'][pu['check_output']]={'output':{'output_correct':result['output_correct']}} 

                 entry['points'].append(point)

          # Record in the repository
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Entry prepared! Recording in the repository ...')

          ii={'cm_run_module_uoa':ini['cm_module_uid'],
              'cm_action':'add',
              'cm_array':entry 
             }
          if repo!='': ii['cm_repo_uoa']=repo
          return cm_kernel.access(ii)

    return {'cm_return':0}

# ============================================================================
def analyze_variation(i):
    """
    Analyze variation in characteristics

    Input:  {
              (data)                           - list of data entries to analyze
              (prune_by_summary_point_relaxed) - if data is not provided, search will be performed for all entries;
                                                 returned entries can be pruned by this dictionary 
                                                 using 'summary_point_relaxed' in entries
              characteristic_key               - key to get characteristic (will be converted to float)
              percent_threshold                - percent threshold to detect benchmarks where characteristic has high variability 
              (cm_repo_uoa)                    - working repository
            }

    Output: {
              cm_return   - if =0, success
            }
    """

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Analyzing variation in characteristics ...')

    repo=i.get('cm_repo_uoa','')

    ck=i.get('characteristic_key','')
    if ck=='':
       return {'cm_return':1,'cm_error':'characteristic_key is not defined'}

    # Get benchmarks and their variation
    ii={'data':i.get('data',{}),
        'prune_by_summary_point_relaxed':i.get('prune_by_summary_point_relaxed',{}),
        'characteristic_key':ck}
    if repo!='': ii['cm_repo_uoa']=repo
    r=get_stats_for_benchmarks(i)
    if r['cm_return']>0: return r

    variation=r['stats']
    if len(variation)>0:
       # Find benchmarks with high variation
       hbench=[] #benchmarks with high variability
       pt=float(i.get('percent_threshold','0.10'))
       for bench in variation:
           var=variation[bench]
           v1=float(var['percent_delta_to_min'])

           if v1>pt:
              hbench.append(bench)

       # Print results
       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Results:')

       for t in ('high', 'normal'):
           cm_kernel.print_for_con('')
           if t=='high':
              cm_kernel.print_for_con('  Benchmarks with high variability:')
           else:
              cm_kernel.print_for_con('  Normal benchmarks (percent < '+str(pt*100)+'%) :')

           cm_kernel.print_for_con('')
           for bench in variation:
               var=variation[bench]
               if t=='high' and bench in hbench or \
                  t=='normal' and bench not in hbench:
                  smin=str(float(var['min']))
                  smax=str(float(var['max']))
                  sper=str(float(var['percent_delta_to_min'])*100)
                  cm_kernel.print_for_con('     '+bench+'   min='+smin+'; max='+smax+'; percent_variation='+sper)

    return {'cm_return':0}

# ============================================================================
def compare_systems(i):
    """
    Compare systems through benchmarking

    Input:  {
            }

    Output: {
              cm_return   - if =0, success
            }
    """

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Comparing systems through benchmarking ...')

    repo=i.get('cm_repo_uoa','')

    ck=i.get('characteristic_key','')
    if ck=='':
       return {'cm_return':1,'cm_error':'characteristic_key is not defined'}

    # Get benchmarks (1) and their variation
    ii={'data':i.get('data1',[]),
        'prune_by_summary_point_relaxed':i.get('prune_by_summary_point_relaxed1',{}),
        'characteristic_key':ck}
    if repo!='': ii['cm_repo_uoa']=repo
    r=get_stats_for_benchmarks(ii)
    if r['cm_return']>0: return r
    variation1=r['stats']

    # Get benchmarks (2) and their variation
    ii={'data':i.get('data2',[]),
        'prune_by_summary_point_relaxed':i.get('prune_by_summary_point_relaxed2',{}),
        'characteristic_key':ck}
    if repo!='': ii['cm_repo_uoa']=repo
    r=get_stats_for_benchmarks(ii)
    if r['cm_return']>0: return r
    variation2=r['stats']

    if len(variation1)>0 and len(variation2)>0:
       # Compare systems

       cm_kernel.print_for_con('  Testing            min1      <  min2      <  max2      <  max1')
       cm_kernel.print_for_con('')

       hbench=[] #benchmarks with high variability
       fail=False
       for bench in variation1:
           var1=variation1[bench]
           var2=variation2[bench]

           smin1=str(float(var1['min']))
           smax1=str(float(var1['max']))

           smin2=str(float(var2['min']))
           smax2=str(float(var2['max']))

           stest=''
           if var1['min']>var2['min'] or var2['max']>var1['max']:
              stest=' - test failed'
              fail=True

           cm_kernel.print_for_con('     '+bench+'   '+smin1+' <= '+smin2+' <= '+smax2+' <= '+smax1+'   '+stest)

       cm_kernel.print_for_con('')
       if fail:
           cm_kernel.print_for_con('  Test failed - systems are not similar')
       else:
           cm_kernel.print_for_con('  Systems similar')

    return {'cm_return':0}

# ============================================================================
def get_stats_for_benchmarks(i):
    """
    Compare systems through benchmarking

    Input:  {
              (data)                           - list of data entries to analyze
              (prune_by_summary_point_relaxed) - if data is not provided, search will be performed for all entries;
                                                 returned entries can be pruned by this dictionary 
                                                 using 'summary_point_relaxed' in entries
              characteristic_key               - key to get characteristic (will be converted to float)
              (cm_repo_uoa)                    - working repository
            }

    Output: {
              cm_return   - if =0, success
              stats       - dictionary of format
                            {
                              "benchmark1":{
                                            "min"                  - min value
                                            "max"                  - max value
                                            "delta"                - max-min
                                            "percent_delta_to_min" - delta/min
                                            "mean"                 - mean
                                            "std"                  - standard deviation
                                            "percent_std_to_min"   - standard deviation/min  
                                           }
                              ...    
                            }

            }
    """

    repo=i.get('cm_repo_uoa','')

    ck=i.get('characteristic_key','')
    if ck=='':
       return {'cm_return':1,'cm_error':'characteristic_key is not defined'}

    benchmarks={} # collect stats per benchmark

    stats={}

    # Get either all data, or using search or directly from array
    data=i.get('data',[])
    if len(data)==0:
       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Selecting/searching entries ...')
       cm_kernel.print_for_con('')

       ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'],
           'cm_action':'search',
           'module_selection':[ini['cm_module_uid']]
          }
       if repo!='': ii['repo_selection']=[repo]

       ii['use_flat_array']='yes'
       ii['show_search_progress']='yes'
       ii['timeout']='900'

       # Check if pruning is needed
       sum1=i.get('prune_by_summary_point_relaxed',{})
       if len(sum1)>0:
          j=0
          for k in sum1:
              ii['key_'+str(j)]='##summary_point_relaxed#'+k
              ii['value_'+str(j)]=sum1[k]
              j+=1
          ii['strict_match']='yes'

       r=cm_kernel.access(ii)
       if r['cm_return']>0: return r

       # Adding entries to data
       for q in r['all']:
           data.append(q['cm_data_uoa'])

    # If data entries are selected
    if len(data)>0:
       # Collect statistics per benchmark in an array for further analysis
       num=0
       for q in data:
           num+=1

           # Load data entry
           cm_kernel.print_for_con('')
           cm_kernel.print_for_con('Loading entry '+q+' ('+str(num)+' of '+str(len(data))+') ...')

           ii={'cm_run_module_uoa':ini['cm_module_uid'],
               'cm_action':'load',
               'cm_data_uoa':q}
           if repo!='': ii['cm_repo_uoa']=repo
           r=cm_kernel.access(ii)
           if r['cm_return']>0: return r
               
           d=r['cm_data_obj']['cfg']

           points=d.get('points',[])

           # Go through points; each point is an individual benchmark execution in an entry
           cm_kernel.print_for_con('')
           for p in points:
               bench=p.get('pipeline_output',{}).get('state',{}).get('f53e1f0898493e87',{}).get('output',{}).get('global_source_code_name','')

               rg=cm_kernel.get_value_by_flattened_key({'cm_array':p, 'cm_key':ck})
               if rg['cm_return']>0: return rg
               v=rg['cm_value']
               if v==None: v=''

               fv=float(v)

               cm_kernel.print_for_con(bench+' - '+v)

               # If first time, add list to the stat array
               if bench not in benchmarks:
                  benchmarks[bench]=[]

               benchmarks[bench].append(v)

       # Process statistically
#       print json.dumps(benchmarks, indent=2)       

       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Checking variation ...')
       ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['math.variation'],
           'cm_action':'process',
           'array':benchmarks}
       r=cm_kernel.access(ii)
       if r['cm_return']>0: return r

       stats=r['array']

    return {'cm_return':0, 'stats':stats}

# ============================================================================
def scenario(i):
    """
    Build and run program, save to repository

    Input:  {
              caller_module_uid    - caller module UID
              state                - pipeline state
              exploration_scenario - exploraiton scenario parameters
              summary_point        - summary dimensions describing aggregated point
              aggregated_entry     - all aggregated points
              space                - if aggregated, current space
              original_input       - to get other additional forms per scenario
              cm_data_desc         - data description (from the original form)
              cm_form_explore      - form explore (form the original form)
              number_of_iterations - max number of iterations
            }

    Output: {
              cm_return   - if =0, success
            }
    """

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Not implemented ...')

    return {'cm_return':0}

# ============================================================================
def prepare_input_params(i):
    """
    Prepare input parameters for this scenario

    Input:  {
              cm_form_array - array with forms
              form_exists   - yes or no
              cm_array1     - array from form 1 (coarse-grain points)
              cm_array2     - array from form 2 (additional parameters) just in case
              caller_cfg    - caller module cfg
            }

    Output: {
              cm_return   - if =0, success
            }
    """

    return {'cm_return':0}

# ============================================================================
def web_view_header(i):
    """
    Prepare header

    Input:  {
              header_desc - header description
            }

    Output: {
              cm_return   - if =0, success
              cm_html     - header html
              cm_length   - header length
            }
    """

    # Get web style
    if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
    else:
       return {'cm_return':1, 'cm_error':'web style is not defined'}

    local_pipeline_desc=ini['cfg'].get('pipeline_point_viewer_desc',{})

    x1=''
    if 'table_bgcolor_line5' in web: x1=' bgcolor="'+web['table_bgcolor_line5']+'" ' 
    x2=''
    if 'table_bgcolor_line6' in web: x2=' bgcolor="'+web['table_bgcolor_line6']+'" '

    vsx=ini['cfg']['pipeline_point_viewer']
    vs=vsx.get('main',[])
    ks=vsx.get('key_prefix','')

    desc=i.get('header_desc',{})

    h=''

    col=True; 
    for kx in vs:
        k=ks+kx
        ky='#'+kx
        v=desc.get(ky,{}).get('desc_text','')
        if v=='': v=local_pipeline_desc.get(ky,{}).get('desc_text','')

        if col: col=False ; x=x1
        else:   col=True  ; x=x2
        
        h+='<td align="center"'+x+'><small><b>'+v+':</b></small></td>'

    return {'cm_return':0, 'cm_html':h, 'cm_length':str(len(vs))}

# ============================================================================
def web_view_data(i):
    """
    Prepare header

    Input:  {
              cm_data        - data object from pipeline
              header_desc    - header description
              cur_line_style - { 
                                 cur    - cur (style1 or style2)
                                 style1
                                 style2
                               }
              point          - point to visualize
            }

    Output: {
              cm_return   - if =0, success
              cm_html     - header html
            }
    """

    # Get web style
    if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
    else:
       return {'cm_return':1, 'cm_error':'web style is not defined'}

    desc=i.get('header_desc',{})

    # Current style
    cls=i.get('cur_line_style',{})
    y=cls.get('cur','')
    y1=cls.get('style1','')
    y2=cls.get('style2','')

    # Prepare some vars to change colors for each second line
    x1=''
    if 'table_bgcolor_line3' in web: x1=' bgcolor="'+web['table_bgcolor_line3']+'" ' 
    x2=''
    if 'table_bgcolor_line4' in web: x2=' bgcolor="'+web['table_bgcolor_line4']+'" '
    x3=''
    if 'table_bgcolor_line5' in web: x3=' bgcolor="'+web['table_bgcolor_line5']+'" ' 
    x4=''
    if 'table_bgcolor_line6' in web: x4=' bgcolor="'+web['table_bgcolor_line6']+'" '

    vsx=ini['cfg']['pipeline_point_viewer']
    vs=vsx.get('main',[])
    vsf=vsx.get('fail',[])
    vss=vsx.get('strange',[])
    vsff=vsx.get('full_fail',[])
    vnt=vsx.get('normality_test',[])

    ks=vsx.get('key_prefix','')

    d=i.get('cm_data',{})
    points=d.get('cm_data_obj',{}).get('cfg',{}).get('points',[])

    h=''

    col=True; 
    for kx in vs:

        k=ks+kx

        if col: 
           col=False 
           if y==y1: x=x1
           else: x=x3
        else: 
           col=True 
           if y==y1: x=x2
           else: x=x4

        p=points[int(i.get('point','0'))]
        rg=cm_kernel.get_value_by_flattened_key({'cm_array':p, 'cm_key':k})
        if rg['cm_return']>0: return rg
        v=rg['cm_value']
        if v==None: v=''

        d1=desc.get('#'+kx,{})
        d1['cm_view_as_text']='yes'
        ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
            'cm_action':'convert_field_to_html',
            'cm_value':str(v),
            'cm_key':'',
            'cm_mode':'view',
            'cm_data_desc':d1,
            'cm_disable_coloring':'yes'}
        r1=cm_kernel.access(ii)
        if r1['cm_return']>0: return r1
        vh=r1['cm_value_html']
        vt=r1['cm_value_text']

        h+='<td '+x+' align="right" valign="top"><small>'
        if kx in vsf:
           if v=='' or int(v)!=0:
              h+='<span style="color:#9F0000"><B>Fail ('+str(v)+')</B></span>'
           else:
              h+=str(v)
        elif kx in vss:
           if v=='' or int(v)!=0:
              h+='<span style="color:#9F0000">Strange ('+str(v)+')</span>'
           else:
              h+=str(v)
        elif kx in vsff:
           if v=='yes':
              h+='<span style="color:#9F0000"><b>Fail</b></span>'
           else:
              h+='no'
        elif kx in vnt:
           if v!='' and float(v)<0.05:
              h+='<span style="color:#9F0000"><B>'+str(v)+'</B></span>'
           else:
              h+=str(v)
        else:
           h+=vh
        h+='<BR>\n'
        h+='</small></td>'

    return {'cm_return':0, 'cm_html':h}