cM API Documentation

module.py

00001 #
00002 # Collective Mind
00003 #
00004 # See cM LICENSE.txt for licensing details.
00005 # See cM Copyright.txt for copyright details.
00006 #
00007 # Developer(s): (C) Grigori Fursin, started on 2011.09
00008 #
00009 
00010 # Should always be here
00011 ini={}
00012 cm_kernel=None
00013 
00014 # Local settings
00015 import copy
00016 import time
00017 import json
00018 import random
00019 import os
00020 
00021 # ============================================================================
00022 def init(i):
00023     return {'cm_return':0}
00024 
00025 # ============================================================================
00026 def scenario(i):
00027     """
00028     Build and run program, save to repository
00029 
00030     Input:  {
00031               caller_module_uid       - caller module UID
00032               state                   - pipeline state
00033               exploration_scenario    - exploration scenario parameters
00034                 work_repo_uoa         - repository to save results
00035 
00036               summary_point           - summary dimensions describing aggregated point
00037               aggregated_entry        - all aggregated points
00038               space                   - if aggregated, current space
00039               original_input          - to get other additional forms per scenario
00040               cm_data_desc            - data description (from the original form)
00041               cm_form_explore         - form explore (form the original form)
00042               number_of_iterations    - max number of iterations
00043               mobile_mode             - 'build' or 'process' - mode during crowdsourcing auto-tuning using mobiles
00044               copy_built_executables  - if 'yes', copy all built executables to a_<UID>.out,
00045           state to a_<UID>.json and space to ct_space.json
00046 
00047 
00048               (new_src_uoa)                   - new src UOA in tmp repository
00049               (new_src_path)                  - new src path
00050               (measured_frequency)            - measured frequency (if supported)
00051               (execution_info)                - list of list of execution times per binary - 1 coarse-grain exploration step
00052               (execution_info_min)            - list of minimal execution times per 1 coarse-grain exploration step
00053               (execution_info_ct_repeat_main) - CT_REPEAT_MAIN got during calibration
00054               (built_files)                   - list of unique built files (with UIDs)
00055             }
00056 
00057     Output: {
00058               cm_return   - if =0, success
00059             }
00060     """
00061 
00062     mobile_mode=i.get('mobile_mode','')
00063     cbe=i.get('copy_built_executables','')
00064 
00065     if cbe=='yes':
00066        built_files=[]
00067 
00068     # Prepare mobile report variables
00069     if mobile_mode=='report':
00070       new_src_path=i.get('new_src_path','')
00071       execution_info=i.get('execution_info',[])
00072       execution_info_min=i.get('execution_info_min',[])
00073       built_files=i.get('built_files',[])
00074       ct_repeat_main=i.get('execution_info_ct_repeat_main','')
00075 
00076     # Exploration scenarion parameters
00077     es=i.get('exploration_scenario',{})
00078     sr=es.get('save_results_in_repository','')
00079     aggr=es.get('aggregate_in_one_entry','')
00080     wrepo=es.get('work_repo_uoa','')
00081     pmd=i.get('pipeline_module_desc',{})
00082 
00083     desc=i.get('cm_data_desc',{})
00084     desc_copy=copy.deepcopy(desc) # description will be dynamically changing
00085                                   # this is the original one
00086 
00087     state_orig=i.get('state',{})
00088     state=copy.deepcopy(state_orig) # not to mix up original state
00089                                     # otherwise some strange aggregations of datasets, flags, etc
00090     ae=i.get('aggregated_entry',{})
00091 
00092     # Load aggregated entry if found ...
00093     dae={}
00094     if aggr=='yes' and len(ae)>0:
00095        r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
00096                            'cm_action':'load',
00097                            'cm_data_uoa':ae['cm_data_uoa'],
00098                            'cm_repo_uoa':wrepo})
00099        if r['cm_return']>0:
00100           if r['cm_return']!=16: return r
00101        else:
00102           dae=r['cm_data_obj']['cfg']
00103 
00104     # Checked solutions during this exploration
00105     checked_solutions=dae.get('checked_solutions',[])
00106     checked_solutions_ref=dae.get('checked_solutions_ref', [])
00107 
00108     space=i.get('space',{})
00109 
00110     fe=i.get('cm_form_explore',{})
00111     fe_copy=copy.deepcopy(fe)
00112 
00113     # Check additional scenario parameters
00114     ioi=i.get('original_input',{})
00115 
00116     frm1='#form_'+ini['cm_module_uid']+'_1'
00117     frm2='#form_'+ini['cm_module_uid']+'_2'
00118 
00119     a1={}
00120     r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
00121                         'cm_action':'detect_form_params',
00122                         'cm_array':ioi, 
00123                         'cm_prefix':frm1})
00124     if r['cm_return']>0: return r
00125     cm_form_array1=r['cm_array']
00126 
00127     r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1, 
00128                                          'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
00129     if r['cm_return']>0: return r
00130     a1=r['cm_array']
00131 
00132     prm=a1.get('profitable_results_module','')
00133 
00134     # Detect data in the forms
00135     a2={}
00136     r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
00137                         'cm_action':'detect_form_params',
00138                         'cm_array':ioi, 
00139                         'cm_prefix':frm2})
00140     if r['cm_return']>0: return r
00141     cm_form_array2=r['cm_array']
00142 
00143     r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array2, 
00144                                          'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
00145     if r['cm_return']>0: return r
00146     a2=r['cm_array']
00147 
00148     # Get pipeline module and description
00149     mu=i['caller_module_uid']
00150     r=cm_kernel.get_data_description({'cm_module_uoa':mu})
00151     if r['cm_return']>0: return r
00152     pu=r['cm_data_obj']['cfg']['cm_p_uids']
00153 
00154 
00155     # Get target OS
00156     tos_uoa=state[pu['init']]['input']['target_os_uoa']
00157     r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['os'],
00158                         'cm_action':'load',
00159                         'cm_data_uoa':tos_uoa})
00160     if r['cm_return']>0: return r
00161     ctos=r['cm_data_obj']['cfg']
00162     
00163     # Get tuning objective
00164     to=state.get(pu['init'], {}).get('input',{}).get('tuning_objective','')
00165 
00166     # Prepare description of flags
00167     kp='##'+pu['prepare_compiler_flags']+'#input#all_compiler_flags#' # compiler description prefix in descriptions
00168     r=cm_kernel.find_keys_by_prefix({'cm_array':desc,
00169                                      'cm_prefix':kp,
00170                                      'cm_sort_key_int':'sort_index'})
00171     if r['cm_return']>0: return r
00172     keys=r['cm_keys']
00173 
00174     # Check if already have a point and take repeat from it
00175     repeat=''
00176     frequency=''
00177     past_frequency=''
00178     if len(space.get('points',[]))>0:
00179        sp=space['points'][0].get('pipeline_output',{}).get('state',{})
00180        repeat=sp.get(pu['run_program'], {}).get('input',{}).get('run_vars',{}).get('CT_REPEAT_MAIN','')
00181        frequency=sp.get(pu['get_system_state'],{}).get('output',{}).get('system_state',{}).get('properties',{}).get('cpu_frequency','')
00182        past_frequency=sp.get(pu['get_system_state'],{}).get('output',{}).get('system_state',{}).get('properties',{}).get('cpu_frequency','')
00183 
00184        # Check if mobile submit is not matching during reporting
00185        if mobile_mode=='report' and repeat!='' and int(repeat)>1 and int(ct_repeat_main)>1:
00186           if ct_repeat_main!=repeat:
00187              return {'cm_return':1, 'cm_error':'CT_REPEAT_MAIN is not matching with the existing one ('+repeat+' vs '+ct_repeat_main+' !'}
00188     else:
00189        space['points']=[]
00190 
00191     if a1.get('force_repeat_number','')!='':
00192        repeat=a1['force_repeat_number']
00193     elif a1.get('repeat_from_entry_uoa','')!='':
00194        kk={}
00195        kk['cm_repo_uoa']=wrepo
00196        kk['cm_run_module_uoa']=ae['cm_module_uoa']
00197        kk['cm_data_uoa']=a1['repeat_from_entry_uoa']
00198        kk['cm_action']='load'
00199        kk['lock_acquire']='yes'
00200        r=cm_kernel.access(kk)
00201        if r['cm_return']>0: return r
00202        else:
00203           d=r['cm_data_obj']['cfg']
00204           kkx=d['points']
00205           if len(kkx)>0:
00206              repeat1=kkx[0].get('pipeline_output',{}).get('state',{}).get(pu['run_program'], {}).get('input',{}).get('run_vars',{}).get('CT_REPEAT_MAIN','')
00207              if repeat1!='':
00208                 repeat=repeat1
00209 
00210     # Start exploration
00211     cm_explore_state=[]
00212     iter=0
00213     start_time=time.time()
00214 
00215     nis=i.get('number_of_iterations','')
00216     ni=0
00217     if nis!='': ni=int(nis)
00218 
00219     finish=False
00220 
00221     # Load ctuning compiler description
00222     ccu=state.get(pu['init'], {}).get('input',{}).get('ctuning_compiler_uoa','')
00223     r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.compiler'],
00224                         'cm_action':'load',
00225                         'cm_data_uoa':ccu})
00226     if r['cm_return']>0: return r
00227     dccu=r['cm_data_obj']['cfg']
00228 
00229     # Load ctuning compiler optimizations (classes) description
00230     opt_classes=[]
00231     ccu=state.get(pu['init'], {}).get('input',{}).get('ctuning_compiler_uoa','')
00232     r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.compiler.optimizations'],
00233                         'cm_action':'load',
00234                         'cm_data_uoa':ccu,
00235                         'cm_repo_uoa':wrepo})
00236     if r['cm_return']==0: 
00237        dccou=r['cm_data_obj']['cfg']
00238        opt_classes=dccou.get('opt_classes',{}).get(to,[])
00239 
00240     # Prepare base classes (first human approximation)
00241     iter_ref=0
00242     start_ref=False
00243 
00244     kbf=kp+'base_flag'
00245     ref_base=desc.get(kbf,{}).get('default_value','')
00246     ref_choice=desc.get(kbf,{}).get('choice',[])
00247  
00248     # rebuild choices starting from default
00249     rc=[]
00250     if ref_base=='' or ref_base not in ref_choice:
00251        rc=ref_choice
00252     else:
00253        rc.append(ref_base)
00254        if a1.get('check_all_base_flags_first','')=='yes':
00255           for q in ref_choice:
00256               if q!=ref_base: rc.append(q)
00257 
00258     # Check if validation mode
00259     validate=False
00260     validate_success=False
00261     if len(space['points'])>0:
00262        validate=True
00263 
00264     # Set seed if needed
00265     if a1.get('random_seed','')!='':
00266        cm_kernel.print_for_con('')
00267        cm_kernel.print_for_con('Setting random seed to '+str(a1['random_seed']))
00268        cm_kernel.print_for_con('')
00269        random.seed(a1['random_seed'])
00270 
00271     recorded=False # Keep last recorded solution (for mobile mode)
00272     validated=True # Record if validation failes (for mobile mode)
00273 
00274     # Start exploration
00275     while not finish:
00276        iter+=1
00277        if ni>0 and iter>ni: break
00278        recorded=False
00279 
00280        # Explore fine-grain choices
00281        cm_kernel.print_for_con('************************************************************************************')
00282        x=''
00283        if ni>0: x=' out of '+str(ni)
00284        cm_kernel.print_for_con('Fine-grain iteration:    '+str(iter)+x)
00285 
00286        t=time.time()-start_time
00287        ts="%.1f" % t
00288        cm_kernel.print_for_con('Time elapsed: '+ts+' sec.')
00289 
00290        time.sleep(2)
00291 
00292        desc=copy.deepcopy(desc_copy)
00293        opt_class={}
00294 
00295        force_keep_point=False
00296        unexplored=False
00297        opt_class_uid=''
00298 
00299        # Restore original state and description
00300        if mobile_mode=='report':
00301           # If mobile report ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
00302           xiter=iter-1
00303 
00304           cm_kernel.print_for_con('')
00305           cm_kernel.print_for_con('Preloading state from the stored file '+built_files[xiter]['name']+' ...')
00306 
00307           r4=cm_kernel.load_json_file({'cm_filename':os.path.join(new_src_path, built_files[xiter]['name'])+'.before.json'})
00308           if r4['cm_return']>0: return r4
00309           state=r4['cm_array']
00310 
00311           state[pu['run_program']]['input']['calibrate']='no'
00312           state[pu['run_program']]['input']['run_vars']['CT_REPEAT_MAIN']=ct_repeat_main
00313 
00314           ii={}
00315           ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['ctuning.pipeline.build_and_run_program']
00316           ii['cm_action']='run_stat'
00317           ii['state']=state
00318           ii_copy=copy.deepcopy(ii)
00319 
00320           r4=cm_kernel.load_json_file({'cm_filename':os.path.join(new_src_path, built_files[xiter]['name'])+'.after.json'})
00321           if r4['cm_return']>0: return r4
00322           statex=r4['cm_array']
00323 
00324           statex[pu['run_program']]['output']['exit_code']='0'
00325           eim=execution_info_min[xiter]
00326           statex[pu['run_program']]['output']['run_time_by_module']=execution_info_min[xiter]
00327 
00328           feim=float(eim)
00329           rpt=int(ct_repeat_main)
00330           
00331           statex[pu['run_program']]['output']['global_execution_time_div_by_repeat']=str(feim/rpt)
00332           statex[pu['run_program']]['input']['run_vars']['CT_REPEAT_MAIN']=ct_repeat_main
00333           statex[pu['check_output']]['output']['output_correct']='0'
00334 
00335           statex[pu['get_system_state']]={'output':{'system_state':{'properties':{'cpu_frequency':i.get('measured_frequency','')}}}}
00336 
00337           # Ref execution
00338           if xiter==0:
00339              force_keep_point=True
00340 
00341           mstat=a1.get('statistics_module','')
00342           if mstat!='':
00343              rx=cm_kernel.access({'cm_run_module_uoa':mstat,
00344                                   'cm_action':'process',
00345                                   'state':statex, 
00346                                   'desc':desc})
00347              if rx['cm_return']>0: return rx
00348              statex=rx['state']
00349 
00350           rp={'cm_return':0, 'fail':False}
00351           rp['state']=copy.deepcopy(statex)
00352 
00353        else:
00354           state=copy.deepcopy(state_orig)
00355 
00356           if repeat!='':
00357              state[pu['run_program']]['input']['calibrate']='no'
00358              if 'run_vars' not in state[pu['run_program']]['input']: 
00359                 state[pu['run_program']]['input']['run_vars']={}
00360              state[pu['run_program']]['input']['run_vars']['CT_REPEAT_MAIN']=repeat
00361 
00362           # Check if need to check output
00363           if a1.get('check_output','')=='yes':
00364              state[pu['check_output']]['enable']='yes'
00365 
00366           cm_kernel.print_for_con('')
00367           cm_kernel.print_for_con('Preparing compiler flags ...')
00368 
00369           # If classes (combinations of flags) already exist in the compiler description,
00370           #  check if we evaluated them all and if yes, start generating new random flags
00371           #  otherwise use base flag and use as soon reference point + calibrate
00372 
00373           # For now, order is always fixed
00374           keys_order=sorted(keys, key=lambda k: (int(desc.get(k,{}).get('sort_index','999999')), desc.get(k,{}).get('desc_text','').lower()))
00375           keys_order1=[]
00376           for q in keys_order:
00377               q1=q
00378               z=q.rfind('#')
00379               if z>=0: q1=q[z+1:]
00380               keys_order1.append(q1)
00381           state[pu['prepare_compiler_flags']]['input']['all_compiler_flags_order']=keys_order1
00382 
00383           # We need to enable all flags - this option was included not to mix up with library building ...
00384           state[pu['prepare_compiler_flags']]['input']['use_only_base_flag']='no'
00385 
00386           found_ref=False
00387           if (start_ref and iter_ref<len(rc)) or len(space['points'])==0 or validate:
00388              if not start_ref and not validate: 
00389                 start_ref=True
00390              # Check base run and calibrarate  ***********************************************************************************
00391              # (or compare with the recorded one if exists, and take calibration from there)
00392              # then check current coarse-grain human heuristics (reference flags -O2,-O1,-Os, etc)
00393 
00394              while iter_ref<len(rc):
00395                 o=rc[iter_ref]
00396                 iter_ref+=1
00397 
00398                 if o==ref_base or o not in checked_solutions_ref:
00399                    found_ref=True
00400                    force_keep_point=True
00401 
00402                    checked_solutions_ref.append(o)
00403 
00404                    state[pu['prepare_compiler_flags']]['input']['all_compiler_flags']['base_flag']=o
00405 
00406                    # disable all flags
00407                    for q in keys:
00408                        if q in fe:
00409                           fe[q]['type']='disabled'
00410 
00411                    # set base flag
00412                    fe[kbf]['type']='fixed'
00413 
00414                    break
00415 
00416           if not found_ref:
00417              # Check if there are some collective solutions that were not explored
00418              found=False
00419 
00420              if a1.get('check_existing_collective_solutions','')=='yes':
00421                 for q in opt_classes:
00422                     if q['cm_uid'] not in checked_solutions:
00423                        found=True
00424                        break
00425 
00426              if found:
00427                 cm_kernel.print_for_con('')
00428                 cm_kernel.print_for_con('Found unexplored in this session collective solution (out of '+str(len(opt_classes))+')')
00429                 cm_kernel.print_for_con('  Use it ('+q['cm_uid']+')')
00430                 
00431                 opt_class=copy.deepcopy(q)
00432 
00433                 for u in keys:
00434                     fe[u]['type']='fixed'
00435                     v=q.get('flags',{}).get(u,{}).get('value')
00436                     if v!=None:
00437                        rx=cm_kernel.set_value_by_flattened_key({'cm_array':state, 'cm_key':u, 'cm_value':v})
00438                        if rx['cm_return']>0: return rx
00439 
00440                 checked_solutions.append(q['cm_uid'])
00441 
00442                 opt_class_uid=q['cm_uid']
00443              else:
00444                 if a1.get('generate_random_solution','')!='yes':
00445                    cm_kernel.print_for_con('')
00446                    cm_kernel.print_for_con('All solutions have been explored - finishing!')
00447 
00448                    finish=True
00449                    break
00450 
00451                 cm_kernel.print_for_con('')
00452                 cm_kernel.print_for_con('Generating new random solution ...')
00453 
00454                 unexplored=True
00455 
00456                 r=cm_kernel.gen_uid({})
00457                 if r['cm_return']>0: return r
00458                 uid=r['cm_uid']
00459 
00460                 opt_class['cm_uid']=uid
00461                 opt_class['flags']={}
00462                 opt_class['keys_order']=keys_order
00463                 opt_class_uid=uid
00464 
00465                 for q in keys:
00466                     if q in fe: # to avoid special flags such as openmp
00467                        if q!=kbf: fe[q]['type']='random'
00468                        else:      fe[q]['type']=a1.get('tuning_base_flag','random')
00469 
00470                        opt_class['flags'][q]={}
00471                        opt_class['flags'][q]['type']=fe[q]['type']
00472 
00473                 # set default base flag (combine best flag with turning on and off flags for online clustering)
00474 
00475           state[pu['prepare_compiler_flags']]['input']['opt_class_uid']=opt_class_uid
00476 
00477           # Set choices
00478           jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-choice'],
00479               'cm_action':'prepare_choices',
00480               'cm_array':state,
00481               'cm_data_desc':desc,
00482               'cm_explore':fe,
00483               'cm_explore_state':[],
00484               'cm_explore_level':'1'}
00485           r=cm_kernel.access(jj)
00486           if r['cm_return']>0: return r
00487 
00488           cm_explore_state=r['cm_explore_state']
00489 
00490           # Run pipeline
00491           ii={}
00492           ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['ctuning.pipeline.build_and_run_program']
00493           ii['cm_action']='run_stat'
00494           ii['state']=state
00495           ii_copy=copy.deepcopy(ii)
00496           # we may move next 2 after deepcopy to avoid creating too big data ...
00497           ii['desc']=desc # needed for stats ...
00498           ii['additional_params']=a1
00499 
00500           if mobile_mode=='build':
00501              # don't run code
00502              state[pu['run_program']]['enable']='no'
00503 
00504           fcbe=''
00505           if cbe!='': 
00506              rx=cm_kernel.gen_uid({})
00507              if rx['cm_return']>0: return rx
00508              fcbe='a-'+rx['cm_uid'] #+ctos['file_extensions']['exe'] # FGG: I decided not to add it here, since universal cleaning script removes it ...
00509 
00510              state[pu['build_program']]['input']['copy_target_file_to']=fcbe
00511 
00512           rp=cm_kernel.access(ii)
00513           if rp['cm_return']>0: return rp
00514 
00515        statex=rp['state']
00516        failx=rp['fail']
00517 
00518        # FGG: TBD -> checking changes in frequency
00519        frequency=statex.get(pu['get_system_state'],{}).get('output',{}).get('system_state',{}).get('properties',{}).get('cpu_frequency','')
00520 
00521        if cbe!='': 
00522           fnx=state[pu['build_program']]['output'].get('target_file_size','0')
00523           if fnx.startswith('./'):
00524              fnx=fnx[2:]
00525           built_files.append({'name':fcbe, \
00526                               'size':fnx,
00527                               'prepared_compiler_flags':state[pu['prepare_compiler_flags']]['output'].get('prepared_compiler_flags','')})
00528 
00529           r4=cm_kernel.save_array_to_file_as_json({'cm_filename':fcbe+'.before.json', 'cm_array':ii_copy['state']})
00530           if r4['cm_return']>0: return r4
00531 
00532           r4=cm_kernel.save_array_to_file_as_json({'cm_filename':fcbe+'.after.json', 'cm_array':statex})
00533           if r4['cm_return']>0: return r4
00534 
00535        if repeat=='':
00536           repeat=statex.get(pu['run_program'], {}).get('input',{}).get('run_vars',{}).get('CT_REPEAT_MAIN','')
00537 
00538        fcbe=''
00539        if cbe!='': 
00540           rx=cm_kernel.gen_uid({})
00541           if rx['cm_return']>0: return rx
00542           fcbe='a-'+rx['cm_uid']
00543 
00544           state[pu['build_program']]['input']['copy_target_file_to']=fcbe
00545 
00546        # Lock UID for space
00547        lock_uid=''
00548 
00549        # If found better solution after pruning
00550        found_better_solution=False
00551  
00552        if not failx:
00553           tc=[]
00554 
00555           if to!='':
00556              tc=pmd.get('tuning_objectives',{}).get(to,{}).get('characteristics',[])
00557 
00558           # Aggregate global statistics (min/max/all data) - useful for violin graphs to see density
00559           mstat=a1.get('statistics_module','')
00560           if mstat!='':
00561              rx=cm_kernel.access({'cm_run_module_uoa':mstat,
00562                                   'cm_action':'process',
00563                                   'state':statex, 
00564                                   'state1':space,
00565                                   'desc':desc})
00566              if rx['cm_return']>0: return rx
00567              space=rx['state1']
00568 
00569           # FGG: Validate results (we can in the future validate N random points
00570           #                   and use it as characterization by reaction to optimizations)
00571           #   Or if result is different, possibly different frequency, memory, etc
00572 
00573           if validate and mobile_mode!='build':
00574              cm_kernel.print_for_con('')
00575              cm_kernel.print_for_con('Validating reference run ...')
00576 
00577              etmm=statex.get(pu['run_program'], {}).get('output',{}).get('run_time_by_module','')
00578 
00579              if etmm!='':
00580                 # Search for reference run in past points
00581                 for p in space['points']:
00582                     opt=p.get('pipeline_output',{}).get('state',{}).get(pu['prepare_compiler_flags'], {}).get('output',{}).get('prepared_compiler_flags','')
00583                     if opt.strip()==ref_base:
00584                        etmm1=p.get('pipeline_output',{}).get('state',{}).get('cm_stat_characteristics',{}).get(pu['run_program'],{}).get('output',{}).get('run_time_by_module',{}).get('min','')
00585                        if etmm1!='':
00586                           d=abs((float(etmm)-float(etmm1)))/float(etmm1)
00587 
00588                           cm_kernel.print_for_con('')
00589                           cm_kernel.print_for_con('Current execution time:  '+etmm)
00590                           cm_kernel.print_for_con('Recorded execution time: '+etmm1)
00591                           cm_kernel.print_for_con('Percent difference:      '+("%.2f" % (d*100)))
00592 
00593                           if d<0.04:
00594                              validate_success=True
00595                        break      
00596 
00597           else:
00598              # Online learning/pruning of profitable results
00599              must_add=True
00600 
00601              # Reload space and lock it
00602              if sr=='yes' and len(ae)>0:
00603                 cm_kernel.print_for_con('')
00604                 cm_kernel.print_for_con('Reloading and locking space ...')
00605 
00606                 kk={}
00607                 kk['cm_repo_uoa']=wrepo
00608                 kk['cm_run_module_uoa']=ae['cm_module_uoa']
00609                 kk['cm_data_uoa']=ae['cm_data_uoa']
00610                 kk['cm_action']='load'
00611                 kk['lock_acquire']='yes'
00612                 r=cm_kernel.access(kk)
00613                 if r['cm_return']>0: 
00614                    if r['cm_return']!=16: return r
00615                 else:
00616                    d=r['cm_data_obj']['cfg']
00617                    space['points']=d['points']
00618                    lock_uid=r['lock_uid']
00619 
00620              if prm!='' and not force_keep_point:
00621                 rx=cm_kernel.access({'cm_run_module_uoa':prm,
00622                                      'cm_action':'process',
00623                                      'space':space,
00624                                      'pipeline_input':ii_copy,
00625                                      'pipeline_output':rp,
00626                                      'desc':desc,
00627                                      'tune_characteristics':tc,
00628                                      'calcluate_improvement_over_ref':'global_improvements_over_ref',
00629                                      'max_improvement_threshold':'1.09'})
00630                 if rx['cm_return']>0: return rx
00631 
00632                 space=rx['space']
00633                 must_add=rx['must_add']
00634 
00635                 found_better_solution=must_add
00636 
00637              # Check if after pruning, new solution was found and needs to be saved
00638              if must_add:
00639                 # save real flags
00640                 if len(opt_class)>0:
00641                    for q in keys:
00642                        if q in opt_class['flags']: # to avoid problems with special flags such as openmp
00643                           rg=cm_kernel.get_value_by_flattened_key({'cm_array':state, 'cm_key':q})
00644                           if rg['cm_return']>0: return rg
00645                           v=rg['cm_value']
00646                           opt_class['flags'][q]['value']=v
00647 
00648                 cm_kernel.print_for_con('')
00649                 cm_kernel.print_for_con('Trying to add point to solutions ...')
00650 
00651                 ix={'pipeline_input': ii_copy,
00652                     'pipeline_output': rp}
00653 
00654                 if force_keep_point:
00655                    ix['always_keep_this_point']='yes'
00656 
00657                 if len(opt_class)!=0:
00658    #                ix['opt_class']=opt_class
00659 
00660                    cm_kernel.print_for_con('')
00661                    cm_kernel.print_for_con('Reloading and locking compiler optimization classes ...')
00662 
00663                    # Reload compiler classes and lock them
00664                    xfound=False
00665                    dccou={}
00666                    opt_classes=[]
00667                    lock_uid1=''
00668                    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.compiler.optimizations'],
00669                                        'cm_action':'load',
00670                                        'lock_acquire':'yes',
00671                                        'cm_data_uoa':ccu,
00672                                        'cm_repo_uoa':wrepo})
00673                    if r['cm_return']==0:
00674                       dccou=r['cm_data_obj']['cfg']
00675                       opt_classes=dccou.get('opt_classes',{}).get(to,[])
00676                       xfound=True
00677                       lock_uid1=r['lock_uid']
00678 
00679                    # Add to common compiler class if not there
00680                    if unexplored:
00681                       if opt_class_uid!='': checked_solutions.append(opt_class_uid)
00682 
00683                       if a1.get('classify_random_solutions','')=='yes':
00684                          found=False
00685                          for q in opt_classes:
00686                              if opt_class['cm_uid']==q['cm_uid']:
00687                                 found=True
00688                                 break
00689 
00690                          if not found:
00691                             pcf=state[pu['prepare_compiler_flags']].get('output',{}).get('prepared_compiler_flags','')
00692                             if pcf!='':
00693                                opt_class['prepared_compiler_flags']=pcf
00694 
00695                             cm_kernel.print_for_con('Classifying solution and adding to compiler description ...')
00696                             opt_classes.append(opt_class)
00697 
00698                             dccou['opt_classes']={}
00699                             dccou['opt_classes'][to]=opt_classes
00700 
00701                             ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.compiler.optimizations'],
00702                                 'cm_action':'add',
00703                                 'cm_array':dccou,
00704                                 'cm_data_uoa':ccu,
00705                                 'cm_repo_uoa':wrepo}
00706                             if xfound: 
00707                                ii['cm_action']='update'
00708                                ii['lock_uid']=lock_uid1
00709                             ii['cm_array_update']={'cm_access_control':{'comments_groups':'registered', 'read_groups':'registered', 'write_groups':'registered'}}
00710 
00711                             r=cm_kernel.access(ii)
00712                             if r['cm_return']>0: return r
00713                             lock_uid1=''
00714 
00715                    # If still locked (was not updated), unlock
00716                    if lock_uid1!='':
00717                       r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.compiler.optimizations'],
00718                                           'cm_action':'load',
00719                                           'unlock_uid':lock_uid1,
00720                                           'cm_data_uoa':ccu,
00721                                           'cm_repo_uoa':wrepo})
00722                       if r['cm_return']>0: return r
00723 
00724                 space['points'].append(ix)
00725        else:
00726           if a1.get('record_failed_cases','')=='yes':
00727              cm_kernel.print_for_con('Adding failed point ...')
00728 
00729              ix={'pipeline_input': ii_copy,
00730                  'pipeline_output': rp}
00731 
00732              space['points'].append(ix)
00733           else:
00734              cm_kernel.print_for_con('Skipping failed point ...')
00735 
00736           # If reference run, exit
00737           if iter==1:
00738              finish=True
00739 
00740        # Check if validation failed!
00741        if validate and mobile_mode!='build': 
00742           validate=False
00743           
00744           if not validate_success:
00745              cm_kernel.print_for_con('')
00746              cm_kernel.print_for_con('Validation failed ...')
00747 
00748              validated=False
00749 
00750              finish=True
00751              break
00752              
00753 ##save_only_best_results_in_repo
00754        # Record in repository
00755        if sr=='yes' and \
00756           (a1.get('save_only_best_results_in_repo','')!='yes' or \
00757           (a1.get('save_only_best_results_in_repo','')=='yes' and found_better_solution)):
00758           recorded=True
00759           cm_kernel.print_for_con('')
00760           cm_kernel.print_for_con('Recording point in repository ...')
00761 
00762           diter=int(space.get('number_of_explored_points','0'))
00763           space['number_of_explored_points']=str(diter+1)
00764 
00765           jj={'cm_run_module_uoa':ini['cm_module_uid'],
00766               'cm_action':'add',
00767               'cm_array':space}
00768 
00769           if aggr=='yes' and len(ae)>0:
00770              cm_kernel.print_for_con('')
00771              cm_kernel.print_for_con('Recording point in aggregated entry '+ae['cm_data_uoa']+' ...')
00772              jj['cm_action']='update'
00773              jj.update(ae)
00774              if lock_uid!='':
00775                 jj['lock_uid']=lock_uid # to release lock
00776 
00777           jj['cm_repo_uoa']=wrepo
00778           jj['cm_array_update']={'cm_access_control':{'comments_groups':'registered', 'read_groups':'registered', 'write_groups':'registered'}}
00779           jj_copy=copy.deepcopy(jj)
00780 
00781           r=cm_kernel.access(jj)
00782           if r['cm_return']>0: return r
00783 
00784           if aggr=='yes' and len(ae)==0:
00785              ae['cm_data_uid']=r['cm_uid']
00786              ae['cm_data_uoa']=r['cm_uoa']
00787              ae['cm_module_uoa']=ini['cm_module_uid']
00788              ae['cm_repo_uoa']=wrepo
00789        else:
00790           cm_kernel.print_for_con('')
00791           cm_kernel.print_for_con('Skipping recording of point in repository ...')
00792 
00793           # If still locked (was not updated), unlock
00794           if lock_uid!='':
00795 
00796              cm_kernel.print_for_con('')
00797              cm_kernel.print_for_con('Unlocking point in repository ...')
00798 
00799              r=cm_kernel.access({'cm_run_module_uoa':ae['cm_module_uoa'],
00800                                  'cm_action':'load',
00801                                  'unlock_uid':lock_uid,
00802                                  'cm_data_uoa':ae['cm_data_uoa'],
00803                                  'cm_repo_uoa':wrepo})
00804              if r['cm_return']>0: return r
00805 
00806     # Update explored statistics
00807     cm_kernel.print_for_con('')
00808     cm_kernel.print_for_con('Updating statistics about number of total iterations ...')
00809 
00810     su=cm_kernel.ini['dcfg'].get('cm_internal_statistics_uoa','')
00811     if su=='':
00812        return {'cm_return':1, 'cm_error':'cm_internal_statistics_uoa is not set in kernel configuration'}
00813 
00814     # Load statistics data with lock
00815     lock_uid=''
00816     d={}
00817     diter=0
00818     ii={}
00819     ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-internal']
00820     ii['cm_data_uoa']=su
00821     ii['cm_repo_uoa']=wrepo
00822 
00823     ii_copy=copy.deepcopy(ii)
00824 
00825     ii['cm_action']='load'
00826     ii['lock_acquire']='yes'
00827 
00828     r=cm_kernel.access(ii)
00829     if r['cm_return']==0: 
00830        d=r['cm_data_obj']['cfg']
00831        lock_uid=r['lock_uid']
00832        diter=int(d.get('number_of_latest_explored_points', '0'))
00833 
00834     diter+=iter
00835 
00836     d['number_of_latest_explored_points']=str(diter)
00837 
00838     ii=ii_copy
00839     ii['cm_array']=d
00840     ii['cm_action']='update'
00841     ii['cm_admin']='yes'
00842     if lock_uid!='':
00843        ii['lock_uid']=lock_uid
00844        ii['cm_action']='update'
00845        ii['cm_skip_update_info']='yes'
00846 
00847     r=cm_kernel.access(ii)
00848     if r['cm_return']>0: return r
00849 
00850     rr={'cm_return':0}
00851 
00852     if recorded: rr['recorded']='yes'
00853     if not validated : rr['validated']='no'
00854 
00855     if cbe!='': 
00856        rr['built_files']=built_files
00857 
00858        init=state[pu['init']]['input']
00859        if len(init)>0:
00860           rr['init']=init
00861 
00862        ip=state[pu['init_params']]['output']
00863        if len(ip)>0:
00864           rr['init_parameters']=ip
00865 
00866        rp=state[pu['run_program']].get('input',{})
00867        if len(rp)>0:
00868           rr['run_program']=rp
00869 
00870        pd=state[pu['prepare_dataset']].get('output',{}).get('run_time',{})
00871        if len(pd)>0:
00872           rr['prepare_dataset']=pd
00873 
00874     return rr
00875 
00876 # ============================================================================
00877 def prepare_input_params(i):
00878     """
00879     Prepare input parameters for this scenario
00880 
00881     Input:  {
00882               cm_form_array - array with forms
00883               form_exists   - yes or no
00884               cm_array1     - array from form 1 (coarse-grain points)
00885               cm_array2     - array from form 2 (additional parameters) just in case
00886               caller_cfg    - caller module cfg
00887             }
00888 
00889     Output: {
00890               cm_return   - if =0, success
00891             }
00892     """
00893 
00894     # Get array with forms
00895     af=i.get('cm_form_array',{})
00896     frm1='#form_'+ini['cm_module_uid']+'_1'
00897     frm2='#form_'+ini['cm_module_uid']+'_2'
00898     form_exists=i.get('form_exists','')
00899     xa1=i.get('cm_array1',{})
00900     xa2=i.get('cm_array2',{})
00901     cc=i.get('caller_cfg',{})
00902 
00903     # Detect data in the forms
00904     a1={}
00905     r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
00906                         'cm_action':'detect_form_params',
00907                         'cm_array':af, 
00908                         'cm_prefix':frm1})
00909     if r['cm_return']>0: return r
00910     cm_form_array1=r['cm_array']
00911     cm_form_commands1=r['cm_commands']
00912     cm_form_explore1=r['cm_explore']
00913     cm_data_desc1=ini['cfg']['scenario_desc']
00914 
00915     # Detect data in the forms
00916     a2={}
00917     r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
00918                         'cm_action':'detect_form_params',
00919                         'cm_array':af, 
00920                         'cm_prefix':frm2})
00921     if r['cm_return']>0: return r
00922     cm_form_array2=r['cm_array']
00923     cm_form_commands2=r['cm_commands']
00924     cm_form_explore2=r['cm_explore']
00925 
00926     if form_exists=='yes':
00927        r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1, 
00928                                             'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
00929        if r['cm_return']>0: return r
00930        a1=r['cm_array']
00931 
00932        r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array2, 
00933                                             'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
00934        if r['cm_return']>0: return r
00935        a2=r['cm_array']
00936 
00937     ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
00938         'cm_action':'visualize_data',
00939         'cm_array':a1,
00940         'cm_data_desc':cm_data_desc1,
00941         'cm_form_commands':cm_form_commands1,
00942         'cm_form_explore':cm_form_explore1,
00943         'cm_separator':'#',
00944         'cm_separator_form':frm1+'#',
00945         'cm_forms_exists':form_exists,
00946         'cm_support_raw_edit':'no',
00947         'cm_mode':'add',
00948         'hide_add_new_object':'yes',
00949         'cm_explore':'yes'}
00950     r=cm_kernel.access(ii)
00951     if r['cm_return']>0: return r
00952     cm_kernel.print_for_web(r['cm_string'])
00953 
00954     return {'cm_return':0}
00955 
00956 # ============================================================================
00957 def web_view_header(i):
00958     """
00959     Prepare header
00960 
00961     Input:  {
00962               header_desc - header description
00963             }
00964 
00965     Output: {
00966               cm_return   - if =0, success
00967               cm_html     - header html
00968               cm_length   - header length
00969             }
00970     """
00971 
00972     # Get web style
00973     if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
00974     else:
00975        return {'cm_return':1, 'cm_error':'web style is not defined'}
00976 
00977     local_pipeline_desc=ini['cfg'].get('pipeline_point_viewer_desc',{})
00978 
00979     x1=''
00980     if 'table_bgcolor_line5' in web: x1=' bgcolor="'+web['table_bgcolor_line5']+'" ' 
00981     x2=''
00982     if 'table_bgcolor_line6' in web: x2=' bgcolor="'+web['table_bgcolor_line6']+'" '
00983 
00984     vsx=ini['cfg']['pipeline_point_viewer']
00985     vs=vsx.get('main',[])
00986     ks=vsx.get('key_prefix','')
00987 
00988     desc=i.get('header_desc',{})
00989 
00990     h=''
00991 
00992     col=True; 
00993     for kx in vs:
00994         k=ks+kx
00995         ky='#'+kx
00996         v=desc.get(ky,{}).get('desc_text','')
00997         if v=='': v=local_pipeline_desc.get(ky,{}).get('desc_text','')
00998 
00999         if col: col=False ; x=x1
01000         else:   col=True  ; x=x2
01001         
01002         h+='<td align="center"'+x+'><small><b>'+v+':</b></small></td>'
01003 
01004     return {'cm_return':0, 'cm_html':h, 'cm_length':str(len(vs))}
01005 
01006 # ============================================================================
01007 def web_view_data(i):
01008     """
01009     Prepare header
01010 
01011     Input:  {
01012               cm_data        - data object from pipeline
01013               header_desc    - header description
01014               cur_line_style - { 
01015                                  cur    - cur (style1 or style2)
01016                                  style1
01017                                  style2
01018                                }
01019               point          - point to visualize
01020             }
01021 
01022     Output: {
01023               cm_return          - if =0, success
01024               cm_html            - header html
01025 
01026               point              - all values for the point
01027               point_improvements - all available improvements for the point
01028             }
01029     """
01030 
01031     rr={'cm_return':0}
01032 
01033     point={}
01034     ipoint={} # improvements
01035 
01036     # Get web style
01037     if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
01038     else:
01039        return {'cm_return':1, 'cm_error':'web style is not defined'}
01040 
01041     desc=i.get('header_desc',{})
01042 
01043     # Current style
01044     cls=i.get('cur_line_style',{})
01045     y=cls.get('cur','')
01046     y1=cls.get('style1','')
01047     y2=cls.get('style2','')
01048 
01049     # Prepare some vars to change colors for each second line
01050     x1=''
01051     if 'table_bgcolor_line3' in web: x1=' bgcolor="'+web['table_bgcolor_line3']+'" ' 
01052     x2=''
01053     if 'table_bgcolor_line4' in web: x2=' bgcolor="'+web['table_bgcolor_line4']+'" '
01054     x3=''
01055     if 'table_bgcolor_line5' in web: x3=' bgcolor="'+web['table_bgcolor_line5']+'" ' 
01056     x4=''
01057     if 'table_bgcolor_line6' in web: x4=' bgcolor="'+web['table_bgcolor_line6']+'" '
01058 
01059     vsx=ini['cfg']['pipeline_point_viewer']
01060     vs=vsx.get('main',[])
01061     vsf=vsx.get('fail',[])
01062     vss=vsx.get('strange',[])
01063     vsff=vsx.get('full_fail',[])
01064     vnt=vsx.get('normality_test',[])
01065 
01066     ks=vsx.get('key_prefix','')
01067 
01068     d=i.get('cm_data',{})
01069     d1=d.get('cm_data_obj',{}).get('cfg',{})
01070     points=d1.get('points',[])
01071 
01072     # Get pipeline module and description
01073     mu=d1.get('scenario_module_uoa','')
01074     r=cm_kernel.get_data_description({'cm_module_uoa':mu})
01075     if r['cm_return']>0: return r
01076     pu=r['cm_data_obj']['cfg']['cm_p_uids']
01077 
01078     h=''
01079 
01080     col=True; 
01081 
01082     px=int(i.get('point','0'))
01083     p=points[px]
01084 
01085     if px==0: dbase={}
01086     else:     dbase=i['cm_data']['cm_data_obj']['cfg'].get('dbase',{})
01087 
01088     if 'jscript_copy_to_clipboard' in web and i.get('cm_console')=='web':
01089        cm_kernel.print_for_web(web['jscript_copy_to_clipboard'])
01090 
01091     for kx in vs:
01092         k=ks+kx
01093         k1='#'+kx
01094         d1=desc.get(k1,{})
01095 
01096         if col: 
01097            col=False 
01098            if y==y1: x=x1
01099            else: x=x3
01100         else: 
01101            col=True 
01102            if y==y1: x=x2
01103            else: x=x4
01104 
01105         rg=cm_kernel.get_value_by_flattened_key({'cm_array':p, 'cm_key':k})
01106         if rg['cm_return']>0: return rg
01107         v=rg['cm_value']
01108         if v==None: v=''
01109 
01110         point[kx]=v
01111 
01112         impr=1
01113 
01114         if d1.get('characteristic','')=='yes' or d1.get('stat_characteristic','')=='yes':
01115            if px==0:              
01116               if v!='': dbase[kx]=float(v)
01117            else:
01118               if v!='':
01119                  dv=float(v)
01120                  if dv!=0 and kx in dbase and dbase[kx]!=0:
01121                     impr=dbase[kx]/dv
01122       
01123         d1['cm_view_as_text']='yes'
01124         ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
01125             'cm_action':'convert_field_to_html',
01126             'cm_value':str(v),
01127             'cm_key':'',
01128             'cm_mode':'view',
01129             'cm_data_desc':d1,
01130             'cm_disable_coloring':'yes'}
01131         r1=cm_kernel.access(ii)
01132         if r1['cm_return']>0: return r1
01133         vh=r1['cm_value_html']
01134         vt=r1['cm_value_text']
01135 
01136         h+='<td '+x+' align="right" valign="top"><small>'
01137 
01138         if kx in vsf:
01139            if v=='' or int(v)!=0:
01140               h+='<span style="color:#9F0000"><B>Fail ('+str(v)+')</B></span>'
01141            else:
01142               h+='Ok'
01143         elif kx in vss:
01144            if v=='' or int(v)!=0:
01145               h+='<span style="color:#9F0000">Strange ('+str(v)+')</span>'
01146            else:
01147               h+=str(v)
01148         elif kx in vsff:
01149            if v=='yes':
01150               h+='<span style="color:#9F0000"><b>Fail</b></span>'
01151            else:
01152               h+='no'
01153         elif kx==vsx.get('compiler_flags',''):
01154            rg1=cm_kernel.get_value_by_flattened_key({'cm_array':p, 'cm_key':ks+ini['cfg']['opt_class_uid']})
01155            if rg1['cm_return']>0: return rg1
01156            v1=rg1['cm_value']
01157            if v1==None: v1=''
01158            v2=''
01159 
01160            if v1!='':
01161               ccu=p.get('pipeline_input',{}).get('state',{}).get(pu['init'],{}).get('input',{}).get('ctuning_compiler_uoa','')
01162               to=p.get('pipeline_input',{}).get('state',{}).get(pu['init'],{}).get('input',{}).get('tuning_objective','')
01163               if ccu!='' and to!='':
01164                  # Load module to find opt point
01165                  opt=''
01166                  r2=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.compiler.optimizations'],
01167                                       'cm_action':'load',
01168                                       'cm_data_uoa':ccu})
01169                  if r2['cm_return']==0:
01170                     dd=r2['cm_data_obj']['cfg']
01171                     dd1=dd.get('opt_classes',{}).get(to,[])
01172 
01173                     found=False
01174                     for pxx in range(0, len(dd1)):
01175                         if dd1[pxx].get('cm_uid','')==v1:
01176                            opt=dd1[pxx].get('prepared_compiler_flags','')
01177                            found=True
01178                            break
01179 
01180                     if found:
01181                        v2='&nbsp;'+str(pxx)
01182                        v1='<a href="'+web['http_prefix']+'cm_menu=browse&cm_subaction_view&browse_module_uoa='+ini['cfg']['cm_modules']['ctuning.compiler.optimizations']+'&ignore_desc_expand=yes&browse_data_uoa='+ccu+'###opt_classes#'+to+'@'+str(pxx)+'#prepared_compiler_flags#">'+'<i>Class:'+v2+'</i></a>'
01183 
01184                        if opt!='' and 'jscript_copy_to_clipboard' in web:
01185                           v1+='<br><input type="button" class="cm-button-small" onClick="copyToClipboard(\''+opt+'\');" value="Copy&nbsp;opt.&nbsp;to&nbsp;clipboard"><br>'
01186 
01187            if v2!='':
01188               h+=v1
01189            else:
01190               h+='<input type="button" class="cm-button-small" onClick="copyToClipboard(\''+vh+'\');" value="Copy&nbsp;opt.&nbsp;to&nbsp;clipboard"><br>'
01191               if len(vh)>30: vh=vh[:30]+' ...'
01192               h+=vh
01193 
01194         elif kx in vnt:
01195            if v!='' and float(v)<0.05:
01196               h+='<span style="color:#9F0000"><B>'+str(v)+'</B></span>'
01197            else:
01198               h+=str(v)
01199 
01200         else:
01201            h+=vh
01202 
01203            if impr>1.05:
01204               h+='<BR><span style="color:#FF7F00"><b><i>('+("%.2f" % impr)+')</i></b></span>'
01205 
01206               ipoint[kx]="%.2f" % impr
01207        
01208         h+='<BR>\n'
01209         h+='</small></td>'
01210 
01211     if px==0:
01212        i['cm_data']['cm_data_obj']['cfg']['dbase']=dbase
01213 
01214     rr['point']=point
01215     rr['point_improvements']=ipoint
01216     rr['cm_html']=h
01217 
01218     return rr
01219 
01220 # ============================================================================
01221 def cmd_prune_flags(i):
01222     """
01223     Prune flags from CMD
01224 
01225     Input:  {
01226             }
01227 
01228     Output: {
01229               cm_return          - if =0, success
01230             }
01231     """
01232 
01233     base_flag=i['base_flag']
01234     best_flags=i['best_flags']
01235 
01236     build_cmd_orig=i['build_cmd']
01237 
01238     run_repeat=int(i.get('run_repeat','3'))
01239 
01240     output_file=i['output_file']
01241 
01242     run_cmd=i['run_cmd']
01243 
01244     cbs=i.get('check_binary_size','')
01245 
01246     objdump=i.get('objdump','')
01247     md5sum=i.get('md5sum','')
01248 
01249     tdr=float(i.get('time_delta_relax','0.0'))
01250 
01251     bf=best_flags.split(' ')
01252     lbf=len(bf)
01253     lbfo=lbf
01254 
01255     num_iters=int(i['number_of_iterations'])
01256     st=i.get('strategy','random')
01257 
01258     if st=='one_by_one': num_iters=lbf+1
01259 
01260     base_compile_time=0
01261     base_run_time=0
01262     base_bin_size=0
01263     base_md5=''
01264 
01265     best_compile_time=0
01266     best_run_time=0
01267     best_bin_size=0
01268     best_md5=''
01269 
01270     for k in range(-1, num_iters):
01271         cm_kernel.print_for_con('')
01272 
01273         # Prepare flags
01274         if k==-1:
01275            build_cmd=build_cmd_orig.replace(cm_kernel.convert_str_to_special('cm_flags'),base_flag)
01276         elif k==0:
01277            build_cmd=build_cmd_orig.replace(cm_kernel.convert_str_to_special('cm_flags'),best_flags)
01278         else:
01279            if st=='one_by_one': remove_flag=lbfo-k
01280            else:                remove_flag=random.randint(0,lbf-1)
01281 
01282            # Check if param
01283            remove_flag1=remove_flag
01284            if bf[remove_flag].strip()=='--param':
01285               remove_flag1=remove_flag+1
01286            elif remove_flag>0 and bf[remove_flag-1].strip()=='--param':
01287               remove_flag=remove_flag-1
01288               remove_flag1=remove_flag+1
01289 
01290            s='  Removing flag '+bf[remove_flag]
01291            if remove_flag!=remove_flag1:
01292               s+=' '+bf[remove_flag1]
01293 
01294            cm_kernel.print_for_con(s)
01295 
01296            xbf=''
01297            for q in range(0,lbf):
01298                if q!=remove_flag and q!=remove_flag1:
01299                   xbf+=' '+bf[q]
01300 
01301            build_cmd=build_cmd_orig.replace(cm_kernel.convert_str_to_special('cm_flags'),xbf)
01302 
01303         # Compile
01304         ct=0
01305         bs=0
01306         md5=''
01307         if os.path.isfile(output_file): os.remove(output_file)
01308         for q in range(0, run_repeat):
01309             ii={'cm_run_module_uoa':'code',
01310                 'cm_action':'native_run',
01311                 'skip_output':'yes',
01312                 'cm_unparsed':build_cmd.split(' ')}
01313             r=cm_kernel.access(ii)
01314             if r['cm_return']>0: return r
01315             et=float(r['execution_time'])
01316 
01317             if et>0.5:
01318                if ct==0: ct=et
01319                elif et<ct: ct=et
01320 
01321             if bs==0:
01322                bs=os.path.getsize(output_file);
01323                if md5sum!='' and objdump!='':
01324                   os.system(objdump+' > tmpx1')
01325                   os.system(md5sum+' tmpx1 > tmpx2')
01326 
01327                   if os.path.isfile('tmpx2'):
01328                      f=open('tmpx2','r')
01329                      x=f.read()
01330                      f.close
01331                      md5=x.split(' ')[0].strip()
01332 
01333         # Run
01334         rt=0
01335         if md5!=best_md5:
01336            for q in range(0, run_repeat):
01337                ii={'cm_run_module_uoa':'code',
01338                    'cm_action':'native_run',
01339                    'skip_output':'yes',
01340                   'cm_unparsed':run_cmd.split(' ')}
01341                r=cm_kernel.access(ii)
01342                if r['cm_return']>0: return r
01343                et=float(r['execution_time'])
01344                if et>0.5:
01345                   if rt==0: rt=et
01346                   elif et<rt: rt=et
01347 
01348         # Check timings
01349         if k==-1:
01350            it=' BASE'
01351            base_compile_time=ct
01352            base_run_time=rt
01353            base_bin_size=bs
01354            base_md5=md5
01355            ln=''
01356         elif k==0:
01357            it=' BEST'
01358            best_compile_time=ct
01359            best_run_time=rt
01360            best_bin_size=bs
01361            best_md5=md5
01362            ln=', LEN='+'%5i'%len(bf)
01363         else:
01364            if md5==best_md5 or et<=(best_run_time+tdr): 
01365               if cbs!='yes' or bs<=best_bin_size:
01366                  cm_kernel.print_for_con('  Flag removed!')
01367 
01368                  if remove_flag1!=remove_flag: del(bf[remove_flag1])
01369                  del(bf[remove_flag])
01370 
01371                  lbf=len(bf)
01372                  best_md5=md5
01373 
01374            it='%5i'%k
01375            ln=', LEN='+'%5i'%len(bf)
01376 
01377         cm_kernel.print_for_con(it+') RT='+'%8.3f'%rt+', BS='+'%8i'%bs+', CT='+'%8.3f'%ct+', MD5='+md5+ln)
01378 
01379     # Final combination
01380     xbf=''
01381     for q in range(0,lbf):
01382         xbf+=' '+bf[q]
01383 
01384     cm_kernel.print_for_con('')
01385     cm_kernel.print_for_con('Final flag combination:')
01386     cm_kernel.print_for_con('')
01387     cm_kernel.print_for_con(xbf)
01388 
01389     return {'cm_return':0}
01390 
01391 # ============================================================================
01392 def view_ctuning_space(i):
01393     """
01394     View cTuning space entry
01395 
01396     Input:  {
01397               view_ctuning_space_uoa
01398             }
01399 
01400     Output: {
01401               cm_return          - if =0, success
01402             }
01403     """
01404 
01405     vse=i.get('view_ctuning_space_uoa','')
01406     if vse=='':
01407        return {'cm_return':1, 'cm_error':'view_ctuning_space_uoa is not defined!'}
01408 
01409     e={}
01410 
01411     # Load entry (first from scenario and if not found, from ctuning.space)
01412     r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
01413                         'cm_action':'load',
01414                         'cm_data_uoa':vse})
01415     if r['cm_return']!=0:
01416        r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.space'],
01417                            'cm_action':'load',
01418                            'cm_data_uoa':vse})
01419        if r['cm_return']>0: return r
01420     e=r['cm_data_obj']['cfg']
01421 
01422     # Check points
01423     points=e.get('points',[])
01424     npoints=len(points)
01425 
01426     cm_kernel.print_for_con('Number of explored points in the entry: '+str(npoints))
01427 
01428     n=0
01429     for q in points:
01430         cm_kernel.print_for_con('=========================================================')
01431         cm_kernel.print_for_con('Point: '+str(n))
01432 
01433   pi=q.get('pipeline_input',{})
01434   po=q.get('pipeline_output',{})
01435 
01436   fail=po.get('fail',False)
01437         pos=po.get('state',{})
01438 
01439         flags=pos.get('1048eba38df5b240',{}).get('output',{}).get('prepared_compiler_flags','')
01440 
01441         # Show flags
01442         cm_kernel.print_for_con('')
01443         cm_kernel.print_for_con('Compiler flags:')
01444 
01445         cm_kernel.print_for_con('')
01446         cm_kernel.print_for_con(flags)
01447 
01448         if fail:
01449            cm_kernel.print_for_con('')
01450            cm_kernel.print_for_con('WARNING: Pipeline failed!')
01451 
01452         oc=pos.get('1f0b7684a4e82b4d',{}).get('output',{}).get('output_correct','')
01453         if oc!='' and oc!='0':
01454            cm_kernel.print_for_con('')
01455            cm_kernel.print_for_con('WARNING: Output differs!')
01456 
01457         # Show data
01458         if i.get('output_known_characteristics','')=='yes':
01459            r=cm_kernel.access({'cm_run_module_uoa':'ctuning.pipeline.build_and_run_program',
01460                                'cm_action':'output_known_characteristics',
01461                                'cm_characteristics':pos.get('cm_characteristics',{})})
01462            if r['cm_return']>0: return r
01463         
01464         n+=1
01465 
01466     return {'cm_return':0}
Generated on Wed May 28 02:49:02 2014 for Collective Mind Framework by DoxyGen 1.6.1
Concept, design and coordination: Grigori Fursin (C) 1993-2013