00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011 ini={}
00012 cm_kernel=None
00013
00014
00015 import copy
00016 import json
00017 import time
00018 import os
00019 import shutil
00020 import csv
00021 import filecmp
00022 import math
00023
00024
00025 def init(i):
00026 return {'cm_return':0}
00027
00028
00029 def run(i):
00030 """
00031 Run collective tuning pipeline
00032
00033 Input: {
00034 state - init state
00035
00036 Some additional debug/developer parameters
00037
00038 cm_show_json - if 'yes', show input json (normally used before pipeline)
00039 cm_save_json_to_file - if !='', save input json to file (normally used before pipeline)
00040 cm_show_output_json - if 'yes', show final output state json on console
00041 cm_save_output_json_to_file - if !='', save final output state json to file
00042 }
00043
00044 Output: {
00045 cm_return - if =0, success
00046 state - final state
00047 fail - True, if pipeline fail
00048 }
00049 """
00050
00051 pipeline_fail=False
00052
00053 pu=ini['cfg']['cm_p_uids']
00054
00055
00056 cm_kernel.print_for_con('')
00057 cm_kernel.print_for_con('********************* Initializing pipeline ...')
00058
00059 state={}
00060 if 'state' in i: state=i['state']
00061
00062 state_copy=copy.deepcopy(state)
00063
00064 if 'cm_characteristics' not in state: state['cm_characteristics']={}
00065 if 'misc' not in state: state['misc']={}
00066
00067
00068 r=cm_kernel.get_current_date_time({})
00069 if r['cm_return']>0: return r
00070 state['misc']['cm_iso_datetime_started']=r['cm_iso_datetime']
00071
00072 if cm_kernel.ini['web_auth'].get('cm_user_uid','')!='':
00073 state['misc']['cm_user_uid']=cm_kernel.ini['web_auth']['cm_user_uid']
00074
00075 state[pu['finish']]={}
00076 state[pu['finish']]['input']={}
00077 state[pu['finish']]['output']={}
00078
00079
00080 ii={}
00081 ii_init={}
00082 update_input_from_state_ll(ii, state, pu['init'])
00083
00084
00085 if ii.get('tuning_repo_uoa','')!='':
00086 ii_init['work_dir_repo_uoa']=ii['tuning_repo_uoa']
00087
00088 if ii.get('host_os_uoa','')!='':
00089 ii_init["run_host_os_uoa"]=ii['host_os_uoa']
00090 ii_init["run_target_os_uoa"]=ii['host_os_uoa']
00091
00092
00093 jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['os'],
00094 'cm_action':'load',
00095 'cm_data_uoa':ii['host_os_uoa']}
00096 r=cm_kernel.access(jj)
00097 if r['cm_return']>0: return r
00098 host_os_cfg=r['cm_data_obj']['cfg']
00099 else: return {'cm_return':1, 'cm_error':'parameter "host_os_uoa" is not set'}
00100
00101 if ii.get('target_os_uoa','')!='':
00102 ii_init["build_target_os_uoa"]=ii['target_os_uoa']
00103 if pu['prepare_dataset'] not in state:
00104 state[pu['prepare_dataset']]={}
00105 if 'input' not in pu['prepare_dataset']:
00106 state[pu['prepare_dataset']]['input']={}
00107 state[pu['prepare_dataset']]['input']['os_uoa']=ii['target_os_uoa']
00108 if pu['run_program'] not in state:
00109 state[pu['run_program']]={}
00110 state[pu['run_program']]['input']={}
00111 state[pu['run_program']]['input']['run_target_os_uoa']=ii['target_os_uoa']
00112
00113
00114 jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['os'],
00115 'cm_action':'load',
00116 'cm_data_uoa':ii['target_os_uoa']}
00117 r=cm_kernel.access(jj)
00118 if r['cm_return']>0: return r
00119 target_os_cfg=r['cm_data_obj']['cfg']
00120 else: return {'cm_return':1, 'cm_error':'parameter "target_os_uoa" is not set'}
00121
00122 if ii.get('target_processor_uoa','')!='':
00123 ii_init["run_target_processor_uoa"]=ii['target_processor_uoa']
00124
00125 if ii.get('program_uoa','')!='': ii_init['work_dir_data_uoa']=ii['program_uoa']
00126 else: return {'cm_return':1, 'cm_error':'parameter "program_uoa" is not set'}
00127
00128 if ii.get('program_module_uoa','')!='': ii_init['work_dir_module_uoa']=ii['program_module_uoa']
00129 else: ii_init['work_dir_module_uoa']=ini['cfg']['cm_modules']['code.source']
00130
00131 if ii.get('compiler_code_uoa','')!='': ii_init['compiler_code_uoa']=ii['compiler_code_uoa']
00132 else: return {'cm_return':1, 'cm_error':'parameter "compiler_code_uoa" is not set'}
00133
00134 if ii.get('ctuning_compiler_uoa','')!='': ii_init['ctuning_compiler_uoa']=ii['ctuning_compiler_uoa']
00135
00136 if ii.get('accelerator_compiler_code_uoa','')!='': ii_init['accelerator_compiler_code_uoa']=ii['accelerator_compiler_code_uoa']
00137 if ii.get('ctuning_accelerator_compiler_uoa','')!='': ii_init['ctuning_accelerator_compiler_uoa']=ii['ctuning_accelerator_compiler_uoa']
00138
00139 ii_init['keep_all_files']=i.get('state',{}).get('keep_all_files','')
00140 ii_init['dataset_uoa']=ii.get('dataset_uoa','')
00141 ii_init['compilation_type']=ii.get('compilation_type','')
00142 ii_init['run_cmd_key']=ii.get('run_cmd_key','')
00143
00144 state[pu['init_params']]={}
00145 state[pu['init_params']]['input']=ii_init
00146 state[pu['init_params']]['output']={}
00147
00148 cm_kernel.print_for_con('********************* Loading info about program ...')
00149
00150 jj={'cm_run_module_uoa':ii_init['work_dir_module_uoa'],
00151 'cm_action':'load',
00152 'cm_data_uoa':ii_init['work_dir_data_uoa']}
00153 if 'work_dir_repo_uoa' in ii_init: jj['cm_repo_uoa']=ii_init['work_dir_repo_uoa']
00154 r=cm_kernel.access(jj)
00155 if r['cm_return']>0: return r
00156 program_info=r['cm_data_obj']['cfg']
00157 program_path=r['cm_path']
00158 source_files=program_info.get('source_files',[])
00159
00160 if program_info.get('cm_display_as_alias','')!='':
00161 state[pu['init_params']]['output']['global_source_code_name']=program_info['cm_display_as_alias']
00162
00163 state[pu['init_params']]['output']['global_source_code_uid']=r['cm_uid']
00164
00165 cm_kernel.print_for_con('********************* Loading info about dataset ...')
00166
00167 if ii_init['dataset_uoa']!='':
00168 jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['dataset'],
00169 'cm_action':'load',
00170 'cm_data_uoa':ii_init['dataset_uoa']}
00171 r=cm_kernel.access(jj)
00172 if r['cm_return']>0: return r
00173 dataset_info=r['cm_data_obj']['cfg']
00174 if dataset_info.get('cm_display_as_alias','')!='':
00175 state[pu['init_params']]['output']['global_dataset_name']=dataset_info['cm_display_as_alias']
00176
00177 state[pu['init_params']]['output']['global_dataset_uid']=r['cm_uid']
00178
00179
00180
00181
00182 run_cmd_cfg={}
00183 if ii_init['run_cmd_key']!='':
00184 if 'run_cmds' in program_info and ii_init['run_cmd_key'] in program_info['run_cmds']:
00185 run_cmd_cfg=program_info['run_cmds'][ii_init['run_cmd_key']]
00186
00187 cm_kernel.print_for_con('********************* Loading compiler description ...')
00188
00189 ct_compiler_desc={}
00190 if ii.get('ctuning_compiler_uoa','')!='':
00191 kk={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.compiler'],
00192 'cm_data_uoa':ii['ctuning_compiler_uoa'],
00193 'cm_action':'load'}
00194 r=cm_kernel.access(kk)
00195 if r['cm_return']>0: return r
00196 ct_compiler_desc=r['cm_data_obj']['cfg']
00197
00198 if ct_compiler_desc.get('cm_display_as_alias','')!='':
00199 state[pu['init_params']]['output']['global_compiler_desc_name']=ct_compiler_desc['cm_display_as_alias']
00200
00201 compiler_code_desc={}
00202 if ii.get('compiler_code_uoa','')!='':
00203 kk={'cm_run_module_uoa':ini['cfg']['cm_modules']['code'],
00204 'cm_data_uoa':ii['compiler_code_uoa'],
00205 'cm_action':'load'}
00206 r=cm_kernel.access(kk)
00207 if r['cm_return']>0: return r
00208 compiler_code_desc=r['cm_data_obj']['cfg']
00209
00210 if compiler_code_desc.get('cm_display_as_alias','')!='':
00211 state[pu['init_params']]['output']['global_compiler_package_name']=compiler_code_desc['cm_display_as_alias']
00212
00213
00214 cm_kernel.print_for_con('')
00215 cm_kernel.print_for_con('********************* Preparing run-time dataset ...')
00216
00217 jj={}
00218 jj['cm_run_module_uoa']=ini['cfg']['cm_modules']['code']
00219 jj['cm_action']='prepare_cmd'
00220 jj['code_module_uoa']=ii_init['work_dir_module_uoa']
00221 jj['code_data_uoa']=ii_init['work_dir_data_uoa']
00222 jj['dataset_uoa']=ii_init['dataset_uoa']
00223 jj['compilation_type']=ii_init['compilation_type']
00224 jj['run_cmd_key']=ii_init['run_cmd_key']
00225 r=update_input_from_state_ll(jj, state, pu['prepare_dataset'], True)
00226
00227 dataset_build_compiler_vars=r.get('build_compiler_vars',{})
00228
00229 run_correctness_output_files=r.get('run_time',{}).get('run_correctness_output_files',[])
00230
00231
00232
00233 if state[pu['clean_program']]['enable']=='yes':
00234 cm_kernel.print_for_con('')
00235 cm_kernel.print_for_con('********************* Cleaning program ...')
00236
00237 rft=state[pu['clean_program']]['input'].get('refresh_from_templates','')
00238 if rft=='yes':
00239 sft=program_info.get('source_files_templates',[])
00240 if len(sft)>0:
00241 cm_kernel.print_for_con('')
00242 cm_kernel.print_for_con('Refreshing from templates ...')
00243
00244 for q in sft:
00245 p1=os.path.join(program_path, q+ini['cfg']['template_extension'])
00246 p2=os.path.join(program_path, q)
00247
00248 shutil.copyfile(p1, p2)
00249
00250 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
00251 'cm_action':'build',
00252 'ignore_script_error':'yes'}
00253
00254 ii['keep_all_files']=ii_init.get('keep_all_files','')
00255
00256
00257 if len(state[pu['build_program']]['input'].get('cm_dependencies',{}))>0:
00258 ii['cm_dependencies']=state[pu['build_program']]['input']['cm_dependencies']
00259
00260 if 'build_set_env' in run_cmd_cfg:
00261 if 'run_set_env2' not in ii: ii['run_set_env2']={}
00262 ii['run_set_env2'].update(run_cmd_cfg['build_set_env'])
00263
00264 update_input_from_state_ll(ii, state, pu['init_params'])
00265 r=update_input_from_state_ll(ii, state, pu['clean_program'], True)
00266 if r['cm_return']>0: return r
00267
00268
00269 lhc=state.get(pu['use_likwid'],{}).get('input',{})
00270 lhc_uoa=lhc.get('likwid_code_uoa','')
00271
00272 if state.get(pu['use_likwid'],{}).get('enable','')=='yes' and state[pu['use_likwid']]['input'].get('instrumentation','')=='yes':
00273 cm_kernel.print_for_con('')
00274 cm_kernel.print_for_con('********************* Using likwid and instrumentation ...')
00275
00276 if lhc_uoa=='':
00277 return {'cm_return':1, 'cm_error':'likwid_code_uoa is not defined'}
00278
00279 sft=program_info.get('source_files_templates',[])
00280 if len(sft)==0:
00281 return {'cm_return':1, 'cm_error':'this code does not have templates to instrument'}
00282
00283 cm_kernel.print_for_con('')
00284 cm_kernel.print_for_con('Performing instrumentation using templates ...')
00285
00286 for q in sft:
00287 p1=os.path.join(program_path, q+ini['cfg']['template_extension'])
00288 p2=os.path.join(program_path, q)
00289
00290 f=open(p1,'r')
00291 source=f.read().replace('\r','')
00292 f.close()
00293
00294
00295 qk1=ini['cfg']['source_templates_c']
00296 qk2=ini['cfg']['likwid_instrumentation_c']
00297 for k in qk1.keys():
00298 ka=qk1[k]
00299 kb=qk2[k]
00300 kk=source.find(ka)
00301 if kk>0:
00302 source=source[:kk-1]+'\n'+kb+'\n'+source[kk:]
00303
00304 f=open(p2,'w')
00305 f.write(source)
00306 f.close()
00307
00308 if len(ii.get('code_deps',[]))==0: ii['code_deps']=[]
00309 ii['code_deps'].append({"CM_CODE_DEP_LIKWID":lhc_uoa})
00310
00311 if len(ii.get('include_deps',[]))==0: ii['include_deps']=[]
00312 ii['include_deps'].append({"CM_CODE_DEP_LIKWID":lhc_uoa})
00313
00314 if len(ii.get('lib_deps',[]))==0: ii['lib_deps']=[]
00315 ii['lib_deps'].append({"CM_CODE_DEP_LIKWID":lhc_uoa})
00316
00317 if 'run_set_env2' not in ii: ii['run_set_env2']={}
00318
00319 pthread_flag=ct_compiler_desc.get('misc_ld_flags',{}).get('pthread_lib','')
00320
00321 x=ii['run_set_env2'].get('CM_LD_FLAGS_MISC','')
00322 x+=' $CM_'+lhc_uoa+'_LIB/$CM_'+lhc_uoa+'_TARGET '+pthread_flag
00323 ii['run_set_env2']['CM_LD_FLAGS_MISC']=x
00324
00325
00326 if state.get(pu['source_to_source_polyhedral_pluto'],{}).get('enable','')=='yes':
00327 cm_kernel.print_for_con('')
00328 cm_kernel.print_for_con('********************* Using polyhedral source-to-source transformer PLUTO (parallelize, tile, unroll) ...')
00329
00330 pc=state[pu['source_to_source_polyhedral_pluto']]['input']
00331 pcc_uoa=pc.get('pluto_compiler_code_uoa','')
00332 if pcc_uoa=='':
00333 return {'cm_return':1, 'cm_error':'pluto_compiler_code_uoa is not defined'}
00334
00335 popt=''
00336
00337 if pc.get('parallelize','')=='yes':
00338 popt+=' --parallel'
00339
00340
00341 state[pu['prepare_compiler_flags']].get('input',{})['use_openmp_flag']='yes'
00342
00343 if pc.get('tile','')=='yes':
00344 popt+=' --tile'
00345
00346 ptff='tile.sizes'
00347 if os.path.isfile(ptff): os.remove(ptff)
00348
00349 ptf=pc.get('tile_factor','')
00350 ptft=pc.get('tile_factors_topology','')
00351
00352 if ptf!='' or ptft!='':
00353 if ptft!='':
00354 ltf=ptft.split(';')
00355 else:
00356 ltf=[ptf]
00357
00358 cm_kernel.print_for_con('')
00359 cm_kernel.print_for_con(' Preparing file with tile sizes ...')
00360
00361 f=open(ptff, 'wt')
00362 for q in ltf:
00363 f.write(q+'\n')
00364 f.close()
00365
00366 if pc.get('unroll','')=='yes':
00367 popt+=' --unroll'
00368
00369 puf=pc.get('unroll_factor','')
00370 if puf!='':
00371 popt+=' --ufactor='+puf
00372
00373 cm_kernel.print_for_con('')
00374 source_files_new=[]
00375 for q in source_files:
00376 qf, qe = os.path.splitext(q)
00377 qo=qf+'.pluto'+qe
00378 if os.path.isfile(qo): os.remove(qo)
00379 cmd='polycc '+popt+' '+q
00380 cm_kernel.print_for_con(' Executing "'+cmd+'" ...')
00381
00382
00383 kk={}
00384 kk['cm_run_module_uoa']=ini['cfg']['cm_modules']['code']
00385 kk['cm_action']='run'
00386 kk['work_dir_module_uoa']=ii_init['work_dir_module_uoa']
00387 kk['work_dir_data_uoa']=ii_init['work_dir_data_uoa']
00388 kk['work_dir_repo_uoa']=ii_init.get('work_dir_repo_uoa','')
00389 kk['keep_all_files']=ii_init.get('keep_all_files','')
00390
00391 kk['run_host_os_uoa']=ii_init["run_host_os_uoa"]
00392 kk['run_target_os_uoa']=ii_init["run_target_os_uoa"]
00393 kk['run_target_processor_uoa']=ii_init["run_target_processor_uoa"]
00394
00395 kk['code_deps']=[]
00396 kk['code_deps'].append({"CM_CODE_DEP_PLUTO_COMPILER":pcc_uoa})
00397
00398 kk['run_cmd']=cmd
00399
00400 r=cm_kernel.access(kk)
00401 if r['cm_return']>0: return r
00402
00403 if not os.path.isfile(qo):
00404 return {'cm_return':1, 'cm_error':'PLUTO did not generate file for q'}
00405
00406 source_files_new.append(qo)
00407
00408 cm_kernel.print_for_con('')
00409 cm_kernel.print_for_con('Updating source files list after PLUTO compiler ...')
00410
00411 source_files=source_files_new
00412
00413
00414 prepared_flags=''
00415 if state[pu['prepare_compiler_flags']]['enable']=='yes':
00416 cm_kernel.print_for_con('')
00417 cm_kernel.print_for_con('********************* Preparing compiler flags ...')
00418
00419 all_flags_order=[]
00420
00421 es=state[pu['prepare_compiler_flags']].get('input',{}).get('external_flag_selection','')
00422 if es!='':
00423 prepared_flags=es
00424 else:
00425 if state[pu['prepare_compiler_flags']].get('input',{}).get('use_only_base_flag','')=='yes':
00426 all_flags={'base_flag':state[pu['prepare_compiler_flags']]['input'].get('all_compiler_flags',{}).get('base_flag','')}
00427 all_flags_order=[]
00428 else:
00429 all_flags=state[pu['prepare_compiler_flags']]['input'].get('all_compiler_flags',{})
00430 all_flags_order=state[pu['prepare_compiler_flags']]['input'].get('all_compiler_flags_order',[])
00431 if len(all_flags_order)==0 and len(state[pu['prepare_compiler_flags']].get('output',{}).get('all_compiler_flags_order',[]))>0:
00432 all_flags_order=state[pu['prepare_compiler_flags']]['output']['all_compiler_flags_order']
00433
00434 if state[pu['prepare_compiler_flags']].get('input',{}).get('use_openmp_flag','')=='yes':
00435 openmp_desc=ct_compiler_desc.get('all_compiler_flags_desc',{}).get('##openmp_flag',{}).get('choice',[])
00436 if len(openmp_desc)==0:
00437 return {'cm_return':1, 'cm_error':'OpenMP flag must be used, but it is not found in compiler description'}
00438 all_flags['openmp_flag']=openmp_desc[0]
00439 else:
00440
00441 all_flags['openmp_flag']=''
00442
00443
00444 kk={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.compiler'],
00445 'cm_action':'prepare_flags',
00446 'cm_data_uoa':ii_init.get('ctuning_compiler_uoa',''),
00447 'all_flags':all_flags,
00448 'all_flags_order':all_flags_order}
00449 r=cm_kernel.access(kk)
00450 if r['cm_return']>0: return r
00451
00452 all_flags_order=r['all_flags_order']
00453
00454 prepared_flags=r['prepared_flags']+' '+state[pu['prepare_compiler_flags']]['input'].get('additional_compiler_flags','')
00455 prepared_flags=prepared_flags.strip()
00456
00457 cm_kernel.print_for_con('')
00458 cm_kernel.print_for_con('Prepared flags:')
00459 cm_kernel.print_for_con('')
00460 cm_kernel.print_for_con(prepared_flags)
00461
00462 state[pu['prepare_compiler_flags']]['output']={}
00463 state[pu['prepare_compiler_flags']]['output']['prepared_compiler_flags']=prepared_flags
00464 state[pu['prepare_compiler_flags']]['output']['all_flags_order']=all_flags_order
00465 if state[pu['prepare_compiler_flags']]['input'].get('opt_class_uid','')!='':
00466 state[pu['prepare_compiler_flags']]['output']['opt_class_uid']=state[pu['prepare_compiler_flags']]['input']['opt_class_uid']
00467
00468 if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
00469 if 'CM_COMPILER_FLAGS_CC_OPTS' not in ii['run_set_env2']: ii['run_set_env2']['CM_COMPILER_FLAGS_CC_OPTS']=''
00470
00471
00472 if state[pu['use_compiler_profiling']]['enable']=='yes':
00473 cm_kernel.print_for_con('')
00474 cm_kernel.print_for_con('********************* Using compiler profiling ...')
00475
00476 prof_flag=ct_compiler_desc.get('profile',{}).get('flag','')
00477
00478 if prof_flag!='':
00479 ii['run_set_env2']['CM_COMPILER_FLAGS_CC_OPTS']=prof_flag+' '+ii['run_set_env2']['CM_COMPILER_FLAGS_CC_OPTS']
00480
00481
00482 if state[pu['use_ctuning_milepost']]['enable']=='yes':
00483 cm_kernel.print_for_con('')
00484 cm_kernel.print_for_con('********************* Using cTuning CC / MILEPOST GCC ...')
00485
00486 milepost_compiler_code_uoa=state[pu['use_ctuning_milepost']]['input'].get('milepost_compiler_code_uoa','')
00487 milepost_compiler_extract_after_pass=state[pu['use_ctuning_milepost']]['input'].get('milepost_compiler_extract_after_pass','')
00488 ctuning_compiler_code_uoa=state[pu['use_ctuning_milepost']]['input'].get('ctuning_compiler_code_uoa','')
00489 ctuning_compiler_plugins_code_uoa=state[pu['use_ctuning_milepost']]['input'].get('ctuning_compiler_plugins_code_uoa','')
00490 mode=state[pu['use_ctuning_milepost']]['input'].get('mode','')
00491 modex=''
00492 if mode=='extract_program_structure': modex=' --ct-extract-structure '
00493 elif mode=='extract_program_static_properties': modex=' --ct-extract-features '
00494
00495 ii['run_set_env2']['CM_COMPILER_FLAGS_CC_OPTS']=modex+' '+ii['run_set_env2']['CM_COMPILER_FLAGS_CC_OPTS']
00496
00497 if milepost_compiler_code_uoa=='' or ctuning_compiler_code_uoa=='' or ctuning_compiler_plugins_code_uoa=='':
00498 return {'cm_return':1,'cm_error':'milepost_compiler_code_uoa or ctuning_compiler_code_uoa or ctuning_compiler_plugins_code_uoa is not defined'}
00499
00500 if len(ii.get('code_deps',[]))==0: ii['code_deps']=[]
00501 ii['code_deps'].append({"CM_CODE_DEP_MILEPOST_GCC":milepost_compiler_code_uoa})
00502 ii['code_deps'].append({"CM_CODE_DEP_CTUNING_CC_PLUGIN":ctuning_compiler_plugins_code_uoa})
00503 ii['code_deps'].append({"CM_CODE_DEP_MAIN_COMPILER":ii_init['compiler_code_uoa']})
00504 ii['code_deps'].append({"CM_CODE_DEP_COMPILER":ctuning_compiler_code_uoa})
00505
00506 if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
00507 if milepost_compiler_extract_after_pass!='': ii['run_set_env2']['ICI_PROG_FEAT_PASS']=milepost_compiler_extract_after_pass
00508
00509 ici_structure='ici_passes_function.'
00510 ici_properties='ici_features_function.'
00511
00512
00513 if ii_init.get('keep_all_files','')!='yes':
00514 dirList=os.listdir('.')
00515 for fn in dirList:
00516 if os.path.isfile(fn) and (fn.startswith(ici_structure) or fn.startswith(ici_properties)):
00517 os.remove(fn)
00518
00519
00520 functions=state[pu['use_ctuning_milepost']]['input'].get('hot_functions',[])
00521 file_with_funcs=''
00522 if len(functions)>0:
00523 rx=cm_kernel.gen_uid({})
00524 if rx['cm_return']>0: return rx
00525 file_with_funcs=os.path.join(os.getcwd(), 'tmp-funcs-'+rx['cm_uid']+'.txt')
00526 f=open(file_with_funcs,'wt')
00527 for q in functions:
00528 f.write(q+'\n')
00529 f.close()
00530 ii['run_set_env2']['ICI_FILE_SELECT_FUNCTIONS']=file_with_funcs
00531
00532
00533 if state[pu['use_universal_alchemist']]['enable']=='yes':
00534 cm_kernel.print_for_con('')
00535 cm_kernel.print_for_con('********************* Using Universal Alchemist ...')
00536
00537 alchemist_plugin_code_uoa=state[pu['use_universal_alchemist']]['input'].get('universal_alchemist_plugin_code_uoa','')
00538
00539 if alchemist_plugin_code_uoa!='':
00540 r=cm_kernel.gen_uid({})
00541 if r['cm_return']>0: return r
00542 uni_alchemist_tmp1=os.path.join(os.getcwd(), 'tmp-alchemist-input-'+r['cm_uid']+'.json')
00543
00544 if state[pu['use_universal_alchemist']]['input'].get('output_file','')!='':
00545 uni_alchemist_tmp2=state[pu['use_universal_alchemist']]['input']['output_file']
00546 d_tmp2=False
00547 else:
00548 r=cm_kernel.gen_uid({})
00549 if r['cm_return']>0: return r
00550 uni_alchemist_tmp2=os.path.join(os.getcwd(), 'tmp-alchemist-output-'+r['cm_uid']+'.json')
00551 d_tmp2=True
00552
00553 if state[pu['use_universal_alchemist']]['input'].get('input_file','')!='':
00554 uni_alchemist_tmp3=state[pu['use_universal_alchemist']]['input']['input_file']
00555 d_tmp3=False
00556
00557 if state[pu['use_universal_alchemist']]['input'].get('force_unroll_factor','')!='':
00558 if os.path.isfile(uni_alchemist_tmp3):
00559 r=cm_kernel.load_json_file({'cm_filename':uni_alchemist_tmp3})
00560 if r['cm_return']>0: return r
00561 uao=r['cm_array']
00562
00563 uao1=uao.get('transformations',{})
00564
00565
00566 fuf=state[pu['use_universal_alchemist']]['input']['force_unroll_factor']
00567 if 'last_no' in uao1:
00568 if 'tmp_last_no' in uao1: del(uao1['tmp_last_no'])
00569 for ua in uao1:
00570 if ua!='last_no':
00571 ubx=uao1[ua]
00572 if ubx.get('transformation','')=='unroll':
00573 if 'factor' in ubx:
00574 ubx['factor']=fuf
00575 else:
00576 for ua in uao1:
00577 for ub in uao1[ua]:
00578 ubx=uao1[ua][ub]
00579 if ubx.get('transformation','')=='unroll':
00580 if 'factor' in ubx:
00581 ubx['factor']=fuf
00582
00583 r4=cm_kernel.save_array_to_file_as_json({'cm_filename':uni_alchemist_tmp3, 'cm_array':uao})
00584 if r4['cm_return']>0: return r4
00585
00586 else:
00587 r=cm_kernel.gen_uid({})
00588 if r['cm_return']>0: return r
00589 uni_alchemist_tmp3=os.path.join(os.getcwd(), 'tmp-alchemist-input-'+r['cm_uid']+'.json')
00590 d_tmp3=True
00591
00592 ai={}
00593 mode=state[pu['use_universal_alchemist']]['input'].get('mode','')
00594 if mode=='record_transformations':
00595 ai={'action':mode, 'output_file': uni_alchemist_tmp2}
00596 elif mode=='apply_transformations':
00597 ai={'action':mode, 'input_file': uni_alchemist_tmp3}
00598
00599 r4=cm_kernel.save_array_to_file_as_json({'cm_filename':uni_alchemist_tmp1, 'cm_array':ai})
00600 if r4['cm_return']>0: return r4
00601
00602 if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
00603 ii['run_set_env2']['UNI_ALCHEMIST_INPUT_FILE']=uni_alchemist_tmp1
00604
00605 if len(ii.get('code_deps',[]))==0: ii['code_deps']=[]
00606 ii['code_deps'].append({"CM_CODE_DEP_UNI_ALCHEMIST_PLUGIN":alchemist_plugin_code_uoa})
00607
00608 ii['run_set_env2']['UNI_ALCHEMIST_PLUGINS']=host_os_cfg.get('env_var_start','')+ \
00609 'CM_'+alchemist_plugin_code_uoa+'_LIB'+host_os_cfg.get('env_var_stop','')+ \
00610 host_os_cfg.get('dir_sep','')+host_os_cfg.get('env_var_start','')+ \
00611 'CM_'+alchemist_plugin_code_uoa+'_TARGET'+host_os_cfg.get('env_var_stop','')
00612
00613 ii['run_set_env2']['UNI_ALCHEMIST_USE']='1'
00614
00615
00616 if state[pu['use_alchemist']]['enable']=='yes':
00617 cm_kernel.print_for_con('')
00618 cm_kernel.print_for_con('********************* Using Alchemist ...')
00619
00620 alchemist_plugin_code_uoa=state[pu['use_alchemist']]['input'].get('alchemist_plugin_code_uoa','')
00621 alchemist_subplugin_code_uoa=state[pu['use_alchemist']]['input'].get('alchemist_subplugin_code_uoa','')
00622
00623 if alchemist_plugin_code_uoa!='' and alchemist_subplugin_code_uoa!='':
00624 r=cm_kernel.gen_uid({})
00625 if r['cm_return']>0: return r
00626 alchemist_tmp1=os.path.join(os.getcwd(), 'tmp-alchemist-input-'+r['cm_uid']+'.json')
00627
00628 r=cm_kernel.gen_uid({})
00629 if r['cm_return']>0: return r
00630 alchemist_tmp2=os.path.join(os.getcwd(), 'tmp-alchemist-output-'+r['cm_uid']+'.json')
00631
00632 ai={}
00633 mode=state[pu['use_alchemist']]['input'].get('mode','')
00634 if mode=='record_function_names' or mode=='record_executed_passes':
00635 ai={'action':mode, 'output_file': alchemist_tmp2}
00636
00637 r4=cm_kernel.save_array_to_file_as_json({'cm_filename':alchemist_tmp1, 'cm_array':ai})
00638 if r4['cm_return']>0: return r4
00639
00640 if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
00641 ii['run_set_env2']['CM_ALCHEMIST_INPUT_FILE']=alchemist_tmp1
00642
00643 if len(ii.get('code_deps',[]))==0: ii['code_deps']=[]
00644 ii['code_deps'].append({"CM_CODE_DEP_ALCHEMIST_PLUGIN":alchemist_plugin_code_uoa})
00645 ii['code_deps'].append({"CM_CODE_DEP_ALCHEMIST_SUBPLUGIN":alchemist_subplugin_code_uoa})
00646
00647 plugin_flag=ct_compiler_desc.get('plugins',{}).get('flag','')
00648
00649 if plugin_flag!='':
00650 ml=program_info.get('main_language','')
00651 mlx="CC"
00652 if ml=='cpp': mlx='CPP'
00653 elif ml=='fortran': mlx='FC'
00654
00655 ii['run_set_env2']['CM_COMPILER_FLAGS_CC_OPTS']=plugin_flag+host_os_cfg.get('env_var_start','')+ \
00656 'CM_'+alchemist_plugin_code_uoa+'_LIB_'+mlx+host_os_cfg.get('env_var_stop','')
00657
00658 ii['run_set_env2']['CM_ALCHEMIST_PLUGINS']=host_os_cfg.get('env_var_start','')+ \
00659 'CM_'+alchemist_subplugin_code_uoa+'_LIB_'+mlx+host_os_cfg.get('env_var_stop','')
00660
00661
00662 if state[pu['use_openme']]['enable']=='yes':
00663 cm_kernel.print_for_con('')
00664 cm_kernel.print_for_con('********************* Using OpenME ...')
00665
00666 cm_deps=state[pu['build_program']]['input'].get('cm_dependencies',{})
00667 cm_deps.append({"741ddb394cd917fb":state[pu['use_openme']]['input'].get('openme_library','')})
00668 cm_deps.append({"974b24d4ca79bf6e":state[pu['use_openme']]['input'].get('cjson_library','')})
00669 cm_deps.append({"15fa030f4d9e8925":state[pu['use_openme']]['input'].get('openme_library_ctuning_plugin','')})
00670 state[pu['build_program']]['input']['cm_dependencies']=cm_deps
00671 if len(ii.get('compiler_vars',{}))==0: ii['compiler_vars']={}
00672 ii['compiler_vars']['OPENME']='1'
00673
00674
00675 if state[pu['build_program']]['enable']=='yes':
00676 cm_kernel.print_for_con('')
00677 cm_kernel.print_for_con('********************* Building program ...')
00678
00679 if len(source_files)>0:
00680 ii['source_files']=source_files
00681
00682 if 'CM_COMPILER_FLAGS_CC_OPTS' not in ii['run_set_env2']: ii['run_set_env2']['CM_COMPILER_FLAGS_CC_OPTS']=''
00683
00684 ii['keep_all_files']=ii_init.get('keep_all_files','')
00685
00686
00687 ii['ignore_script_error']='yes'
00688
00689 ii['skip_target_file_check']='no'
00690 if len(ii.get('code_deps',[]))==0: ii['code_deps']=[]
00691 if state[pu['use_ctuning_milepost']]['enable']=='no':
00692 ii['code_deps'].append({"CM_CODE_DEP_COMPILER":ii_init['compiler_code_uoa']})
00693 if ii_init.get('accelerator_compiler_code_uoa','')!='':
00694
00695 gfound=False
00696 for g in ii['code_deps']:
00697 g1=g.keys()[0]
00698 g2=g[g1]
00699
00700 if g1=='CM_CODE_DEP_ACC_COMPILER' and g2==ii_init['accelerator_compiler_code_uoa']:
00701 gfound=True
00702 break
00703 if not gfound:
00704 ii['code_deps'].append({"CM_CODE_DEP_ACC_COMPILER":ii_init['accelerator_compiler_code_uoa']})
00705 ii['run_set_env2']['CM_COMPILER_FLAGS_CC_OPTS']+=' '+prepared_flags
00706 if len(ii.get('compiler_vars',{}))==0: ii['compiler_vars']={}
00707 ii['compiler_vars'].update(program_info.get('build_compiler_vars',{}))
00708
00709 program_build_properties=state[pu['build_program']]['input'].get('program_build_properties',{})
00710 if len(program_build_properties)>0:
00711 for q in program_build_properties:
00712 ii['compiler_vars'][q]=program_build_properties[q]
00713
00714
00715 if len(dataset_build_compiler_vars)>0:
00716 for q in dataset_build_compiler_vars:
00717 ii['compiler_vars'][q]=dataset_build_compiler_vars[q]
00718
00719 if state[pu['build_program']].get('input',{}).get('skip_output_redirect','')!='yes':
00720 ii['run_cmd_out1']='tmp-build-output1.tmp'
00721 ii['run_cmd_out2']='tmp-build-output2.tmp'
00722
00723 if 'build_set_env' in run_cmd_cfg:
00724 if 'run_set_env2' not in ii: ii['run_set_env2']={}
00725 ii['run_set_env2'].update(run_cmd_cfg['build_set_env'])
00726
00727 r=update_input_from_state_ll(ii, state, pu['build_program'], True)
00728 if r['cm_return']>0: return r
00729
00730 if r.get('timed_out', '')=='yes' or \
00731 r.get('failed_run', '')=='yes' or \
00732 int(r.get('target_file_size','0'))==0:
00733
00734 cm_kernel.print_for_con('')
00735 cm_kernel.print_for_con('Problem: compilation failed - exiting pipeline!')
00736
00737 state[pu['build_program']]['output']['fail']='yes'
00738
00739 pipeline_fail=True
00740
00741 else:
00742
00743 executable=state[pu['build_program']]['output'].get('target_file','')
00744
00745 ctf=state[pu['build_program']].get('input',{}).get('copy_target_file_to','')
00746 if ctf!='':
00747 if os.path.isfile(executable):
00748 shutil.copyfile(executable, ctf)
00749
00750 state['cm_characteristics']['global_code_size']=r.get('target_file_size','0')
00751
00752
00753 if state[pu['get_objdump_and_md5sum']]['enable']=='yes':
00754 cm_kernel.print_for_con('')
00755 cm_kernel.print_for_con('********************* Get objdump and md5sum ...')
00756
00757 objdump=compiler_code_desc.get('objdump','')
00758 if objdump=='':
00759 cm_kernel.print_for_con('')
00760 cm_kernel.print_for_con('No defined objdump tool in used compiler - skipping this section ... ')
00761 else:
00762 md5sum=host_os_cfg.get('md5sum','')
00763 if objdump=='':
00764 cm_kernel.print_for_con('')
00765 cm_kernel.print_for_con('No defined md5sum tool in host OS - skipping this section ... ')
00766 else:
00767 r=cm_kernel.gen_uid({})
00768 if r['cm_return']>0: return r
00769 objdump_tmp=os.path.join('tmp-objdump-output-'+r['cm_uid']+'.dat')
00770
00771 cmd=objdump+' '+executable+' > '+objdump_tmp
00772
00773
00774 kk={}
00775 kk['cm_run_module_uoa']=ini['cfg']['cm_modules']['code']
00776 kk['cm_action']='run'
00777 kk['work_dir_module_uoa']=ii_init['work_dir_module_uoa']
00778 kk['work_dir_data_uoa']=ii_init['work_dir_data_uoa']
00779 kk['work_dir_repo_uoa']=ii_init.get('work_dir_repo_uoa','')
00780 kk['keep_all_files']=ii_init.get('keep_all_files','')
00781
00782 kk['run_host_os_uoa']=ii_init["run_host_os_uoa"]
00783 kk['run_target_os_uoa']=ii_init["run_target_os_uoa"]
00784 kk['run_target_processor_uoa']=ii_init["run_target_processor_uoa"]
00785
00786 kk['code_deps']=[]
00787 kk['code_deps'].append({"CM_CODE_DEP_COMPILER":ii_init['compiler_code_uoa']})
00788
00789 kk['run_cmd']=cmd
00790
00791 r=cm_kernel.access(kk)
00792 if r['cm_return']>0: return r
00793
00794 if not os.path.isfile(objdump_tmp):
00795 cm_kernel.print_for_con('')
00796 cm_kernel.print_for_con('Object dump file was not created - skipping this section ... ')
00797 else:
00798 r=cm_kernel.gen_uid({})
00799 if r['cm_return']>0: return r
00800 md5sum_tmp=os.path.join('tmp-md5sum-output-'+r['cm_uid']+'.dat')
00801
00802 cmd=md5sum+' '+objdump_tmp+' > '+md5sum_tmp
00803
00804
00805 kk={}
00806 kk['cm_run_module_uoa']=ini['cfg']['cm_modules']['code']
00807 kk['cm_action']='run'
00808 kk['work_dir_module_uoa']=ii_init['work_dir_module_uoa']
00809 kk['work_dir_data_uoa']=ii_init['work_dir_data_uoa']
00810 kk['work_dir_repo_uoa']=ii_init.get('work_dir_repo_uoa','')
00811 kk['keep_all_files']=ii_init.get('keep_all_files','')
00812
00813 kk['run_host_os_uoa']=ii_init["run_host_os_uoa"]
00814 kk['run_target_os_uoa']=ii_init["run_target_os_uoa"]
00815 kk['run_target_processor_uoa']=ii_init["run_target_processor_uoa"]
00816
00817 kk['code_deps']=[]
00818
00819 kk['run_cmd']=cmd
00820
00821 r=cm_kernel.access(kk)
00822 if r['cm_return']>0: return r
00823
00824 if ii_init.get('keep_all_files','')!='yes': os.remove(objdump_tmp)
00825
00826 if not os.path.isfile(md5sum_tmp):
00827 cm_kernel.print_for_con('')
00828 cm_kernel.print_for_con('md5sum file was not created - skipping this section ... ')
00829 else:
00830 f=open(md5sum_tmp,'rt')
00831 l = f.readline().strip()
00832 lx=l.find(' ')
00833 if lx>0:
00834 md5sum_result=l[:lx].strip()
00835 f.close()
00836 if ii_init.get('keep_all_files','')!='yes': os.remove(md5sum_tmp)
00837
00838
00839 state[pu['get_objdump_and_md5sum']]['output']['md5sum']=md5sum_result
00840
00841 state['cm_characteristics']['global_code_md5sum']=md5sum_result
00842
00843 cm_kernel.print_for_con('')
00844 cm_kernel.print_for_con('md5sum='+md5sum_result)
00845
00846
00847 ao={}
00848 if state[pu['use_alchemist']]['enable']=='yes':
00849 cm_kernel.print_for_con('')
00850 cm_kernel.print_for_con('********************* Finish using Alchemist ...')
00851
00852 if os.path.isfile(alchemist_tmp2):
00853 r=cm_kernel.load_json_file({'cm_filename':alchemist_tmp2})
00854 if r['cm_return']>0: return r
00855 ao=r['cm_array']
00856
00857 if ii_init.get('keep_all_files','')!='yes':
00858 os.remove(alchemist_tmp2)
00859 if os.path.isfile(alchemist_tmp1):
00860 os.remove(alchemist_tmp1)
00861
00862 if len(ao)>0:
00863 state[pu['finish_alchemist']]={}
00864 state[pu['finish_alchemist']]['input']={}
00865 state[pu['finish_alchemist']]['output']={}
00866 state[pu['finish_alchemist']]['output']['raw_plugin_output']=ao
00867
00868
00869 uao={}
00870 if state[pu['use_universal_alchemist']]['enable']=='yes':
00871 cm_kernel.print_for_con('')
00872 cm_kernel.print_for_con('********************* Finish using Universal Alchemist ...')
00873
00874 if os.path.isfile(uni_alchemist_tmp2):
00875 r=cm_kernel.load_json_file({'cm_filename':uni_alchemist_tmp2})
00876 if r['cm_return']>0: return r
00877 uao=r['cm_array']
00878
00879 if ii_init.get('keep_all_files','')!='yes':
00880 if d_tmp2 and os.path.isfile(uni_alchemist_tmp2):
00881 os.remove(uni_alchemist_tmp2)
00882 if d_tmp3 and os.path.isfile(uni_alchemist_tmp3):
00883 os.remove(uni_alchemist_tmp3)
00884 if os.path.isfile(uni_alchemist_tmp1):
00885 os.remove(uni_alchemist_tmp1)
00886
00887 if len(uao)>0:
00888 state[pu['finish_universal_alchemist']]={}
00889 state[pu['finish_universal_alchemist']]['input']={}
00890 state[pu['finish_universal_alchemist']]['output']={}
00891 state[pu['finish_universal_alchemist']]['output']['raw_plugin_output']=uao
00892
00893
00894 structure={}
00895 if state[pu['use_ctuning_milepost']]['enable']=='yes':
00896 cm_kernel.print_for_con('')
00897 cm_kernel.print_for_con('********************* Finish using cTuning CC / MILEPOST GCC ...')
00898
00899 if ii_init.get('keep_all_files','')!='yes':
00900 if os.path.isfile(file_with_funcs):
00901 os.remove(file_with_funcs)
00902
00903
00904 dirList=os.listdir('.')
00905 for fn in dirList:
00906 if os.path.isfile(fn):
00907 if fn.startswith(ici_structure):
00908 i1=fn.find('.')
00909 if i1>0:
00910 i2=fn.find('.',i1+1)
00911 if i2>0:
00912 func=fn[i1+1:i2]
00913 if func not in structure:
00914 structure[func]={}
00915 structure[func]['compiler_passes']=[]
00916 f=open(fn,'rt')
00917 l='x'
00918 while (l != "" ):
00919 l = f.readline().strip()
00920 structure[func]['compiler_passes'].append(l)
00921 f.close()
00922 elif fn.startswith(ici_properties):
00923 i1=fn.find('.')
00924 if i1>0:
00925 i2=fn.find('.',i1+1)
00926 if i2>0:
00927 func=fn[i1+1:i2]
00928 i3=fn.find('.',i2+1)
00929 if i3>0:
00930 ps=fn[i2+1:i3]
00931 if func not in structure:
00932 structure[func]={}
00933 if 'program_static_properties' not in structure[func]:
00934 structure[func]['program_static_properties']={}
00935 if ps not in structure[func]['program_static_properties']:
00936 structure[func]['program_static_properties'][ps]={}
00937 f=open(fn,'rt')
00938 l = f.readline().strip()
00939 y=-1
00940 while True:
00941 y=l.find('ft', y+1)
00942 if y>=0:
00943 i1=l.find('=',y+1)
00944 if i1>0:
00945 n=l[y+2:i1]
00946 i2=l.find(',',i1+1)
00947 if i2==-1: i2=len(l)
00948 v=l[i1+1:i2]
00949 structure[func]['program_static_properties'][ps][n]=v
00950 else: break
00951 else: break
00952 f.close()
00953 if len(structure)>0:
00954 state[pu['finish_ctuning_milepost']]={}
00955 state[pu['finish_ctuning_milepost']]['input']={}
00956 state[pu['finish_ctuning_milepost']]['output']={}
00957 state[pu['finish_ctuning_milepost']]['output']['program_structure']=structure
00958
00959 executable=state[pu['build_program']]['output'].get('target_file','')
00960
00961
00962 if state[pu['run_program']]['enable']=='yes':
00963 cm_kernel.print_for_con('')
00964 cm_kernel.print_for_con('********************* Preparing run program ...')
00965
00966 ii=jj
00967 ii.update(r.get('run_time',{}))
00968 if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
00969
00970 if state[pu['run_program']]['input'].get('web_run_cmd1','')!='':
00971 x=ii.get('run_cmd1','')
00972 y=state[pu['run_program']]['input']['web_run_cmd1']
00973 ii['run_cmd1']=y+' '+x
00974
00975 if state[pu['build_program']].get('input',{}).get('compilation_type','')=='dynamic':
00976
00977
00978 if target_os_cfg.get('remote','')!='yes':
00979 if 'code_deps' not in ii: ii['code_deps']=[]
00980 ii['code_deps'].append({"CM_CODE_DEP_COMPILER":ii_init['compiler_code_uoa']})
00981 if ii_init.get('accelerator_compiler_code_uoa','')!='':
00982
00983 gfound=False
00984 for g in ii['code_deps']:
00985 g1=g.keys()[0]
00986 g2=g[g1]
00987
00988 if g1=='CM_CODE_DEP_ACC_COMPILER' and g2==ii_init['accelerator_compiler_code_uoa']:
00989 gfound=True
00990 break
00991 if not gfound:
00992 ii['code_deps'].append({"CM_CODE_DEP_ACC_COMPILER":ii_init['accelerator_compiler_code_uoa']})
00993
00994
00995 if state[pu['prepare_vtune']]['enable']=='yes':
00996 cm_kernel.print_for_con('')
00997 cm_kernel.print_for_con('********************* Preparing vtune ...')
00998
00999 if 'code_deps' not in ii: ii['code_deps']=[]
01000 vtune_code_deps={"CM_CODE_DEP_VTUNE":state[pu['prepare_vtune']]['input'].get('vtune_code_uoa','')}
01001 ii['code_deps'].append(vtune_code_deps)
01002
01003 x=ii.get('run_cmd1','')
01004 x='amplxe-runsa -event-based-counts -- '+x
01005
01006 ii['run_cmd1']=x
01007
01008 r=cm_kernel.gen_uid({})
01009 if r['cm_return']>0: return r
01010 vtune_tmp='tmp-vtune-stat-'+r['cm_uid']+'.csv'
01011
01012
01013 executable1=executable
01014 if executable.startswith('./'):
01015 executable1=executable[2:]
01016 vtune_process_cmd='amplxe-cl -report hw-events -r r000 -report-output='+vtune_tmp+' -format csv -csv-delimiter=comma -filter module='+executable1
01017
01018
01019
01020
01021 if len(ii.get('run_output_files',[]))==0: ii['run_output_files']=[]
01022 if vtune_tmp not in ii['run_output_files']: ii['run_output_files'].append(vtune_tmp)
01023
01024
01025 if state[pu['prepare_perf']]['enable']=='yes':
01026 cm_kernel.print_for_con('')
01027 cm_kernel.print_for_con('********************* Preparing perf ...')
01028
01029
01030 stderr=state[pu['prepare_dataset']].get('output',{}).get('run_time',{}).get('run_cmd_out2','')
01031 if stderr=='':
01032 cm_kernel.print_for_con('')
01033 cm_kernel.print_for_con(' STDERR output is not defined in a program - required to detect perf variables!')
01034 else:
01035 y=ii.get('run_cmd1','')
01036 x=target_os_cfg.get('cm_tool_prefix','')+'perf stat -x, '
01037 if state[pu['prepare_perf']]['input'].get('support_output_to_file','')=='yes':
01038 r=cm_kernel.gen_uid({})
01039 if r['cm_return']>0: return r
01040 perf_tmp='tmp-perf-stat-'+r['cm_uid']+'.txt'
01041 if len(ii.get('run_output_files',[]))==0: ii['run_output_files']=[]
01042 if perf_tmp not in ii['run_output_files']: ii['run_output_files'].append(perf_tmp)
01043
01044 x+='--output '+perf_tmp
01045 x+=y
01046
01047 ii['run_cmd1']=x
01048
01049
01050 if state[pu['use_compiler_profiling']]['enable']=='yes':
01051 cm_kernel.print_for_con('')
01052 cm_kernel.print_for_con('********************* Using compiler profiling ...')
01053
01054 prof_files=ct_compiler_desc.get('profile',{}).get('files',[])
01055 if len(prof_files)>0:
01056 if len(ii.get('run_output_files',[]))==0: ii['run_output_files']=[]
01057 for g in prof_files:
01058 if g not in ii['run_output_files']: ii['run_output_files'].append(g)
01059
01060
01061 if state[pu['set_cpu_frequency']]['enable']=='yes':
01062 cm_kernel.print_for_con('')
01063 cm_kernel.print_for_con('********************* Setting cpu frequency ...')
01064
01065 freq=state[pu['set_cpu_frequency']]['input'].get('frequency_str','')
01066 if freq=='':
01067 freq=state[pu['set_cpu_frequency']]['input'].get('frequency','')
01068 if freq=='':
01069 return {'cm_return':1,'cm_error':'frequency is not defined'}
01070
01071 jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['system'],
01072 'cm_action':'set_cpu_frequency',
01073 'host_os_uoa':ii_init["run_host_os_uoa"],
01074 'target_os_uoa':state[pu['run_program']]['input']['run_target_os_uoa'],
01075 'cpu_frequency':state[pu['set_cpu_frequency']]['input']['frequency'],
01076 'use_sudo':state[pu['set_cpu_frequency']]['input'].get('use_sudo','')}
01077 r=update_input_from_state_ll(jj, state, pu['set_cpu_frequency'], True)
01078 if r['cm_return']>0: return r
01079
01080
01081 if state[pu['get_system_state']]['enable']=='yes':
01082 cm_kernel.print_for_con('')
01083 cm_kernel.print_for_con('********************* Getting system state ...')
01084
01085 jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['system'],
01086 'cm_action':'get_state',
01087 'host_os_uoa':ii_init["run_host_os_uoa"],
01088 'target_os_uoa':state[pu['run_program']]['input']['run_target_os_uoa']}
01089 r=cm_kernel.access(jj)
01090 if r['cm_return']>0: return r
01091
01092 ss=r['system_state']
01093
01094 state[pu['get_system_state']]['output']=r
01095
01096 cm_kernel.print_for_con(json.dumps(r, indent=2, sort_keys=True))
01097
01098
01099 cm_kernel.print_for_con('')
01100 cm_kernel.print_for_con('********************* Running program ...')
01101
01102
01103
01104
01105 if target_os_cfg.get('remote','')!='yes':
01106 if len(state[pu['build_program']]['input'].get('cm_dependencies',{}))>0:
01107 ii['cm_dependencies']=state[pu['build_program']]['input']['cm_dependencies']
01108
01109 ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['code']
01110 ii['cm_action']='run'
01111 ii['work_dir_module_uoa']=ii_init['work_dir_module_uoa']
01112 ii['work_dir_data_uoa']=ii_init['work_dir_data_uoa']
01113 ii['work_dir_repo_uoa']=ii_init.get('work_dir_repo_uoa','')
01114 ii['keep_all_files']=ii_init.get('keep_all_files','')
01115
01116
01117 if state[pu['use_openme']]['enable']=='yes':
01118 if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
01119 ii['run_set_env2']['OPENME_USE']='1'
01120
01121
01122 jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['code'],
01123 'cm_action':'load',
01124 'cm_data_uoa':state[pu['use_openme']]['input'].get('openme_library_ctuning_plugin','')}
01125 r=cm_kernel.access(jj)
01126 if r['cm_return']>0: return r
01127 d=r['cm_data_obj']['cfg']
01128 p=r['cm_path']
01129 plugin=d.get('state_input',{}).get('run_set_env2',{}).get('CM_TARGET_FILE','')
01130 lib_dir=d.get('state_input',{}).get('run_set_env2',{}).get('CM_OS_LIB_DIR','')
01131 p1=os.path.join(p,lib_dir,plugin)
01132
01133 p2=p1
01134 if target_os_cfg.get('remote','')=='yes':
01135 if len(ii.get('run_input_files',[]))==0: ii['run_input_files']=[]
01136 if p1 not in ii['run_input_files']: ii['run_input_files'].append(p1)
01137 p2=target_os_cfg.get('exec_prefix','')+plugin
01138
01139 ii['run_set_env2']['OPENME_PLUGINS']=p2
01140
01141 r=cm_kernel.gen_uid({})
01142 if r['cm_return']>0: return r
01143 openme_tmp='tmp-openme-stat-'+r['cm_uid']+'.json'
01144
01145 ii['run_set_env2']['OPENME_OUTPUT_FILE']=openme_tmp
01146
01147 if len(ii.get('run_output_files',[]))==0: ii['run_output_files']=[]
01148 if openme_tmp not in ii['run_output_files']: ii['run_output_files'].append(openme_tmp)
01149
01150
01151 calibrate=state[pu['run_program']]['input'].get('calibrate','')
01152
01153
01154 if 'run_vars_desc' not in program_info or \
01155 '##CT_REPEAT_MAIN' not in program_info['run_vars_desc']:
01156 calibrate='no'
01157 cm_kernel.print_for_con('')
01158 cm_kernel.print_for_con('Calibration is not described in the source code - hence, explictly disabled here ...')
01159 cm_kernel.print_for_con('')
01160
01161 calibrate_max_iter=int(state[pu['run_program']]['input'].get('calibrate_max_iters','1'))
01162 calibrate_time=float(state[pu['run_program']]['input'].get('calibrate_time','0'))
01163
01164
01165 omp_nt=state[pu['run_program']]['input'].get('openmp_number_of_threads','')
01166 if omp_nt!='':
01167 if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
01168 ii['run_set_env2']['OMP_NUM_THREADS']=omp_nt
01169
01170
01171 if state.get(pu['use_likwid'],{}).get('enable','')=='yes':
01172 cm_kernel.print_for_con('')
01173 cm_kernel.print_for_con('********************* Prefexing likwid ...')
01174
01175 if len(ii.get('code_deps',[]))==0: ii['code_deps']=[]
01176 ii['code_deps'].append({"CM_CODE_DEP_LIKWID":lhc_uoa})
01177
01178 lhc=state[pu['use_likwid']]['input']
01179 lhc_uoa=lhc.get('likwid_code_uoa','')
01180
01181 y=ii.get('run_cmd1','')
01182
01183 x=''
01184 if lhc.get('use_sudo','')=='yes':
01185 x+='sudo -E '
01186
01187 x+=' likwid-perfctr -C '+lhc.get('processor_pinning','')
01188 x+=' -g '+lhc.get('measure_performance_group','')+' '+lhc.get('measure_event_set','')+' '
01189
01190 if lhc.get('instrumentation','')=='yes':
01191 x+=' -m '
01192
01193 r=cm_kernel.gen_uid({})
01194 if r['cm_return']>0: return r
01195 likwid_tmp='tmp-perf-stat-'+r['cm_uid']+'.txt'
01196 if len(ii.get('run_output_files',[]))==0: ii['run_output_files']=[]
01197 if likwid_tmp not in ii['run_output_files']: ii['run_output_files'].append(likwid_tmp)
01198
01199 x+='-o '+likwid_tmp+' '
01200 x+=y
01201
01202 ii['run_cmd1']=x
01203
01204
01205 ii_copy=copy.deepcopy(ii)
01206 state_copy=copy.deepcopy(state)
01207 rt=0
01208 repeat=int(-1)
01209 failed=False
01210 calibrate_success=False
01211 run_output_files=[]
01212
01213
01214 if 'CT_REPEAT_MAIN' in state[pu['run_program']]['input'].get('run_vars',{}):
01215 repeat=int(state[pu['run_program']]['input']['run_vars']['CT_REPEAT_MAIN'])
01216 if calibrate!='yes' or repeat==-1: calibrate_max_iter=1
01217
01218
01219 if calibrate=='yes' and \
01220 state[pu['run_program']].get('output',{}).get('calibration_failed','')=='no' and \
01221 state[pu['run_program']].get('output',{}).get('ct_repeat_main','')!='':
01222
01223 calibrate_max_iter=1
01224 repeat=-1
01225 state[pu['run_program']]['input']['run_vars']['CT_REPEAT_MAIN']=state[pu['run_program']]['output']['ct_repeat_main']
01226
01227 for iter in range(0, calibrate_max_iter):
01228
01229 if state[pu['prepare_vtune']]['enable']=='yes':
01230 jj={}
01231 jj['cm_run_module_uoa']=ini['cfg']['cm_modules']['code']
01232 jj['cm_action']='change_work_dir'
01233 jj['work_dir_module_uoa']=ii_init['work_dir_module_uoa']
01234 jj['work_dir_data_uoa']=ii_init['work_dir_data_uoa']
01235 jj['work_dir_repo_uoa']=ii_init.get('work_dir_repo_uoa','')
01236 r=cm_kernel.access(jj)
01237 if r['cm_return']>0: return r
01238
01239
01240 dirList=os.listdir('.')
01241 for fn in dirList:
01242 if len(fn)>3 and fn[0:1]=='r' and fn[1:2].isdigit() and fn[2:3].isdigit() and fn[3:4].isdigit():
01243 cm_kernel.print_for_con('Deleting old stat dir '+fn+' ...')
01244 shutil.rmtree(fn, True)
01245
01246 ii=copy.deepcopy(ii_copy)
01247 state=copy.deepcopy(state_copy)
01248
01249 if repeat>0: state[pu['run_program']]['input']['run_vars']['CT_REPEAT_MAIN']=str(repeat)
01250
01251
01252 if len(state[pu['run_program']]['input'].get('run_vars',{}))>0:
01253 ii['run_set_env2'].update(state[pu['run_program']]['input']['run_vars'])
01254
01255 r=update_input_from_state_ll(ii, state, pu['run_program'], True)
01256 if r['cm_return']>0: return r
01257
01258 run_output_files=r.get('run_output_files',[])
01259 state[pu['run_program']]['output']['run_output_files']='yes'
01260
01261 if r.get('timed_out', '')!='yes':
01262 rt=float(r.get('run_time_by_module','0'))
01263
01264 if calibrate!='yes' or repeat==-1: break
01265
01266 orepeat=repeat
01267 if rt<0.5: repeat*=10
01268 elif 0.8<(calibrate_time/rt)<1.4:
01269 calibrate_success=True
01270 break
01271 else:
01272 repeat*=float(calibrate_time/rt)
01273 repeat=int(repeat)
01274
01275 if repeat==orepeat or repeat<1:
01276 calibrate_success=True
01277 break
01278
01279 cm_kernel.print_for_con('')
01280 cm_kernel.print_for_con('### Calibration: time='+str(rt)+'; CT_REPEAT_MAIN='+str(orepeat)+'; new CT_REPEAT_MAIN='+str(repeat))
01281 else:
01282 state[pu['run_program']]['output']['fail']='yes'
01283 failed=True
01284 break
01285
01286 if calibrate=='yes' and not calibrate_success:
01287 state[pu['run_program']]['output']['fail']='yes'
01288 state[pu['run_program']]['output']['calibration_failed']='yes'
01289
01290 cm_kernel.print_for_con('')
01291 cm_kernel.print_for_con('Problem: calibration failed - exiting pipeline!')
01292
01293 pipeline_fail=True
01294
01295 elif failed or rt<0.05:
01296 state[pu['run_program']]['output']['fail']='yes'
01297
01298 if calibrate=='yes':
01299 state[pu['run_program']]['output']['calibration_failed']='no'
01300 state[pu['run_program']]['output']['ct_repeat_main']=str(repeat)
01301
01302 cm_kernel.print_for_con('')
01303 cm_kernel.print_for_con('Problem: execution failed - exiting pipeline!')
01304
01305 pipeline_fail=True
01306
01307 else:
01308
01309 if repeat>0:
01310 state[pu['run_program']]['output']['number_of_kernel_repetitions']=str(repeat)
01311 state[pu['run_program']]['output']['global_execution_time_div_by_repeat']=str(rt/repeat)
01312 else:
01313 state[pu['run_program']]['output']['number_of_kernel_repetitions']='1'
01314 state[pu['run_program']]['output']['global_execution_time_div_by_repeat']=str(rt)
01315
01316 state['cm_characteristics']['kernel_repeat']=state[pu['run_program']]['output']['number_of_kernel_repetitions']
01317
01318
01319
01320
01321 output_correct=True
01322 if state[pu['check_output']]['enable']=='yes' and (calibrate!='yes' or calibrate_success):
01323 cm_kernel.print_for_con('')
01324 cm_kernel.print_for_con('Checking output ...')
01325
01326 dataset_output_dir=ii_init['dataset_uoa']
01327 if dataset_output_dir=='':
01328 dataset_output_dir='default'
01329
01330 dir_output=ini['cfg']['dir_output']+'.'+dataset_output_dir+'.'+str(repeat)
01331
01332
01333 if os.path.isdir(dir_output):
01334
01335
01336 for x in run_correctness_output_files:
01337 d1=os.path.join(dir_output, x)
01338 if not os.path.isfile(x) or not os.path.isfile(d1) or not filecmp.cmp(x, d1):
01339 output_correct=False
01340 break
01341
01342 else:
01343
01344 os.makedirs(dir_output)
01345
01346 for x in run_correctness_output_files:
01347 if os.path.isfile(x):
01348 d1=os.path.join(dir_output, x)
01349 shutil.copyfile(x, d1)
01350
01351 state[pu['check_output']]['output']={}
01352 oc=0
01353 if not output_correct:
01354 oc=1
01355 cm_kernel.print_for_con('Fail!')
01356 else:
01357 cm_kernel.print_for_con('Ok!')
01358
01359 state[pu['check_output']]['output']['output_correct']=str(oc)
01360
01361 if not output_correct:
01362 pipeline_fail=True
01363 else:
01364
01365 if state.get(pu['use_likwid'],{}).get('enable','')=='yes':
01366 cm_kernel.print_for_con('')
01367 cm_kernel.print_for_con('********************* Finishing likwid ...')
01368
01369 state[pu['finish_likwid']]={}
01370 state[pu['finish_likwid']]['input']={}
01371 state[pu['finish_likwid']]['output']={}
01372
01373 if os.path.isfile(likwid_tmp):
01374 cm_kernel.print_for_con('Parsing output ...')
01375
01376 lik={}
01377 st=''
01378 f=open(likwid_tmp,'rt')
01379 lx="x"
01380 region=''
01381 found=False
01382 while (lx != "" ):
01383 lx = f.readline()
01384 l=lx.strip()
01385 l1=l.lower()
01386 if st=='':
01387 if l.startswith('---'): st='a'
01388 elif st=='a':
01389 if l.startswith('---'): st='b'
01390 lik['general']={}
01391 elif st=='b':
01392 if l.startswith('---'):
01393 if lhc.get('instrumentation','')=='yes':
01394 st='c'
01395 else:
01396 st='e'
01397 region='all'
01398 lik['by_region']={}
01399 lik['by_region'][region]={}
01400 lik['by_region'][region]['info']={}
01401 lik['by_region'][region]['event']={}
01402 lik['by_region'][region]['metric']={}
01403 else:
01404 a=l.find(':')
01405 if i>0:
01406 r=filter_special_str({'cm_string':l1[:a]})
01407 k=r['cm_string']
01408 v=l[a:].replace('\t','').strip()
01409 lik['general'][k]=v
01410 elif st=='c':
01411 if l1.startswith('region:'):
01412 st='d'
01413 region=l1[7:].strip()
01414 lik['by_region']={}
01415 lik['by_region'][region]={}
01416 lik['by_region'][region]['info']={}
01417 lik['by_region'][region]['event']={}
01418 lik['by_region'][region]['metric']={}
01419 elif st=='d':
01420 if l1.find('region info')>0:
01421 st='e'
01422 cores=l1.split('|')
01423 elif st=='e':
01424 if l1.find('event')>0:
01425 st='f'
01426 else:
01427 a=l.split('|')
01428 if len(a)>2:
01429 r=filter_special_str({'cm_string':a[1]})
01430 k=r['cm_string'].lower()
01431 v=a[2].replace('\t','').strip()
01432 lik['by_region'][region]['info'][k]=v
01433 elif st=='f':
01434 if l1.find('metric')>0:
01435 st='g'
01436 else:
01437 a=l.split('|')
01438 if len(a)>2:
01439 r=filter_special_str({'cm_string':a[1]})
01440 k=r['cm_string'].lower()
01441 v=a[2].replace('\t','').strip()
01442 lik['by_region'][region]['event'][k]=v
01443 if 'cm_data_description' not in state: state['cm_data_description']={}
01444 state['cm_data_description']['##'+pu['finish_likwid']+'#output#global#by_region#'+region+'#event#'+k]={'characteristic':'yes','stat_characteristic':'yes','type':'float'}
01445 elif st=='g':
01446 a=l.split('|')
01447 if len(a)>2:
01448 r=filter_special_str({'cm_string':a[1]})
01449 k=r['cm_string'].lower()
01450 v=a[2].replace('\t','').strip()
01451 lik['by_region'][region]['metric'][k]=v
01452 if 'cm_data_description' not in state: state['cm_data_description']={}
01453 state['cm_data_description']['##'+pu['finish_likwid']+'#output#global#by_region#'+region+'#metric#'+k]={'characteristic':'yes','stat_characteristic':'yes','type':'float'}
01454 f.close()
01455
01456
01457 scpi=lik.get('by_region',{}).get(region,{}).get('metric',{}).get('cpi','')
01458 if scpi!='':
01459 state['cm_characteristics']['global_cpi']=scpi
01460 state['cm_characteristics']['global_instructions_retired']=lik.get('by_region',{}).get(region,{}).get('event',{}).get('instr_retired_any','')
01461
01462 state[pu['finish_likwid']]['output']['global']=lik
01463
01464
01465
01466 if state[pu['use_openme']]['enable']=='yes':
01467 cm_kernel.print_for_con('')
01468 cm_kernel.print_for_con('********************* Finishing OpenME ...')
01469
01470 state[pu['finish_openme']]={}
01471 state[pu['finish_openme']]['input']={}
01472 state[pu['finish_openme']]['output']={}
01473
01474 if os.path.isfile(openme_tmp):
01475
01476 r=cm_kernel.load_json_file({'cm_filename':openme_tmp})
01477 if r['cm_return']>0: return r
01478
01479 state[pu['finish_openme']]['output'].update(r['cm_array'])
01480
01481
01482 if state[pu['prepare_perf']]['enable']=='yes':
01483 cm_kernel.print_for_con('')
01484 cm_kernel.print_for_con('********************* Finishing perf ...')
01485
01486 state[pu['finish_perf']]={}
01487 state[pu['finish_perf']]['input']={}
01488 state[pu['finish_perf']]['output']={}
01489
01490 skey=',task-clock'
01491 clk='cycles'
01492 inst='instructions'
01493 val={}
01494
01495 fin=stderr
01496 if state[pu['prepare_perf']]['input'].get('support_output_to_file','')=='yes':
01497 fin=perf_tmp
01498
01499 if os.path.isfile(fin):
01500 f=open(fin,'rt')
01501 lx="x"
01502 found=False
01503 while (lx != "" ):
01504 lx = f.readline()
01505 l=lx.strip()
01506 if found:
01507 x=l.find(',')
01508 if x>0:
01509 v=l[0:x]
01510 key=l[x+1:]
01511 val[key]=v
01512 elif l.find(skey)>0:
01513 found=True
01514
01515 f.close()
01516
01517 if len(val)>0:
01518 if clk in val and inst in val:
01519 cpi=0
01520 try:
01521 cpi=float(val[clk])/float(val[inst])
01522 except ValueError:
01523 pass
01524 if cpi!=0:
01525 state['cm_characteristics']['global_cpi']="%.3f" % cpi
01526 state['cm_characteristics']['global_instructions_retired']=val[inst]
01527
01528 state[pu['finish_perf']]['output']['global']={}
01529 for q in val:
01530 state[pu['finish_perf']]['output']['global'][q]=val[q]
01531
01532
01533 if state[pu['use_compiler_profiling']]['enable']=='yes':
01534 cm_kernel.print_for_con('')
01535 cm_kernel.print_for_con('********************* Finishing compiler profiling ...')
01536
01537 state[pu['finish_compiler_profiling']]={}
01538 state[pu['finish_compiler_profiling']]['input']={}
01539 state[pu['finish_compiler_profiling']]['output']={}
01540
01541 if 'profiler' in compiler_code_desc:
01542 r=cm_kernel.gen_uid({})
01543 if r['cm_return']>0: return r
01544 prof_tmp='tmp-prof-'+r['cm_uid']+'.txt'
01545
01546 cmd=compiler_code_desc['profiler']
01547 if executable!='a.out': cmd+=' '+executable
01548 cmd+=' > '+ prof_tmp
01549
01550 kk=copy.deepcopy(ii_init)
01551 kk['run_cmd']=cmd
01552 kk['cm_action']='run'
01553 kk['cm_run_module_uoa']=ini['cfg']['cm_modules']['code']
01554 kk['code_deps']=[{"CM_CODE_DEP_COMPILER":ii_init['compiler_code_uoa']}]
01555
01556 r=cm_kernel.access(kk)
01557 if r['cm_return']>0: return r
01558
01559 if os.path.isfile(prof_tmp):
01560 f=open(prof_tmp, 'rb')
01561 s=f.read()
01562 f.close()
01563
01564 state[pu['finish_compiler_profiling']]['output']['raw_output']=s
01565
01566
01567 if state[pu['prepare_vtune']]['enable']=='yes':
01568 cm_kernel.print_for_con('')
01569 cm_kernel.print_for_con('********************* Finishing vtune ...')
01570
01571
01572 jj={}
01573 jj['cm_run_module_uoa']=ini['cfg']['cm_modules']['code']
01574 jj['cm_action']='run'
01575 jj['work_dir_module_uoa']=ii_init['work_dir_module_uoa']
01576 jj['work_dir_data_uoa']=ii_init['work_dir_data_uoa']
01577 jj['work_dir_repo_uoa']=ii_init.get('work_dir_repo_uoa','')
01578 jj['keep_all_files']=ii_init.get('keep_all_files','')
01579
01580 jj['run_host_os_uoa']=ii_init["run_host_os_uoa"]
01581 jj['run_target_os_uoa']=ii_init["run_target_os_uoa"]
01582 jj['run_target_processor_uoa']=ii_init["run_target_processor_uoa"]
01583
01584 jj['code_deps']=[]
01585 jj['code_deps'].append(vtune_code_deps)
01586 jj['run_cmd']=vtune_process_cmd
01587
01588 r=cm_kernel.access(jj)
01589 if r['cm_return']>0: return r
01590
01591
01592 state[pu['finish_vtune']]={}
01593 state[pu['finish_vtune']]['input']={}
01594 state[pu['finish_vtune']]['output']={}
01595
01596 clk='CPU_CLK_UNHALTED.THREAD:Hardware Event Count'
01597 clk1='CPU_CLK_UNHALTED.CORE:Hardware Event Count'
01598 inst='INST_RETIRED.ANY:Hardware Event Count'
01599
01600 if os.path.isfile(vtune_tmp):
01601 f=open(vtune_tmp, 'rb')
01602 c=csv.reader(f, delimiter=',')
01603
01604 hc=[]
01605 val={}
01606
01607 first=True
01608 for q in c:
01609 if first:
01610 first=False
01611 if len(q)>1:
01612 for k in range(2,len(q)):
01613 hc.append(q[k])
01614 else:
01615 func=q[0]
01616 module=q[1]
01617
01618 if (func=='['+executable1+']' or func=='main') and module==executable1:
01619 if len(q)>1:
01620 for k in range(2,len(q)):
01621 val[hc[k-2]]=q[k]
01622
01623 f.close()
01624 if ii_init.get('keep_all_files','')!='yes':
01625 os.remove(vtune_tmp)
01626
01627 if len(val)>0:
01628 if (clk in val or clk1 in val) and inst in val:
01629 clkx=clk
01630 if clk1 in val: clkx=clk1
01631 cpi=0
01632 try:
01633 cpi=float(val[clkx])/float(val[inst])
01634 except ValueError:
01635 pass
01636 if cpi!=0:
01637 state['cm_characteristics']['global_cpi']="%.3f" % cpi
01638 state['cm_characteristics']['global_instructions_retired']=val[inst]
01639
01640 state[pu['finish_vtune']]['output']['global']={}
01641 for q in val:
01642 state[pu['finish_vtune']]['output']['global'][q]=val[q]
01643
01644 fail='no'
01645 if pipeline_fail:
01646 fail='yes'
01647 else:
01648
01649 cm_kernel.print_for_con('')
01650 cm_kernel.print_for_con('********************* Finalizing pipeline ')
01651
01652
01653 r=cm_kernel.get_current_date_time({})
01654 if r['cm_return']>0: return r
01655 state['misc']['cm_iso_datetime_finished']=r['cm_iso_datetime']
01656
01657 if i.get('cm_show_output_json','')=='yes':
01658 cm_kernel.print_for_web(json.dumps(state,indent=2,sort_keys=True))
01659 of=i.get('cm_save_output_json_to_file','')
01660 if of!='':
01661 r4=cm_kernel.save_array_to_file_as_json({'cm_filename':of, 'cm_array':state})
01662 if r4['cm_return']>0: return r4
01663 cm_kernel.print_for_web("")
01664 cm_kernel.print_for_web("Output saved to "+of)
01665
01666 state[pu['finish']]['output']['pipeline_fail']=fail
01667
01668
01669 x=state['cm_characteristics'].get('global_run_kernel_time_min','')
01670 if x=='' or x=='-1':
01671 x=state.get(pu['run_program'],{}).get('output',{}).get('global_execution_time_div_by_repeat','')
01672 state['cm_characteristics']['global_run_kernel_time_min']=x
01673 state['cm_characteristics']['global_run_kernel_time_max']=x
01674 x=state.get(pu['build_program'],{}).get('output',{}).get('run_time_by_module','')
01675 state['cm_characteristics']['global_build_time_min']=x
01676 state['cm_characteristics']['global_build_time_max']=x
01677
01678
01679 if state.get(pu['save_in_ctuning_space'],{}).get('input',{}).get('save_in_existing_space','')!='' or \
01680 state.get(pu['save_in_ctuning_space'],{}).get('input',{}).get('save_in_existing_space_text','')!='':
01681
01682 cts=state[pu['save_in_ctuning_space']]['input'].get('save_in_existing_space_text','')
01683 if cts=='': cts=state[pu['save_in_ctuning_space']]['input'].get('save_in_existing_space','')
01684
01685 cm_kernel.print_for_con('')
01686 cm_kernel.print_for_con('Recording in cTuning space ('+cts+') ...')
01687
01688 dcts={}
01689
01690 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.space'],
01691 'cm_action':'load',
01692 'cm_data_uoa':cts}
01693 if ii_init['work_dir_repo_uoa']!='': ii['cm_repo_uoa']=ii_init['work_dir_repo_uoa']
01694 rx=cm_kernel.access(ii)
01695 if rx['cm_return']==0:
01696 dcts=rx['cm_data_obj']['cfg']
01697
01698
01699 if 'points' not in dcts:
01700 dcts['points']=[]
01701
01702 dcts['points'].append({'pipeline_input':{'cm_action':'run', \
01703 'cm_run_module_uoa':ini['cm_module_uid'], \
01704 'state':state_copy}, \
01705 'pipeline_output':{'cm_return':0, \
01706 'fail':pipeline_fail, \
01707 'state':state}})
01708
01709
01710 dcts['pipeline_module_uoa']=ini['cm_module_uid']
01711
01712 ni=int(dcts.get('number_of_explored_points','0'))
01713 ni+=1
01714 dcts['number_of_explored_points']=str(ni)
01715
01716
01717 r=prepare_summary_point({'state':state})
01718 if r['cm_return']>0: return r
01719 sum1=r['summary_point1']
01720 sum2=r['summary_point2']
01721
01722 dcts['summary_point']=sum1
01723 dcts['summary_point_relaxed']=sum2
01724
01725
01726 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.space'],
01727 'cm_action':'update',
01728 'cm_data_uoa':cts,
01729 'cm_array':dcts}
01730 if ii_init['work_dir_repo_uoa']!='': ii['cm_repo_uoa']=ii_init['work_dir_repo_uoa']
01731 rx=cm_kernel.access(ii)
01732 if rx['cm_return']>0: return rx
01733
01734 if not pipeline_fail:
01735
01736 cm_kernel.print_for_con('Number of kernel repetitions: '+state[pu['run_program']]['output'].get('number_of_kernel_repetitions',''))
01737 cm_kernel.print_for_con('Global execution time divided by repeat number: '+state[pu['run_program']]['output'].get('global_execution_time_div_by_repeat','')+' sec.')
01738
01739 cm_kernel.print_for_con('')
01740 cm_kernel.print_for_con('Pipeline execution finished!')
01741
01742 return {'cm_return':0, 'state':state, 'fail':pipeline_fail}
01743
01744
01745 def run_stat(i):
01746 """
01747 Run collective tuning pipeline with statistical analysis of variability
01748
01749 Input: {
01750 state - init state
01751 desc - pipeline description (needed to know characteristics)
01752
01753 additional_params - scenario parameters dict
01754 "repeat_for_statistics" - number of repetitions for statistical analysis
01755 }
01756
01757 Output: {
01758 cm_return - if =0, success
01759 state - final state
01760 }
01761 """
01762
01763 ap=i.get('additional_params',{})
01764 state=i.get('state',{})
01765 desc=i.get('desc',{})
01766
01767 pu=ini['cfg']['cm_p_uids']
01768
01769
01770 ctsx=state.get(pu['save_in_ctuning_space'],{}).get('input',{}).get('save_in_existing_space','')
01771 cts=state.get(pu['save_in_ctuning_space'],{}).get('input',{}).get('save_in_existing_space_text','')
01772
01773
01774 if pu['save_in_ctuning_space'] not in state: state[pu['save_in_ctuning_space']]={}
01775 if 'input' not in state[pu['save_in_ctuning_space']]: state[pu['save_in_ctuning_space']]['input']={}
01776 state[pu['save_in_ctuning_space']]['input']['save_in_existing_space']=''
01777 state[pu['save_in_ctuning_space']]['input']['save_in_existing_space_text']=''
01778
01779 state_copy=copy.deepcopy(state)
01780
01781 rstat=int(ap.get('repeat_for_statistics','1'))
01782 if rstat<1: rstat=1
01783
01784 failed=False
01785 rstat1=0
01786 while rstat1<rstat and not failed:
01787 rstat1+=1
01788 cm_kernel.print_for_con('')
01789 cm_kernel.print_for_con('!!!!!!!! Repetitions for statistics: '+str(rstat1)+' of '+str(rstat))
01790
01791 r=run({'state':state})
01792 if r['cm_return']>0: return r
01793
01794 state=r['state']
01795
01796 if r['fail']:
01797 failed=True
01798 break
01799
01800
01801 mstat=ap.get('statistics_module','')
01802 if mstat!='':
01803
01804
01805
01806 cdl=state.get('cm_data_description',{})
01807 if len(cdl)>0: desc.update(cdl)
01808
01809 rx=cm_kernel.access({'cm_run_module_uoa':mstat,
01810 'cm_action':'process',
01811 'state':state,
01812 'desc':desc})
01813 if rx['cm_return']>0: return rx
01814
01815 state=rx['state']
01816 r['state']=state
01817
01818
01819 csc=state.get('cm_stat_characteristics',{})
01820
01821 pb=pu['build_program']
01822 x=csc.get(pb,{}).get('output',{}).get('run_time_by_module',{})
01823 state['cm_characteristics']['global_build_time_min']=x.get('min','')
01824 state['cm_characteristics']['global_build_time_max']=x.get('max','')
01825
01826 pr=pu['run_program']
01827 x=csc.get(pr,{}).get('output',{}).get('global_execution_time_div_by_repeat',{})
01828 state['cm_characteristics']['global_run_kernel_time_min']=x.get('min','')
01829 state['cm_characteristics']['global_run_kernel_time_max']=x.get('max','')
01830
01831
01832 if cts!='' or ctsx!='':
01833 ii_init=state[pu['init']]
01834
01835 if cts=='': cts=ctsx
01836
01837 cm_kernel.print_for_con('')
01838 cm_kernel.print_for_con('Recording in cTuning space ('+cts+') ...')
01839
01840 dcts={}
01841
01842 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.space'],
01843 'cm_action':'load',
01844 'cm_data_uoa':cts}
01845 if ii_init.get('tuning_repo_uoa','')!='': ii['cm_repo_uoa']=ii_init['tuning_repo_uoa']
01846 rx=cm_kernel.access(ii)
01847 if rx['cm_return']==0:
01848 dcts=rx['cm_data_obj']['cfg']
01849
01850
01851 if 'points' not in dcts:
01852 dcts['points']=[]
01853
01854 dcts['points'].append({'pipeline_input':{'cm_action':'run', \
01855 'cm_run_module_uoa':ini['cm_module_uid'], \
01856 'state':state_copy}, \
01857 'pipeline_output':{'cm_return':0, \
01858 'fail':failed, \
01859 'state':state}})
01860
01861
01862 dcts['pipeline_module_uoa']=ini['cm_module_uid']
01863
01864 ni=int(dcts.get('number_of_explored_points','0'))
01865 ni+=1
01866 dcts['number_of_explored_points']=str(ni)
01867
01868
01869 rx=prepare_summary_point({'state':state})
01870 if rx['cm_return']>0: return rx
01871 sum1=rx['summary_point1']
01872 sum2=rx['summary_point2']
01873
01874 dcts['summary_point']=sum1
01875 dcts['summary_point_relaxed']=sum2
01876
01877
01878 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.space'],
01879 'cm_action':'update',
01880 'cm_data_uoa':cts,
01881 'cm_array':dcts}
01882 if ii_init.get('tuning_repo_uoa','')!='': ii['cm_repo_uoa']=ii_init['tuning_repo_uoa']
01883 rx=cm_kernel.access(ii)
01884 if rx['cm_return']>0: return rx
01885
01886 return r
01887
01888
01889 def update_input_from_state_ll(ii, state, p_uid, execute=False):
01890 """
01891 Update input from state
01892
01893 Input: {
01894 }
01895
01896 Output: {
01897 }
01898 """
01899
01900 r={'cm_return':0}
01901
01902 if p_uid in state and 'input' in state[p_uid]:
01903 if state[p_uid].get('enable','')!='no':
01904 cm_kernel.merge_arrays_ll(ii, state[p_uid]['input'])
01905 if execute:
01906 r=cm_kernel.access(ii)
01907
01908 state[p_uid]['output']=r
01909
01910 return r
01911
01912
01913 def web_run(i):
01914
01915 """
01916 Run pipeline through web
01917
01918 Input: See run(i)
01919 Output: See run(i)
01920
01921 """
01922
01923
01924 if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
01925 else:
01926 return {'cm_return':1, 'cm_error':'web style is not defined'}
01927
01928 cm_kernel.print_for_con('<span class="cm-title">Program tuning pipeline</span><br>')
01929
01930
01931 a1={}
01932 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
01933 'cm_action':'detect_form_params',
01934 'cm_array':i,
01935 'cm_prefix':'#form1'})
01936 if r['cm_return']>0: return r
01937 cm_form_array1=r['cm_array']
01938 cm_form_commands1=r['cm_commands']
01939
01940
01941 r=cm_kernel.get_data_description({'cm_module_uoa':ini['cm_module_uoa'],
01942 'cm_which_action':i['cm_action']})
01943 if r['cm_return']>0: return r
01944 cm_params_default1=r['cm_params_default']
01945
01946 cm_data_desc1=ini['cfg']['pipeline_desc']
01947
01948
01949 forms_exists='yes'
01950 if len(cm_form_array1)==0:
01951 a1=cm_params_default1
01952
01953 forms_exists='no'
01954 else:
01955 r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1,
01956 'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
01957 if r['cm_return']>0: return r
01958 a1=r['cm_array']
01959
01960 show_json=a1.get('cm_show_json','')
01961 save_json=a1.get('cm_save_json_to_file','')
01962
01963
01964 last_command={}
01965
01966 run=False
01967 if 'cm_submit_start' in i:
01968 run=True
01969 del(i['cm_submit_start'])
01970
01971
01972 cm_kernel.print_for_con('<br><table border="0"><tr>')
01973
01974 cm_kernel.print_for_con('<td align="center" valign="top">')
01975 cm_kernel.print_for_con('<FORM ACTION="" name="add_edit" METHOD="POST" enctype="multipart/form-data" accept-charset="utf-8">' )
01976
01977
01978 r=show_legend({})
01979 if r['cm_return']>0: return r
01980 cm_kernel.print_for_web(r['cm_html'])
01981
01982
01983 sc=''
01984 at=[]
01985 for q in cm_data_desc1:
01986
01987 if len(q)>2 and '#' not in q[2:] and '@' not in q[2:] and cm_data_desc1[q].get('type')=='dict':
01988 at.append(q)
01989
01990 ati=sorted(at, key=lambda k: (int(cm_data_desc1.get(k,{}).get('sort_index','999999')), cm_data_desc1.get(k,{}).get('desc_text','').lower()))
01991 for q in ati:
01992 qq=cm_data_desc1[q]
01993 if sc!='': sc+=' '
01994 qq1=qq.get('desc_text',q).replace(' ',' ')
01995 sc+='<a href="#'+q+'#">'+qq1+'</a>'
01996
01997 cm_kernel.print_for_web('<small><i><b>Shortcuts:</b> '+sc+'</i></small>')
01998
01999 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
02000 'cm_action':'visualize_data',
02001 'cm_array':a1,
02002 'cm_data_desc':cm_data_desc1,
02003 'cm_form_commands':cm_form_commands1,
02004 'cm_separator':'#',
02005 'cm_separator_form':'#form1#',
02006 'cm_forms_exists':forms_exists,
02007 'cm_support_raw_edit':'no',
02008 'cm_mode':'add'}
02009 if 'cm_raw_edit' in i: ii['cm_raw_edit']=i['cm_raw_edit']
02010 if 'cm_back_json' in i: ii['cm_back_json']=i['cm_back_json']
02011 r=cm_kernel.access(ii)
02012 if r['cm_return']>0: return r
02013 cm_kernel.print_for_web(r['cm_string'])
02014
02015 cm_kernel.print_for_con('<input type="submit" class="cm-button" name="cm_submit_start" value="Start">')
02016 cm_kernel.print_for_con('<input type="submit" class="cm-button" value="Refresh">')
02017
02018 cm_kernel.print_for_con('</FORM><br>')
02019 cm_kernel.print_for_con('</td>')
02020 cm_kernel.print_for_con('<td valign="TOP">')
02021
02022
02023 ii={}
02024 ii['cm_run_module_uoa']=ini['cm_module_uoa']
02025 ii['cm_action']='run'
02026 ii['cm_detach_console']='yes'
02027
02028 if 'cm_show_json' in a1:
02029 ii['cm_show_json']=a1['cm_show_json']
02030 if 'cm_show_output_json' in a1:
02031 ii['cm_show_output_json']=a1['cm_show_output_json']
02032 if 'cm_save_output_json_to_file' in a1:
02033 ii['cm_save_output_json_to_file']=a1['cm_save_output_json_to_file']
02034
02035 if 'keep_all_files' in a1:
02036 ii['keep_all_files']=a1['keep_all_files']
02037
02038 ii['state']=a1
02039
02040 last_command=copy.deepcopy(ii)
02041
02042 if run and show_json!='yes' and save_json=='':
02043 r=cm_kernel.access_fe_through_cmd(ii)
02044 if r['cm_return']>0: return r
02045
02046
02047 if show_json=='yes':
02048 cm_kernel.print_for_web('<div class="cm-round-div" align="left">')
02049 cm_kernel.print_for_con('<i><B><small>JSON to reproduce last command</small></B></i>')
02050 cm_kernel.print_for_con('<HR class="cm-hr">')
02051
02052 if 'cm_detach_console' in last_command: del(last_command['cm_detach_console'])
02053 cm_kernel.print_for_web(json.dumps(last_command, indent=2, sort_keys=True).replace('\n','\n<BR>\n'))
02054
02055 if last_command.get('cm_run_module_uoa','')!='':
02056 cm_kernel.print_for_con('<HR class="cm-hr">')
02057 cm_kernel.print_for_con('<b>Envoke command from CMD:</b><BR>')
02058 cm_kernel.print_for_con('<i>cm '+last_command['cm_run_module_uoa']+' @input.json</i>')
02059
02060 cm_kernel.print_for_web('</div>')
02061
02062 if save_json!='':
02063 r4=cm_kernel.save_array_to_file_as_json({'cm_filename':save_json, 'cm_array':last_command})
02064 if r4['cm_return']>0: return r4
02065
02066 cm_kernel.print_for_con('</td></tr></table>')
02067
02068 return {'cm_return':0}
02069
02070
02071 def explore(i):
02072
02073 """
02074 Explore pipeline choices
02075
02076 Input: {
02077 To update ...
02078
02079 exploration_scenario - exploration scenario for this pipeline
02080 exploration_scenario_module_uoa - exploration scenario module UOA
02081 number_of_iterations - number of coarse grain iterations
02082
02083 skip_if_entry_exists - if 'yes' and coarse-grain entry exists, skip it
02084 useful for exploring only new coarse-grain points in space
02085
02086 state - init state
02087 desc - state description
02088
02089 (mobile_mode) - 'build' or 'process' - mode during crowdsourcing auto-tuning using mobiles
02090 (copy_built_executables) - if 'yes', copy all built executables to a_<UID>.out
02091 (copy_src_to_tmp) - if 'yes', copy sources to a tmp repo
02092 (useful for parallel cloud/mobile tuning)
02093
02094 (system_info) - detected system info by the mobile/cloud/grid system
02095
02096 (new_src_uoa) - new src UOA in tmp repository
02097 (measured_frequency) - measured frequency (if supported)
02098 (execution_info) - list of list of execution times per binary - 1 coarse-grain exploration step
02099 (execution_info_min) - list of minimal execution times per 1 coarse-grain exploration step
02100 (execution_info_ct_repeat_main) - CT_REPEAT_MAIN got during calibration
02101 (built_files) - list of unique built files (with UIDs)
02102 }
02103
02104 Output: {
02105 cm_return - if =0, success
02106
02107 (additional parameters from the last scenario)
02108
02109 (built_files) =[{"name","size","prepared_compiler_flags"},...]
02110 (new_src_uoa) - if src were copied, new UID in tmp repo
02111 }
02112
02113 """
02114
02115 mobile_mode=i.get('mobile_mode','')
02116 cbe=i.get('copy_built_executables','')
02117 cst=i.get('copy_src_to_tmp','')
02118 drx=''
02119
02120 es=i.get('exploration_scenario',{})
02121
02122 nis=es.get('number_of_iterations','')
02123 ni=0
02124 if nis!='': ni=int(nis)
02125
02126 nis1=es.get('number_of_iterations_1','')
02127
02128 em=es.get('exploration_scenario_module_uoa','')
02129
02130 sie=es.get('skip_if_entry_exists','')
02131
02132
02133 r=cm_kernel.get_data_description({'cm_module_uoa':ini['cm_module_uoa'],
02134 'cm_which_action':i['cm_action']})
02135 if r['cm_return']>0: return r
02136 cm_params_default1=r['cm_params_default']
02137
02138 cm_data_desc1=i['desc']
02139 cm_data_desc1o=copy.deepcopy(cm_data_desc1)
02140
02141 cm_form_explore1=i.get('cm_explore',{})
02142
02143 state=i.get('state',{})
02144 cm_explore_state=[]
02145 iter=0
02146 start_time=time.time()
02147 start_time_local=start_time
02148
02149
02150 if mobile_mode=='':
02151 cm_kernel.print_for_con('************************************************************************************')
02152 cm_kernel.print_for_con('Detecting system info ...')
02153
02154 host_os_uoa=state.get(ini['cfg']['cm_p_uids']['init'],{}).get('input',{}).get('host_os_uoa','')
02155 if host_os_uoa=='':
02156 return {'cm_return':1,'cm_error':'"host_os_uoa" is not defined'}
02157
02158 target_os_uoa=state.get(ini['cfg']['cm_p_uids']['init'],{}).get('input',{}).get('target_os_uoa','')
02159 if target_os_uoa=='':
02160 return {'cm_return':1,'cm_error':'"target_os_uoa" is not defined'}
02161
02162 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['system'],
02163 'cm_action':'detect',
02164 'host_os_uoa':host_os_uoa,
02165 'target_os_uoa':target_os_uoa,
02166 'get_uoa_from_repo':'no'}
02167 r=cm_kernel.access(ii)
02168 if r['cm_return']>0: return r
02169
02170 system_info=r
02171 else:
02172
02173 system_info=i.get('system_info',{})
02174
02175 if mobile_mode!='report':
02176 system_string=system_info.get('system_string','')
02177 processor_string=system_info.get('processor_string','')
02178 os_string=system_info.get('os_string','')
02179 mem_string=system_info.get('memory_string','')
02180 all_freqs=system_info.get('all_frequencies_string','')
02181
02182
02183 state[ini['cfg']['cm_p_uids']['detected_system_info']]['output']={}
02184
02185 if system_string!='': state[ini['cfg']['cm_p_uids']['detected_system_info']]['output']['system_string']=system_string
02186 if processor_string!='': state[ini['cfg']['cm_p_uids']['detected_system_info']]['output']['processor_string']=processor_string
02187 if os_string!='': state[ini['cfg']['cm_p_uids']['detected_system_info']]['output']['os_string']=os_string
02188 if mem_string!='': state[ini['cfg']['cm_p_uids']['detected_system_info']]['output']['memory_string']=mem_string
02189 if all_freqs!='': state[ini['cfg']['cm_p_uids']['detected_system_info']]['output']['all_frequencies_string']=all_freqs
02190
02191
02192 new_src_uoa=i.get('new_src_uoa','')
02193 new_src_path=''
02194 if mobile_mode=='report':
02195 cm_kernel.print_for_con('')
02196 cm_kernel.print_for_con('Finding path to new source code in tmp:'+new_src_uoa+' ...')
02197 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
02198 'cm_action':'load',
02199 'cm_data_uoa':new_src_uoa,
02200 'cm_repo_uoa':ini['cfg']['tmp_repo']}
02201 rx=cm_kernel.access(ii)
02202 if rx['cm_return']>0: return rx
02203 new_src_path=rx['cm_path']
02204
02205
02206 rr={'cm_return':0}
02207 while True:
02208 iter+=1
02209 if ni>0 and iter>ni: break
02210
02211
02212 cm_kernel.print_for_con('************************************************************************************')
02213
02214 space={}
02215 if mobile_mode!='report':
02216 x=''
02217 if ni>0: x=' out of '+str(ni)
02218 cm_kernel.print_for_con('Coarse-grain iteration: '+str(iter)+x)
02219
02220 t=time.time()-start_time
02221 ts="%.1f" % t
02222 cm_kernel.print_for_con('Time elapsed: '+ts+' sec.')
02223
02224 cm_kernel.print_for_con('')
02225 cm_kernel.print_for_con('Preparing choices (coarse-grain point in multi-dimensional space):')
02226
02227 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-choice'],
02228 'cm_action':'prepare_choices',
02229 'cm_array':state,
02230 'cm_data_desc':cm_data_desc1,
02231 'cm_explore':cm_form_explore1,
02232 'cm_explore_state':cm_explore_state,
02233 'cm_explore_level':''}
02234 r=cm_kernel.access(ii)
02235 if r['cm_return']>0: return r
02236
02237 if r['cm_exploration_finished']=='yes' and r['cm_last_iteration']!='yes': break
02238
02239 cm_explore_state=r['cm_explore_state']
02240
02241
02242 r=prepare_summary_point({'state':state})
02243 if r['cm_return']>0: return r
02244 sum1=r['summary_point1']
02245 sum2=r['summary_point2']
02246
02247
02248 space['scenario_module_uoa']=em
02249 space['pipeline_module_uoa']=ini['cm_module_uid']
02250 space['summary_point']=sum1
02251 space['summary_point_relaxed']=sum2
02252 space['points']=[]
02253 else:
02254
02255 r4=cm_kernel.load_json_file({'cm_filename':os.path.join(new_src_path,'ct_space.json')})
02256 if r4['cm_return']>0: return r4
02257 space=r4['cm_array']
02258 sum1=space['summary_point']
02259
02260
02261 ae=[]
02262 mae_uoa=es.get('manual_aggregated_entry_uoa','')
02263
02264 if es.get('aggregate_in_one_entry','')=='yes':
02265 if mae_uoa!='':
02266 ae.append({'cm_module_uoa':em, 'cm_data_uoa':mae_uoa})
02267 else:
02268
02269 ii={}
02270 ii['work_repo_uoa']=es.get('work_repo_uoa','')
02271 ii['record_module_uoa']=em
02272 ii['summary_point']=sum1
02273
02274 r=find_aggregated_entry(ii)
02275 if r['cm_return']>0: return r
02276
02277 ae=r['aggregated_entries']
02278
02279 if len(ae)==0:
02280 cm_kernel.print_for_con('')
02281 cm_kernel.print_for_con(' Aggregated entry not found ...')
02282
02283 if len(ae)>0:
02284 af=False
02285 for q in ae:
02286
02287 ii={}
02288 ii['cm_data_uoa']=q['cm_data_uoa']
02289 ii['cm_repo_uoa']=es.get('work_repo_uoa','')
02290 ii['cm_run_module_uoa']=em
02291 ii['cm_action']='load'
02292 r=cm_kernel.access(ii)
02293 if r['cm_return']==0:
02294 ae=[q]
02295
02296 d=r['cm_data_obj']['cfg']
02297
02298 cm_kernel.print_for_con('')
02299 cm_kernel.print_for_con(' Found aggregated entry in repository ('+ae[0]['cm_data_uoa']+') ...')
02300
02301 if sie=='yes':
02302
02303
02304 cm_kernel.print_for_con('')
02305 cm_kernel.print_for_con('Aggregated entry exists and mode to skip if exists is turned on:')
02306 cm_kernel.print_for_con(' Skipping iteration!')
02307
02308 continue
02309
02310
02311 space=d
02312
02313 cm_kernel.print_for_con('')
02314 cm_kernel.print_for_con('Aggregated point preloaded from '+ii['cm_data_uoa'])
02315
02316 break
02317
02318
02319 if cst=='yes':
02320 rx=cm_kernel.gen_uid({})
02321 if rx['cm_return']>0: return rx
02322 drx=rx['cm_uid']
02323
02324 cm_kernel.print_for_con('')
02325 cm_kernel.print_for_con('Copying source code to tmp:'+drx+' ...')
02326 ii={'cm_run_module_uoa':'core',
02327 'cm_action':'copy',
02328 'orig_module_uoa':ini['cfg']['cm_modules']['code.source'],
02329 'orig_data_uoa':state[ini['cfg']['cm_p_uids']['init']]['input']['program_uoa'],
02330 'target_repo_uoa':ini['cfg']['tmp_repo'],
02331 'target_data_uid':drx,
02332 'remove_alias':'yes'}
02333
02334 rx=cm_kernel.access(ii)
02335 if rx['cm_return']>0: return rx
02336
02337
02338 state[ini['cfg']['cm_p_uids']['init']]['input']['program_uoa']=drx
02339 state[ini['cfg']['cm_p_uids']['init']]['input']['tuning_repo_uoa']=ini['cfg']['tmp_repo']
02340
02341
02342 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
02343 'cm_action':'load',
02344 'cm_data_uoa':drx,
02345 'cm_repo_uoa':ini['cfg']['tmp_repo']}
02346 rx=cm_kernel.access(ii)
02347 if rx['cm_return']>0: return rx
02348 new_src_path=rx['cm_path']
02349
02350
02351 r4=cm_kernel.save_array_to_file_as_json({'cm_filename':os.path.join(new_src_path,'ct_space.json'), 'cm_array':space})
02352 if r4['cm_return']>0: return r4
02353
02354 cm_kernel.print_for_con('************************************************************************************')
02355 cm_kernel.print_for_con('Executing scenario ...')
02356
02357 ii={}
02358 ii['cm_run_module_uoa']=em
02359 ii['cm_action']='scenario'
02360 ii['state']=state
02361 ii['pipeline_module_desc']=ini['cfg']
02362 ii['summary_point']=sum1
02363 ii['cm_data_desc']=cm_data_desc1
02364 ii['cm_form_explore']=cm_form_explore1
02365 ii['exploration_scenario']=es
02366 if len(ae)>0: ii['aggregated_entry']=ae[0]
02367 ii['space']=space
02368 if 'original_input' in i: ii['original_input']=i['original_input']
02369 ii['caller_module_uid']=ini['cm_module_uid']
02370 if nis1!='': ii['number_of_iterations']=nis1
02371
02372 ii['mobile_mode']=mobile_mode
02373 ii['copy_built_executables']=cbe
02374 ii['measured_frequency']=i.get('measured_frequency','')
02375
02376 if mobile_mode=='report':
02377 ii['new_src_uoa']=new_src_uoa
02378 ii['new_src_path']=new_src_path
02379 ii['execution_info']=i.get('execution_info',[])
02380 ii['execution_info_min']=i.get('execution_info_min',[])
02381 ii['execution_info_ct_repeat_main']=i.get('execution_info_ct_repeat_main','')
02382 ii['built_files']=i.get('built_files',[])
02383
02384 rr=cm_kernel.access(ii)
02385 if rr['cm_return']>0: return rr
02386
02387 if cst=='yes':
02388 rr['new_src_uoa']=drx
02389
02390 if len(ae)>0: rr['aggregated_point_found']='yes'
02391
02392 if mobile_mode=='report':
02393
02394 cm_kernel.print_for_con('')
02395 cm_kernel.print_for_con('Removing (cleaning) new source code in tmp:'+new_src_uoa+' ...')
02396 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
02397 'cm_action':'delete',
02398 'cm_data_uoa':new_src_uoa,
02399 'cm_repo_uoa':ini['cfg']['tmp_repo'],
02400 'cm_force_delete':'yes',
02401 'cm_admin':'yes'}
02402 rx=cm_kernel.access(ii)
02403 if rx['cm_return']>0: return rx
02404
02405 return rr
02406
02407
02408 def web_explore(i):
02409
02410 """
02411 Explore pipeline choices through web
02412
02413 Input: TBD
02414 Output: TBD
02415
02416 """
02417
02418
02419 if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
02420 else:
02421 return {'cm_return':1, 'cm_error':'web style is not defined'}
02422
02423 cm_kernel.print_for_con('<span class="cm-title">Exploration of program/architecture choices</span><br>')
02424
02425
02426 a1={}
02427 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
02428 'cm_action':'detect_form_params',
02429 'cm_array':i,
02430 'cm_prefix':'#form1'})
02431 if r['cm_return']>0: return r
02432 cm_form_array1=r['cm_array']
02433 cm_form_commands1=r['cm_commands']
02434 cm_form_explore1=r['cm_explore']
02435
02436
02437 r=cm_kernel.get_data_description({'cm_module_uoa':ini['cm_module_uoa'],
02438 'cm_which_action':i['cm_action']})
02439 if r['cm_return']>0: return r
02440 cm_params_default1=r['cm_params_default']
02441
02442 cm_data_desc1=ini['cfg']['pipeline_desc']
02443
02444
02445 a2={}
02446 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
02447 'cm_action':'detect_form_params',
02448 'cm_array':i,
02449 'cm_prefix':'#form2'})
02450 if r['cm_return']>0: return r
02451 cm_form_array2=r['cm_array']
02452 cm_form_commands2=r['cm_commands']
02453 cm_form_explore2=r['cm_explore']
02454 cm_data_desc2=ini['cfg']['exploration_scenario_desc']
02455
02456
02457 forms_exists='yes'
02458 if len(cm_form_array1)==0 and len(cm_form_array2)==0:
02459 a1=cm_params_default1
02460 a2={}
02461 forms_exists='no'
02462 else:
02463 r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1,
02464 'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
02465 if r['cm_return']>0: return r
02466 a1=r['cm_array']
02467
02468 r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array2,
02469 'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
02470 if r['cm_return']>0: return r
02471 a2=r['cm_array']
02472
02473 show_json=a1.get('cm_show_json','')
02474 save_json=a1.get('cm_save_json_to_file','')
02475
02476
02477 last_command={}
02478
02479
02480 start=False
02481 if 'cm_submit_start' in i:
02482 start=True
02483 del(i['cm_submit_start'])
02484
02485
02486 cm_kernel.print_for_con('<br><table border="0"><tr>')
02487
02488 cm_kernel.print_for_con('<td align="center" valign="top">')
02489 cm_kernel.print_for_con('<FORM ACTION="" name="add_edit" METHOD="POST" enctype="multipart/form-data" accept-charset="utf-8">' )
02490
02491 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
02492 'cm_action':'visualize_data',
02493 'cm_array':a2,
02494 'cm_data_desc':cm_data_desc2,
02495 'cm_form_commands':cm_form_commands2,
02496 'cm_form_explore':cm_form_explore2,
02497 'cm_separator':'#',
02498 'cm_separator_form':'#form2#',
02499 'cm_forms_exists':forms_exists,
02500 'cm_support_raw_edit':'no',
02501 'cm_mode':'add',
02502 'hide_add_new_object':'yes',
02503 'cm_explore':'yes'}
02504 if 'cm_raw_edit' in i: ii['cm_raw_edit']=i['cm_raw_edit']
02505 if 'cm_back_json' in i: ii['cm_back_json']=i['cm_back_json']
02506 r=cm_kernel.access(ii)
02507 if r['cm_return']>0: return r
02508 cm_kernel.print_for_web(r['cm_string'])
02509
02510
02511 esm=a2.get('exploration_scenario_module_uoa','')
02512 if esm!='':
02513
02514 ii={'cm_run_module_uoa':esm,
02515 'cm_action':'prepare_input_params',
02516 'cm_form_array':i,
02517 'cm_array1':a1,
02518 'cm_array2':a2,
02519 'form_exists':forms_exists,
02520 'caller_cfg':ini['cfg']}
02521 r=cm_kernel.access(ii)
02522 if r['cm_return']>0: return r
02523
02524 r=show_legend({})
02525 if r['cm_return']>0: return r
02526 cm_kernel.print_for_web(r['cm_html'])
02527
02528 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
02529 'cm_action':'visualize_data',
02530 'cm_array':a1,
02531 'cm_data_desc':cm_data_desc1,
02532 'cm_form_commands':cm_form_commands1,
02533 'cm_form_explore':cm_form_explore1,
02534 'cm_separator':'#',
02535 'cm_separator_form':'#form1#',
02536 'cm_forms_exists':forms_exists,
02537 'cm_support_raw_edit':'no',
02538 'cm_mode':'add',
02539 'hide_add_new_object':'yes',
02540 'cm_explore':'yes'}
02541 if 'cm_raw_edit' in i: ii['cm_raw_edit']=i['cm_raw_edit']
02542 if 'cm_back_json' in i: ii['cm_back_json']=i['cm_back_json']
02543 r=cm_kernel.access(ii)
02544 if r['cm_return']>0: return r
02545 cm_kernel.print_for_web(r['cm_string'])
02546
02547 cm_kernel.print_for_con('<input type="submit" class="cm-button" name="cm_submit_start" value="Start">')
02548 cm_kernel.print_for_con('<input type="submit" class="cm-button" value="Refresh">')
02549
02550 cm_kernel.print_for_con('</FORM><br>')
02551 cm_kernel.print_for_con('</td>')
02552 cm_kernel.print_for_con('<td valign="TOP">')
02553
02554
02555 ii={}
02556 ii['cm_run_module_uoa']=ini['cm_module_uoa']
02557 ii['cm_action']='explore'
02558 ii['cm_detach_console']='yes'
02559
02560 if 'cm_show_json' in a1:
02561 ii['cm_show_json']=a1['cm_show_json']
02562
02563 if 'keep_all_files' in a1:
02564 ii['keep_all_files']=a1['keep_all_files']
02565
02566 ii['state']=a1
02567 ii['desc']=cm_data_desc1
02568 ii['cm_explore']=cm_form_explore1
02569
02570 ii['exploration_scenario']=a2
02571
02572 ii['original_input']=i
02573
02574 last_command=ii
02575 if start and show_json!='yes' and save_json=='':
02576 r=cm_kernel.access_fe_through_cmd(ii)
02577 if r['cm_return']>0: return r
02578
02579
02580 if show_json=='yes':
02581 cm_kernel.print_for_web('<div class="cm-round-div" align="left">')
02582 cm_kernel.print_for_con('<i><B><small>JSON to reproduce last command</small></B></i>')
02583 cm_kernel.print_for_con('<HR class="cm-hr">')
02584
02585 if 'cm_detach_console' in last_command: del(last_command['cm_detach_console'])
02586 cm_kernel.print_for_web(json.dumps(last_command, indent=2, sort_keys=True).replace('\n','\n<BR>\n'))
02587
02588 if last_command.get('cm_run_module_uoa','')!='':
02589 cm_kernel.print_for_con('<HR class="cm-hr">')
02590 cm_kernel.print_for_con('<b>Envoke command from CMD:</b><BR>')
02591 cm_kernel.print_for_con('<i>cm '+last_command['cm_run_module_uoa']+' @input.json</i>')
02592
02593 cm_kernel.print_for_web('</div>')
02594
02595 if save_json!='':
02596 try:
02597 f=open(save_json, 'w')
02598 f.write(json.dumps(last_command, indent=2, sort_keys=True))
02599 f.close()
02600 except Exception as e:
02601 cm_kernel.print_for_web('<hr>')
02602 cm_kernel.print_for_web('<B><I>Error writing json to file ('+format(e)+')</I></B>')
02603
02604 cm_kernel.print_for_con('</td></tr></table>')
02605
02606 return {'cm_return':0}
02607
02608
02609 def find_aggregated_entry(i):
02610
02611 """
02612 Find aggregated entry
02613
02614 Input: {
02615 (work_repo_uoa) - repo UOA
02616 record_module_uoa - module UOA
02617 summary_point - pipeline state to use dimensions for search
02618 }
02619
02620 Output: {
02621 cm_return - if =0, success
02622 aggregated_entries - aggregated_entries
02623 }
02624
02625 """
02626
02627 cm_kernel.print_for_con('')
02628 cm_kernel.print_for_con('Searching one entry according to tuning dimensions ...')
02629
02630 m=i.get('record_module_uoa','')
02631
02632
02633 sum1=i['summary_point']
02634
02635 ee={}
02636
02637 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'],
02638 'cm_action':'search',
02639 'use_flat_array':'yes',
02640 'module_selection':[m]
02641 }
02642 if i.get('work_repo_uoa','')!='': ii['repo_selection']=[i['work_repo_uoa']]
02643
02644 j=0
02645 for k in sum1:
02646 ii['key_'+str(j)]='##summary_point#'+k
02647 ii['value_'+str(j)]=sum1[k]
02648 j+=1
02649
02650 ii['strict_match']='yes'
02651
02652 r=cm_kernel.access(ii)
02653 if r['cm_return']>0: return r
02654
02655 return {'cm_return':0, 'aggregated_entries':r['all']}
02656
02657
02658 def prepare_summary_point(i):
02659
02660 """
02661 Prepare summary point (to save a unique experiment in a repository)
02662
02663 Input: {
02664 state - pipeline state to use dimensions for search
02665 }
02666
02667 Output: {
02668 cm_return - if =0, success
02669 summary_point1 - summary point (dimensions) - strict summary point to detect the same experiments
02670 summary_point2 - summary point (dimensions) - relaxed summary point for viewing and pruning
02671 }
02672 """
02673
02674 sum1={}
02675 sum2={}
02676
02677 s=i.get('state',{})
02678
02679 init=ini['cfg']['cm_p_uids']['init']
02680
02681 sum1['host_os_uoa']=s[init]['input'].get('host_os_uoa','')
02682 sum1['target_os_uoa']=s[init]['input'].get('target_os_uoa','')
02683 sum1['target_processor_uoa']=s[init]['input'].get('target_processor_uoa','')
02684
02685 x=s[init]['input'].get('target_accelerator_processor_uoa','')
02686 if x!='':
02687 sum1['target_accelerator_processor_uoa']=x
02688
02689
02690
02691 if s[ini['cfg']['cm_p_uids']['detected_system_info']].get('output',{}).get('processor_string','')!='':
02692 sum1['processor_string']=s[ini['cfg']['cm_p_uids']['detected_system_info']]['output']['processor_string']
02693
02694 sum1['program_uoa']=s[init]['input'].get('program_uoa','')
02695 sum1['run_cmd_key']=s[init]['input'].get('run_cmd_key','')
02696 sum1['dataset_uoa']=s[init]['input'].get('dataset_uoa','')
02697 sum1['compilation_type']=s[init]['input'].get('compilation_type','')
02698 sum1['tuning_objective']=s[init]['input'].get('tuning_objective','')
02699 sum1['notes']=s[init]['input'].get('notes','')
02700
02701
02702 y=s[init]['input'].get('compiler_code_uoa','')
02703 if y!='':
02704 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['code']}
02705 ii['cm_action']='load'
02706 ii['cm_data_uoa']=y
02707 r=cm_kernel.access(ii)
02708 if r['cm_return']>0: return r
02709
02710 d=r['cm_data_obj']['cfg']
02711 x=d.get('state_input',{}).get('package_data_uoa','')
02712
02713 sum1['compiler_package_uoa']=x
02714 sum1['ctuning_compiler_uoa']=s[init]['input'].get('ctuning_compiler_uoa','')
02715
02716
02717 y=s[init]['input'].get('accelerator_compiler_code_uoa','')
02718 if y!='':
02719 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['code']}
02720 ii['cm_action']='load'
02721 ii['cm_data_uoa']=y
02722 r=cm_kernel.access(ii)
02723 if r['cm_return']>0: return r
02724
02725 d=r['cm_data_obj']['cfg']
02726 x=d.get('state_input',{}).get('package_data_uoa','')
02727
02728 sum1['accelerator_compiler_package_uoa']=x
02729 sum1['ctuning_accelerator_compiler_uoa']=s[init]['input'].get('ctuning_accelerator_compiler_uoa','')
02730
02731 sum2=copy.deepcopy(sum1)
02732 sum2['user_friendly_system_name']=s[init]['input'].get('user_friendly_system_name','')
02733
02734
02735 x=s[ini['cfg']['cm_p_uids']['detected_system_info']].get('output',{}).get('os_string','')
02736 if x!='': sum2['os_string']=x
02737
02738 x=s[ini['cfg']['cm_p_uids']['detected_system_info']].get('output',{}).get('system_string','')
02739 if x!='': sum2['system_string']=x
02740
02741 x=s[ini['cfg']['cm_p_uids']['detected_system_info']].get('output',{}).get('memory_string','')
02742 if x!='': sum2['memory_string']=x
02743
02744 x=s[ini['cfg']['cm_p_uids']['detected_system_info']].get('output',{}).get('all_frequencies_string','')
02745 if x!='': sum2['all_frequencies_string']=x
02746
02747 return {'cm_return':0, 'summary_point1':sum1, 'summary_point2':sum2}
02748
02749
02750 def web_view(i):
02751
02752 """
02753 View aggregated experimental results
02754
02755 Input: {
02756 TBD
02757 (web_title) - web title to substitute default
02758 (calculate_program_similarities) - if 'yes', calculate distance from programs based on properties
02759 (program_features) - use it to calcluate distance
02760
02761 (pruning_similar_programs) - direct pruning parameters
02762 (number_of_returned_items) - number of returned items
02763 }
02764
02765 Output: {
02766 cm_return - if =0, success
02767 (similar_programs) - list of similar programs (if not web)
02768 (similar_programs_points) - generated point per similar program (if not web)
02769 (similar_programs_points_improvements) - generated improvements for points per similar program (if not web)
02770 }
02771
02772 """
02773
02774 rr={'cm_return':0}
02775
02776
02777 if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
02778 else:
02779 return {'cm_return':1, 'cm_error':'web style is not defined'}
02780
02781 wt=i.get('web_title','View aggregated experimental results')
02782 if i.get('cm_console')=='web':
02783 cm_kernel.print_for_con('<div align="left"><span class="cm-title">'+wt+'</span><br>')
02784
02785
02786 if len(i.get('pruning_similar_programs',{}))>0 or i.get('cm_console','')!='web':
02787 a1=i['pruning_similar_programs']
02788 cm_form_array1={}
02789 cm_form_commands1={}
02790 else:
02791 a1={}
02792 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
02793 'cm_action':'detect_form_params',
02794 'cm_array':i,
02795 'cm_prefix':'#form1'})
02796 if r['cm_return']>0: return r
02797 cm_form_array1=r['cm_array']
02798 cm_form_commands1=r['cm_commands']
02799
02800 a9={}
02801 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
02802 'cm_action':'detect_form_params',
02803 'cm_array':i,
02804 'cm_prefix':'#form9'})
02805 if r['cm_return']>0: return r
02806 cm_form_array9=r['cm_array']
02807 cm_form_commands9=r['cm_commands']
02808
02809
02810 r=cm_kernel.get_data_description({'cm_module_uoa':ini['cm_module_uoa'],
02811 'cm_which_action':'web_view'})
02812 if r['cm_return']>0: return r
02813 cm_params_default1=r['cm_params_default']
02814
02815 cm_data_pruner=ini['cfg']['pipeline_data_pruner']
02816 cm_data_viewer=ini['cfg']['pipeline_data_viewer']
02817 cm_data_desc1=ini['cfg']['pipeline_data_pruner_desc']
02818
02819
02820 if 'cm_submit_reset' in i:
02821 if 'cm_submit_prune_state' in i: del(i['cm_submit_prune_state'])
02822 cm_form_array1={}
02823
02824
02825 if i.get('cm_console')=='web':
02826 forms_exists='yes'
02827 if len(cm_form_array1)==0:
02828 a1=cm_params_default1
02829
02830 forms_exists='no'
02831 else:
02832 r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1,
02833 'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
02834 if r['cm_return']>0: return r
02835 a1=r['cm_array']
02836
02837 r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array9,
02838 'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
02839 if r['cm_return']>0: return r
02840 a9=r['cm_array']
02841
02842 cm_kernel.print_for_con('<br><table border="0"><tr>')
02843
02844 cm_kernel.print_for_con('<td align="center">')
02845 cm_kernel.print_for_con('<FORM ACTION="" name="add_edit" METHOD="POST" enctype="multipart/form-data" accept-charset="utf-8">' )
02846
02847 if len(i.get('pruning_form',{}))>0 or i.get('cm_console','')!='web':
02848 a1.update(i['pruning_form'])
02849
02850 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
02851 'cm_action':'visualize_data',
02852 'cm_array':a1,
02853 'cm_data_desc':cm_data_desc1,
02854 'cm_form_commands':cm_form_commands1,
02855 'cm_separator':'#',
02856 'cm_separator_form':'#form1#',
02857 'cm_forms_exists':forms_exists,
02858 'cm_support_raw_edit':'no',
02859 'hide_add_new_object':'yes',
02860 'cm_mode':'add'}
02861 if 'cm_raw_edit' in i: ii['cm_raw_edit']=i['cm_raw_edit']
02862 if 'cm_back_json' in i: ii['cm_back_json']=i['cm_back_json']
02863 r=cm_kernel.access(ii)
02864 if r['cm_return']>0: return r
02865 cm_kernel.print_for_web(r['cm_string'])
02866
02867 cm_kernel.print_for_con('<input type="submit" class="cm-button" name="cm_submit_prune" value="Prune">')
02868 cm_kernel.print_for_con('<input type="submit" class="cm-button" name="cm_submit_reset" value="Reset">')
02869
02870 if 'cm_submit_prune' in i or i.get('cm_submit_prune_state','')=='yes':
02871 if i.get('cm_console')=='web':
02872 cm_kernel.print_for_con(' <input type="hidden" name="cm_submit_prune_state" value="yes">')
02873
02874 if i.get('cm_console')=='web':
02875 cm_kernel.print_for_con('</td>')
02876 cm_kernel.print_for_con('</tr></table>')
02877
02878
02879 if 'cm_submit_prune' in i or i.get('cm_submit_prune_state','')=='yes':
02880
02881 if i.get('cm_console')=='web':
02882 cm_kernel.print_for_con('<HR class="cm-hr">')
02883 cm_kernel.print_for_con('<span class="cm-title">List of experiments</span><br>')
02884
02885 esmu=a1.get('exploration_scenario_module_uoa','')
02886
02887 start_time=time.time()
02888 module_list=[]
02889 if esmu!='':
02890 module_list.append(esmu)
02891 else:
02892
02893 module_list=ini['cfg']['scenario_module_uoa_list']
02894
02895 results=[]
02896 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'],
02897 'cm_action':'search',
02898 'use_flat_array':'yes',
02899 'module_selection':module_list,
02900 'timeout':120
02901 }
02902 if a1.get('tuning_repo_uoa','')!='': ii['repo_selection']=[a1['tuning_repo_uoa']]
02903
02904
02905 pruned={}
02906 j=0
02907 for k in cm_data_pruner:
02908 rg=cm_kernel.get_value_by_flattened_key({'cm_array':a1, 'cm_key':k})
02909 if rg['cm_return']>0: return rg
02910 v=rg['cm_value']
02911
02912 if v!=None and v!='':
02913 ii['key_'+str(j)]='##summary_point_relaxed#'+k[2:]
02914
02915 ii['value_'+str(j)]=v
02916 pruned[k]=True
02917 j+=1
02918 if esmu!='':
02919 pruned['##exploration_scenario_module_uoa']=esmu
02920 if i.get('cm_console')=='web':
02921 cm_kernel.print_for_con(' <input type="hidden" name="ct_module_uoa" value="'+esmu+'">')
02922 cm_kernel.print_for_con(' <input type="hidden" name="use_multiple_graphs" value="on">')
02923
02924 r=cm_kernel.access(ii)
02925 if r['cm_return']>0: return r
02926
02927 all=r['all']
02928
02929 ts="%.1f" % (time.time()-start_time); start_time=time.time()
02930 if i.get('cm_console')=='web':
02931 cm_kernel.print_for_con('<small><i><b>Debug: </b>Pruning time: '+ts+' sec. (number of elements: '+str(len(all))+'); ')
02932
02933
02934 for q in all:
02935 jj={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'],
02936 'cm_action':'load'}
02937 jj.update(q)
02938 r=cm_kernel.access(jj)
02939 if r['cm_return']==0:
02940 results.append(r)
02941
02942 ts="%.1f" % (time.time()-start_time); start_time=time.time()
02943 if i.get('cm_console')=='web':
02944 cm_kernel.print_for_con('Reading all elements time: '+ts+' sec.; ')
02945
02946
02947 lp=len(results)
02948
02949 lts=10
02950 alts=a9.get('page_size','')
02951 if i.get('number_of_returned_items','')!='':
02952 alts=i['number_of_returned_items']
02953 if alts!='':
02954 if alts=='All': lts=lp
02955 else: lts=int(alts)
02956 if lts==0: lts=10
02957
02958 lt1=0
02959 alt=a9.get('page','')
02960 if alt!='':
02961 lt1=(int(alt))*lts
02962 if lp<lt1: lt1=0
02963
02964 lt2=lt1+lts
02965 if lp<lt2: lt2=lp
02966
02967
02968 results5=[]
02969 milepost_distance_min=-1
02970 milepost_distance_max=-1
02971 for qi in range(0, len(results)):
02972 q=results[qi]
02973 quid=q['cm_uid']
02974
02975 rrx={}
02976 dcfg=q['cm_data_obj']['cfg']
02977
02978
02979 sum=dcfg.get('summary_point',{})
02980
02981 sum.update(dcfg.get('summary_point_relaxed',{}))
02982
02983 rrx['milepost_distance']=99999
02984
02985 if len(sum)>0 and sum.get('program_uoa','')!='':
02986
02987 cprog={}
02988 hot_function1=''
02989 ll={'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
02990 'cm_action':'load',
02991 'cm_data_uoa':sum['program_uoa']}
02992 rl=cm_kernel.access(ll)
02993 if rl['cm_return']==0:
02994 cprog=rl['cm_data_obj']['cfg']
02995
02996 pn=rl['cm_display_as_alias']
02997 if pn!='': rrx['program_name']=pn
02998
02999 hf=cprog.get('run_cmds',{}).get(sum['run_cmd_key'],{}).get('hot_functions',[])
03000 if len(hf)>0:
03001 hot_function1=hf[0].get('name','')
03002
03003
03004 lmilepost=''
03005 ll={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.scenario.program.features.milepost'],
03006 'cm_action':'load',
03007 'cm_data_uoa':sum['program_uoa']}
03008 rl=cm_kernel.access(ll)
03009 if rl['cm_return']==0:
03010 rd2={}
03011
03012
03013
03014
03015 lv=web['http_prefix']+'view_cid='+\
03016 ini['cfg']['cm_modules']['ctuning.scenario.program.features.milepost']+\
03017 ':'+sum['program_uoa']
03018
03019
03020
03021 rd=rl['cm_data_obj']['cfg'].get('points',[])
03022 if hot_function1!='' and len(rd)>0:
03023 rd1=rd[0].get('pipeline_output',{}).get('state',{}).get(ini['cfg']['cm_p_uids']['finish_ctuning_milepost'],{})
03024 rd2=rd1.get('output',{}).get('program_structure',{}).get(hot_function1,{}).get('program_static_properties',{})
03025
03026 lv+='###points@0#pipeline_output#state#'+ini['cfg']['cm_p_uids']['finish_ctuning_milepost']+\
03027 '#output#program_structure#susan_corners#program_static_properties#'
03028
03029 lmilepost='<a href="'+lv+'" target="_blank">MILEPOST</a>'
03030
03031 rrx['milepost_link']=lmilepost
03032 rrx['cm_uid']=quid
03033
03034 if i.get('calculate_program_similarities','')=='yes':
03035 pf=i.get('program_features',{})
03036 pfe=rd2.get(ini['cfg']['default_pass_to_extract'],{})
03037
03038 if len(pf)>0 and len(pfe)>0:
03039
03040 rx=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.scenario.program.features.milepost'],
03041 'cm_action':'calculate_similarity',
03042 'program1_milepost_features':pf,
03043 'program2_milepost_features':pfe})
03044 if rx['cm_return']==0:
03045 distance=rx['distance']
03046 fdistance=float(rx['distance'])
03047 xdistance="%.3f" % fdistance
03048
03049 if milepost_distance_min==-1: milepost_distance_min=fdistance
03050 elif fdistance<milepost_distance_min: milepost_distance_min=fdistance
03051
03052 if milepost_distance_max==-1: milepost_distance_max=fdistance
03053 elif fdistance>milepost_distance_max: milepost_distance_max=fdistance
03054
03055
03056 rx=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.scenario.program.features.milepost'],
03057 'cm_action':'calculate_similarity',
03058 'program1_milepost_features':pf,
03059 'program2_milepost_features':pfe})
03060 if rx['cm_return']==0:
03061 distance=rx['distance']
03062 fdistance=float(rx['distance'])
03063 xdistance="%.3f" % fdistance
03064
03065 rrx['milepost_distance']=distance
03066
03067
03068 for kk in range(0,3):
03069 k=cm_data_viewer[kk]
03070
03071 ca=sum
03072 ka=k
03073 if k.count('#')>2:
03074 ca=dcfg
03075 ix=k.rfind('#')
03076 ka='#'+k[ix:]
03077
03078 rg=cm_kernel.get_value_by_flattened_key({'cm_array':ca, 'cm_key':k})
03079 if rg['cm_return']>0: return rg
03080 v=rg['cm_value']
03081 vh=''
03082 vt=''
03083
03084
03085 desc=cm_data_desc1.get(ka,{})
03086 desc['cm_view_as_text']='yes'
03087 desc['cm_disable_coloring']='yes'
03088
03089 vv=''
03090
03091 if v!=None:
03092
03093 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
03094 'cm_action':'convert_field_to_html',
03095 'cm_value':v,
03096 'cm_key':'',
03097 'cm_mode':'view',
03098 'cm_data_desc':desc,
03099 'cm_disable_coloring':'yes'}
03100 r1=cm_kernel.access(ii)
03101 if r1['cm_return']>0: return r1
03102 vh=r1['cm_value_html']
03103 vt=r1['cm_value_text']
03104
03105 rrx[k]=vt.lower()
03106
03107 rrx['index']=qi
03108
03109 if len(rrx)>0: results5.append(rrx)
03110
03111 ts="%.1f" % (time.time()-start_time); start_time=time.time()
03112 if i.get('cm_console')=='web':
03113 cm_kernel.print_for_con('Preparing table for sorting: '+ts+' sec.; ')
03114
03115
03116 if i.get('calculate_program_similarities','')=='yes':
03117 results6 = sorted(results5, key=lambda o: (float(o.get('milepost_distance',0)), \
03118 o.get(cm_data_viewer[0],''), \
03119 o.get(cm_data_viewer[1],''), \
03120 o.get(cm_data_viewer[2],'')))
03121 else:
03122 results6 = sorted(results5, key=lambda o: (o.get(cm_data_viewer[0],''), \
03123 o.get(cm_data_viewer[1],''), \
03124 o.get(cm_data_viewer[2],'')))
03125
03126 ts="%.1f" % (time.time()-start_time); start_time=time.time()
03127 if i.get('cm_console')=='web':
03128 cm_kernel.print_for_con('Sorting all table time: '+ts+' sec.; ')
03129
03130
03131 results2=[]
03132 for q in results6:
03133 results2.append(results[q['index']])
03134
03135
03136 results1=[]
03137
03138 for qp in range(lt1, lt2):
03139 q=results2[qp]
03140 dcfg=q['cm_data_obj']['cfg']
03141
03142 sum=dcfg.get('summary_point',{})
03143
03144 sum.update(dcfg.get('summary_point_relaxed',{}))
03145
03146 sum['exploration_scenario_module_uoa']=q['cm_module_uoa']
03147
03148 if len(sum)>0:
03149 q['sum']=copy.deepcopy(sum)
03150 q['sum_text']={}
03151 q['sum_html']={}
03152 for k in cm_data_viewer:
03153
03154 ca=sum
03155 ka=k
03156 if k.count('#')>2:
03157 ca=dcfg
03158 ix=k.rfind('#')
03159 ka='#'+k[ix:]
03160
03161 rg=cm_kernel.get_value_by_flattened_key({'cm_array':ca, 'cm_key':k})
03162 if rg['cm_return']>0: return rg
03163 v=rg['cm_value']
03164 vh=''
03165 vt=''
03166
03167
03168 desc=cm_data_desc1.get(ka,{})
03169 desc['cm_view_as_text']='yes'
03170 desc['cm_disable_coloring']='yes'
03171
03172 vv=''
03173
03174 if v!=None:
03175
03176 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
03177 'cm_action':'convert_field_to_html',
03178 'cm_value':v,
03179 'cm_key':'',
03180 'cm_mode':'view',
03181 'cm_data_desc':desc,
03182 'cm_disable_coloring':'yes'}
03183 r1=cm_kernel.access(ii)
03184 if r1['cm_return']>0: return r1
03185 vh=r1['cm_value_html']
03186 vt=r1['cm_value_text']
03187
03188 q['sum_text'][k]=vt
03189 q['sum_html'][k]='<small><small>'+vh+'</small></small>'
03190
03191 results1.append(q)
03192
03193 ts="%.1f" % (time.time()-start_time); start_time=time.time()
03194 if i.get('cm_console')=='web':
03195 cm_kernel.print_for_con('Preparing table time: '+ts+' sec.; ')
03196 else:
03197 rr['similar_programs']=[]
03198 rr['similar_programs_points']=[]
03199 rr['similar_programs_points_improvements']=[]
03200 mlts=lts
03201 if len(results6)<mlts: mlts=len(results6)
03202 for q in range(0, mlts):
03203 rr['similar_programs'].append(results6[q])
03204
03205
03206
03207
03208
03209
03210
03211
03212
03213 if i.get('cm_console')=='web':
03214 cm_kernel.print_for_con('</i></small><br>')
03215
03216 if i.get('cm_console')=='web':
03217
03218 v1='Page number: <select class="cm-select" name="#form9##page" onchange="document.add_edit.submit();" style="width:60px;">\n'
03219 for x in range(0, int(math.ceil(float(lp)/float(lts)))):
03220 z=''
03221 if str(x)==alt: z=' SELECTED '
03222 v1+='<option value="'+str(x)+'"'+z+'>'+str(x+1)+'</option>\n'
03223 v1+='</select>\n'
03224
03225 ps=ini['cfg']['page_sizes']
03226 v1+='Page size: <select class="cm-select" name="#form9##page_size" onchange="document.add_edit.submit();" style="width:60px;">\n'
03227 for x in ps:
03228 z=''
03229 if str(x)==alts: z=' SELECTED '
03230 v1+='<option value="'+x+'"'+z+'>'+x+'</option>\n'
03231 v1+='</select>\n'
03232 cm_kernel.print_for_web(v1)
03233
03234
03235 cm_kernel.print_for_web(web['table_init'])
03236
03237 sx='rowspan="2" valign="bottom"'
03238 cm_kernel.print_for_web('<tr>')
03239
03240 cm_kernel.print_for_web('<td '+sx+'><small><b>#:</b></small></td>')
03241 cm_kernel.print_for_web('<td '+sx+'><small><b>Merged Graphs:</b></small></td>')
03242 cm_kernel.print_for_web('<td '+sx+'><small><b>Aggregated entry UID:</b></small></td>')
03243
03244 if i.get('calculate_program_similarities','')=='yes':
03245 cm_kernel.print_for_web('<td '+sx+'><small><b>Program similarity:</b></small></td>')
03246
03247 for k in cm_data_viewer:
03248 ka=k
03249 if k.count('#')>2:
03250 ix=k.rfind('#')
03251 ka='#'+k[ix:]
03252
03253 desc=cm_data_desc1.get(ka,{})
03254 dt=desc.get('desc_text','')
03255 if len(dt)>0: dt=dt[0].upper()+dt[1:]
03256 if k not in pruned or desc.get('show_even_if_pruned','')=='yes':
03257 cm_kernel.print_for_web('<td '+sx+'><small><b>'+dt+':</b></small></td>')
03258
03259
03260 cm_kernel.print_for_web('<td '+sx+'><small><b>Has collected properties:</b></small></td>')
03261
03262 cm_kernel.print_for_web('<td '+sx+'><small><b>Total number of explored points:</b></small></td>')
03263 cm_kernel.print_for_web('<td '+sx+'><small><b>Points:</b></small></td>')
03264
03265
03266 sh=''
03267 if esmu!='':
03268 r=cm_kernel.access({'cm_run_module_uoa':esmu,
03269 'header_desc':ini['cfg']['pipeline_desc'],
03270 'cm_action':'web_view_header'})
03271 if r['cm_return']==0:
03272 sh=r['cm_html']
03273 v=int(r['cm_length'])
03274 vx=''
03275 if v>0: vx=' colspan="'+str(v)+'"'
03276 cm_kernel.print_for_web('<td'+vx+' align="center"><small><b>Scenario results:</b></small></td>')
03277
03278 cm_kernel.print_for_web('<td '+sx+'><small><b>Reproduce:</b></small></td>')
03279
03280 cm_kernel.print_for_web('<td '+sx+'><small><b>View:</b></small></td>')
03281 cm_kernel.print_for_web('<td '+sx+'><small><b>Graph:</b></small></td>')
03282 cm_kernel.print_for_web('</tr>')
03283
03284
03285 cm_kernel.print_for_web('<tr>'+sh+'</tr>')
03286
03287
03288 line=True
03289 line1=True
03290 x1=''
03291 if 'table_bgcolor_line1' in web: x1=' bgcolor="'+web['table_bgcolor_line1']+'" '
03292 x2=''
03293 if 'table_bgcolor_line2' in web: x2=' bgcolor="'+web['table_bgcolor_line2']+'" '
03294
03295
03296 nn=lt1
03297 for qi in range(0,len(results1)):
03298 q=results1[qi]
03299 dcfg=q['cm_data_obj']['cfg']
03300 sum=q['sum']
03301 sum_html=q['sum_html']
03302 points=dcfg.get('points',[])
03303 lp=len(points)
03304 lp_all=lp
03305 if esmu=='': lp=1
03306
03307 if len(sum)>0:
03308
03309
03310 if line1: line1=False; xx=x2
03311 else: line1=True; xx=x1
03312
03313 for l in range(0, lp):
03314
03315 if line: line=False; x=x1
03316 else: line=True; x=x2
03317
03318 if i.get('cm_console')=='web':
03319 xlp=' rowspan="'+str(lp)+'" '
03320
03321 if l==0:
03322 cm_kernel.print_for_web('<TR'+xx+'>')
03323
03324 cm_kernel.print_for_web('<td valign="top" '+xlp+'><small>'+str(nn+1)+'</small></td>')
03325 yy='use_multiple_graphs_'+str(nn+1)
03326 yy1='use_multiple_graphs_uid_'+str(nn+1)
03327 xx=''
03328 if i.get(yy, '')=='on': xx='checked'
03329 cm_kernel.print_for_web('<td valign="top" '+xlp+'><small><input type="checkbox" name="'+yy+'" '+xx+'></small>')
03330 cm_kernel.print_for_con(' <input type="hidden" name="'+yy1+'" value="'+q['cm_uid']+'">')
03331 cm_kernel.print_for_con('</td>')
03332
03333 cm_kernel.print_for_web('<td valign="top" align="center" '+xlp+'><small>'+q['cm_uid'])
03334 cm_kernel.print_for_con('</small></td>')
03335
03336
03337 lmilepost=''
03338 if i.get('calculate_program_similarities','')=='yes':
03339 lmilepost=results6[qi].get('milepost_link','')
03340 distance=str(results6[qi].get('milepost_distance',''))
03341
03342 wtd='<td valign="top" align="right" '+xlp+'><small>'
03343 if distance!='' and distance!='99999':
03344
03345 fdistance=float(distance)
03346 xdistance="%.3f" % fdistance
03347
03348 col='FFFFFF'
03349 if fdistance<=1:
03350 col1=int(55+((fdistance-milepost_distance_min)/(1-milepost_distance_min))*200)
03351 col2=hex(col1)[2:]
03352 if (col1<16): col2='0'+col2
03353 col='FF'+col2+col2
03354 else:
03355 col1=int(55+(1-((fdistance-1)/(milepost_distance_max-1)))*200)
03356 col2=hex(col1)[2:]
03357 if (col1<16): col2='0'+col2
03358 col=col2+col2+'FF'
03359
03360 wtd='<td valign="top" align="right" '+xlp+' style="background-color: #'+col+'; border-width: 1px; width=80px"><small>'
03361
03362 cm_kernel.print_for_web(wtd)
03363
03364 if distance!='':
03365 if distance=='99999':
03366 cm_kernel.print_for_con('<i>Problem</i>')
03367 else:
03368 cm_kernel.print_for_con(xdistance)
03369
03370 cm_kernel.print_for_con('</small></td>')
03371
03372 for k in cm_data_viewer:
03373 ka=k
03374 if k.count('#')>2:
03375 ix=k.rfind('#')
03376 ka='#'+k[ix:]
03377
03378 desc=cm_data_desc1.get(ka,{})
03379 if k not in pruned or desc.get('show_even_if_pruned','')=='yes':
03380 vh=sum_html.get(k,'')
03381 cm_kernel.print_for_web('<td valign="top" '+xlp+'>'+str(vh)+'</td>')
03382
03383
03384 cm_kernel.print_for_web('<td valign="top" align="center" '+xlp+'><small>')
03385 if lmilepost!='':
03386 cm_kernel.print_for_con(lmilepost+'<BR>')
03387
03388 cm_kernel.print_for_con('</small></td>')
03389
03390
03391 cm_kernel.print_for_web('<td valign="top" align="center" '+xlp+'><small>'+dcfg.get('number_of_explored_points',''))
03392 cm_kernel.print_for_con('</small></td>')
03393
03394 m=q['cm_module_uid']
03395
03396 else:
03397 cm_kernel.print_for_web('<TR>')
03398
03399
03400 if esmu!='':
03401 if i.get('cm_console')=='web':
03402 cm_kernel.print_for_web('<td valign="top"'+x+'><small>'+str(l)+'</small></td>')
03403 r=cm_kernel.access({'cm_run_module_uoa':esmu,
03404 'cm_action':'web_view_data',
03405 'header_desc':ini['cfg']['pipeline_desc'],
03406 'cur_line_style':{'cur':x, 'style1':x1, 'style2':x2},
03407 'cm_data':q,
03408 'point':str(l)})
03409 if r['cm_return']==0:
03410 if i.get('cm_console')=='web':
03411 cm_kernel.print_for_web(r['cm_html'])
03412 else:
03413 rr['similar_programs_points'].append(r['point'])
03414 rr['similar_programs_points_improvements'].append(r['point_improvements'])
03415
03416 else:
03417 if i.get('cm_console')=='web':
03418 cm_kernel.print_for_web('<td valign="top"'+x+'><small>'+str(lp_all)+'</small></td>')
03419
03420
03421 if i.get('cm_console')=='web':
03422 lv=web['http_prefix']+'cm_menu=browse&cm_subaction_view&browse_module_uoa='+m+'&ignore_desc_expand=no&browse_data_uoa='+q['cm_uid']+'###points@'+str(l)+'#'
03423 lv1=web['http_prefix']+'cm_menu=browse&cm_subaction_view&browse_module_uoa='+m+'&ignore_desc_expand=yes&browse_data_uoa='+q['cm_uid']+'###points@'+str(l)+'#'
03424 lr=web['http_prefix']+'cm_detach_console=yes&cm_web_module_uoa='+ini['cm_module_uid']+'&cm_web_action=reproduce'+ \
03425 '&reproduce_module_uoa='+m+'&reproduce_data_uoa='+q['cm_uid']+'&point='+str(l)
03426 lg=''
03427
03428 if l==0:
03429 ix={'ct_data_selection':[q['cm_uid']]}
03430 ii={}
03431 cm_kernel.copy_vars({'cm_input_array':ix, 'cm_output_array':ii,
03432 'cm_vars':[]})
03433 r=cm_kernel.convert_cm_array_to_uri({'cm_array':ii})
03434 if r['cm_return']==0:
03435 lg=web['http_prefix']+'cm_menu=scenarios&cm_submenu=ctuning_space_visualize&ct_module_uoa='+m+'&cm_json='+r['cm_string']
03436
03437 if esmu=='':
03438 if l==0: cm_kernel.print_for_web('<td valign="top"'+xlp+'><small><a href="'+lv+'" target="_blank">View</a></small></td>')
03439 else:
03440
03441 if len(points[l].get('pipeline_input',{}))==0:
03442 cm_kernel.print_for_web('<td valign="top"'+x+'><small><strike>Reproduce</strike></small></td>')
03443 else:
03444 cm_kernel.print_for_web('<td valign="top"'+x+'><small><a href="'+lr+'" target="_blank">Reproduce</a></small></td>')
03445
03446 cm_kernel.print_for_web('<td valign="top"'+x+'><small><a href="'+lv+'" target="_blank">View</a> (<a href="'+lv1+'" target="_blank">Faster</a>)</small></td>')
03447
03448 if l==0:
03449 cm_kernel.print_for_web('<td valign="top"'+xlp+'><small><a href="'+lg+'" target="_blank">Graph</a></small></td>')
03450
03451 cm_kernel.print_for_con('</tr>')
03452
03453 nn+=1
03454
03455 if i.get('cm_console')=='web':
03456 cm_kernel.print_for_con('</table></div>')
03457
03458 url=web['http_prefix']+'cm_menu=scenarios&cm_submenu=ctuning_space_visualize'
03459 cm_kernel.print_for_con('<br><div align="left"><input type="submit" class="cm-button" name="show_merged_graph" onclick="document.add_edit.action = \''+url+'\';" value="Show merged graph"></div>')
03460
03461 cm_kernel.print_for_con('</FORM><br>')
03462
03463 return rr
03464
03465
03466 def reproduce(i):
03467
03468 """
03469 Reproduce collective tuning pipeline state
03470
03471 Input: {
03472 reproduce_module_uoa - pipeline state module UOA
03473 reproduce_data_uoa - pipeline state data UOA
03474 (reproduce_repo_uoa) - pipeline state repo UOA
03475 (point) - point in state (by default 0)
03476
03477
03478
03479 data_uoa - use this data UOA to load recorded state
03480 (repo_uoa) - use this repo UOA to load recorded state
03481 (pipeline_uoa) - entry with pipeline
03482 or
03483 (pipeline) - pipeline array [{cm_access input}]
03484 (record_in_space) - if 'yes', record in cTuning space
03485 (record_repo_uoa) - if record, which repo to use
03486 (record_data_uoa) - if record, set data_uoa (if not set, generate randomly)
03487 }
03488
03489 Output: {
03490 cm_return - if =0, success
03491 pipeline_input - input pipeline (original)
03492 pipeline_output - output pipeline (new)
03493 }
03494 """
03495
03496
03497 pu=ini['cfg']['cm_p_uids']
03498
03499 rmu=i.get('reproduce_module_uoa','')
03500 rdu=i.get('reproduce_data_uoa','')
03501 rru=i.get('reproduce_repo_uoa','')
03502 p=int(i.get('point','0'))
03503 if p<0: p=0
03504
03505 if rmu=='': return {'cm_return':1, 'cm_error':'"reproduce_module_uoa" is not defined in "ctuning.pipeline reproduce"'}
03506 if rdu=='': return {'cm_return':1, 'cm_error':'"reproduce_data_uoa" is not defined in "ctuning.pipeline reproduce"'}
03507
03508
03509 cm_kernel.print_for_con('')
03510 cm_kernel.print_for_con('Loading collective tuning pipeline state ...')
03511
03512 ii={'cm_run_module_uoa':rmu,
03513 'cm_action':'load',
03514 'cm_data_uoa':rdu}
03515 if rru!='': ii['cm_repo_uoa']=rru
03516 r=cm_kernel.access(ii)
03517 if r['cm_return']>0: return r
03518 data=r['cm_data_obj']['cfg']
03519
03520 cm_kernel.print_for_con('')
03521 cm_kernel.print_for_con('Preparing pipeline state ...')
03522
03523
03524 points=data.get('points',[])
03525 if len(points)<(p+1):
03526 return {'cm_return':1, 'cm_error':'can\'t find given point in entry'}
03527
03528 point=data['points'][p]
03529
03530 pipeline_input=point.get('pipeline_input',{})
03531 if len(pipeline_input)==0:
03532 return {'cm_return':1, 'cm_error':'pipeline input not found'}
03533 pipeline_input_copy=copy.deepcopy(pipeline_input)
03534
03535 pipeline_output=point.get('pipeline_output',{})
03536 if len(pipeline_output)==0:
03537 return {'cm_return':1, 'cm_error':'pipeline output not found'}
03538
03539
03540 sp=pipeline_output.get('state',{})
03541
03542
03543 repeat=sp.get(pu['run_program'], {}).get('input',{}).get('run_vars',{}).get('CT_REPEAT_MAIN','')
03544 if repeat!='':
03545 pipeline_input['state'][pu['run_program']]['input']['calibrate']='no'
03546 if 'run_vars' not in pipeline_input['state'][pu['run_program']]['input']:
03547 pipeline_input['state'][pu['run_program']]['input']['run_vars']={}
03548 pipeline_input['state'][pu['run_program']]['input']['run_vars']['CT_REPEAT_MAIN']=repeat
03549
03550
03551 cm_kernel.print_for_con('')
03552 cm_kernel.print_for_con('Executing pipeline ...')
03553
03554 r=cm_kernel.access(pipeline_input)
03555 if r['cm_return']>0: return r
03556
03557
03558 cm_kernel.print_for_con('')
03559 cm_kernel.print_for_con('Comparing saved and new states:')
03560
03561
03562 r1=cm_kernel.flatten_array({'cm_array':pipeline_output['state']})
03563 if r1['cm_return']!=0: return r
03564 ar1=r1['cm_array']
03565
03566
03567 r2=cm_kernel.flatten_array({'cm_array':r['state']})
03568 if r2['cm_return']!=0: return r
03569 ar2=r2['cm_array']
03570
03571 for q in ar1:
03572 if q not in ar2:
03573 cm_kernel.print_for_con('')
03574 cm_kernel.print_for_con('Key: '+str(q))
03575 cm_kernel.print_for_con('Original value: '+str(ar1[q]))
03576 cm_kernel.print_for_con('New value: not available')
03577 elif ar1[q]!=ar2[q]:
03578 cm_kernel.print_for_con('')
03579 cm_kernel.print_for_con('Key: '+str(q))
03580 cm_kernel.print_for_con('Original value: '+str(ar1[q]))
03581 cm_kernel.print_for_con('New value: '+str(ar2[q]))
03582
03583 return {'cm_return':0, 'pipeline_input':pipeline_input_copy, 'pipeline_output_orig':pipeline_output['state'], 'pipeline_output_new':r}
03584
03585
03586 def show_legend(i):
03587
03588 """
03589 Show legend for pipeline
03590
03591 Input: {
03592 }
03593
03594 Output: {
03595 cm_return - if =0, success
03596 }
03597 """
03598
03599
03600 if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
03601 else:
03602 return {'cm_return':1, 'cm_error':'web style is not defined'}
03603
03604 html='<table border="0" colspan="5" rowspan="5"><tr>'
03605
03606 html+='<td>Pipeline color legend:</td>'
03607 html+='<td><table class="cm-round-table-characteristic"><TR><TD>Characteristics</TD></TR></table></td>'
03608 html+='<td><table class="cm-round-table-property"><TR><TD>Properties</TD></TR></table></td>'
03609 html+='<td><table class="cm-round-table-state"><TR><TD>State</TD></TR></table></td>'
03610 html+='<td><table class="cm-round-table-cost"><TR><TD>Cost</TD></TR></table></td>'
03611 html+='<td><table class="cm-round-table-explore"><TR><TD>Coarse-grain choices</TD></TR></table></td>'
03612 html+='<td><table class="cm-round-table-explore-1"><TR><TD>Fine-grain choices</TD></TR></table></td>'
03613
03614 html+='</tr></table>'
03615
03616 return {'cm_return':0, 'cm_html':html}
03617
03618
03619 def mobile_node_prepare(i):
03620 """
03621 Prepare execution for mobile system (phone, tablet, cloud)
03622
03623 Input: {
03624
03625 }
03626
03627 Output: {
03628 cm_return - if =0, success
03629 }
03630 """
03631
03632
03633
03634
03635 ii={'cm_action':'web_explore'}
03636 r=web_explore(ii)
03637
03638 return r
03639
03640
03641 def predict_opts(i):
03642 """
03643 Predict optimizations
03644
03645 Input: {
03646 In non-web mode:
03647 (program_milepost_properties) - if not empty, return most close programs
03648
03649 (cm_submit_prune) - if 'yes', prune results to find most close program
03650 (opt_predictor) - 'nearest_neighbour' for nearest neighbour classifier
03651 (cm_repo_uoa) - repo to prune results
03652
03653 (pruning_similar_programs) - direct pruning parameters
03654 (number_of_returned_items) - number of returned items
03655 }
03656
03657 Output: {
03658 cm_return - if =0, success
03659 }
03660 """
03661
03662 rr={'cm_return':0}
03663
03664 if i.get('cm_console')=='web':
03665
03666 reset=False
03667 if i.get('cm_reset','')!='':
03668 reset=True
03669
03670
03671 esp=False
03672 if i.get('cm_submit_extract_st_properties','')!='':
03673 esp=True
03674
03675
03676 if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
03677 else:
03678 return {'cm_return':1, 'cm_error':'web style is not defined'}
03679
03680 cm_kernel.print_for_con('<div align="left">')
03681 cm_kernel.print_for_con('<span class="cm-title">Predict optimizations using semantic program\'s properties (concept demo)</span><br>')
03682
03683 cm_kernel.print_for_con('<small><i><br><div align="right">')
03684
03685 lnk1=web['http_prefix']+'browse_module_uoa=ctuning.setup.local&cm_menu=browse'
03686 lnk2=web['http_prefix']+'browse_module_uoa=ctuning.setup.local&cm_menu=browse&cm_subaction_add=yes&browse_repo_uoa=ctuning-setup'
03687
03688 cm_kernel.print_for_con('cTuning local setup: <a href="'+lnk1+'" target="_blank">View/update</a>; <a href="'+lnk2+'" target="_blank">Add</a>')
03689 cm_kernel.print_for_con('</div><i></small>')
03690
03691
03692 a1={}
03693 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
03694 'cm_action':'detect_form_params',
03695 'cm_array':i,
03696 'cm_prefix':'#form11'})
03697 if r['cm_return']>0: return r
03698 cm_form_array1=r['cm_array']
03699 cm_form_commands1=r['cm_commands']
03700
03701 a2={}
03702 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
03703 'cm_action':'detect_form_params',
03704 'cm_array':i,
03705 'cm_prefix':'#form12'})
03706 if r['cm_return']>0: return r
03707 cm_form_array2=r['cm_array']
03708 cm_form_commands2=r['cm_commands']
03709
03710
03711 cm_params_default1=ini['cfg']['predict_opt']['cm_param_default1']
03712 cm_data_desc1=ini['cfg']['predict_opt']['cm_data_desc1']
03713
03714 cm_params_default2=ini['cfg']['predict_opt']['cm_param_default2']
03715 cm_data_desc2=ini['cfg']['predict_opt']['cm_data_desc2']
03716
03717
03718 forms_exists='yes'
03719 if reset or len(cm_form_array1)==0:
03720 a1=cm_params_default1
03721 a2=cm_params_default2
03722
03723 forms_exists='no'
03724 else:
03725 r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1,
03726 'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
03727 if r['cm_return']>0: return r
03728 a1=r['cm_array']
03729
03730 r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array2,
03731 'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
03732 if r['cm_return']>0: return r
03733 a2=r['cm_array']
03734
03735 cm_kernel.print_for_con('<td align="center" valign="top">')
03736 cm_kernel.print_for_con('<FORM ACTION="" name="add_edit" METHOD="POST" enctype="multipart/form-data" accept-charset="utf-8">' )
03737
03738 cm_kernel.print_for_con('<table border="0"><tr><td valign="top"')
03739
03740 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
03741 'cm_action':'visualize_data',
03742 'cm_array':a1,
03743 'cm_data_desc':cm_data_desc1,
03744 'cm_form_commands':cm_form_commands1,
03745 'cm_separator':'#',
03746 'cm_separator_form':'#form11#',
03747 'cm_forms_exists':forms_exists,
03748 'cm_support_raw_edit':'no',
03749 'cm_mode':'add'}
03750 if 'cm_raw_edit' in i: ii['cm_raw_edit']=i['cm_raw_edit']
03751 if 'cm_back_json' in i: ii['cm_back_json']=i['cm_back_json']
03752 r=cm_kernel.access(ii)
03753 if r['cm_return']>0: return r
03754 cm_kernel.print_for_web(r['cm_string'])
03755
03756
03757 of={}
03758 if esp:
03759 cm_kernel.print_for_con('<td valign="top"><i><small><br>')
03760 cm_kernel.print_for_con('<b>Extracting static program features using MILEPOST GCC:</b><BR><BR>')
03761
03762
03763 csl=a1.get('ctuning_setup_local','')
03764 if csl!='':
03765 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.setup.local'],
03766 'cm_action':'load',
03767 'cm_data_uoa':csl}
03768 r=cm_kernel.access(ii)
03769 if r['cm_return']>0: return r
03770
03771 dcsl=r['cm_data_obj']['cfg']
03772
03773
03774 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
03775 'cm_action':'add',
03776 'cm_repo_uoa':ini['cfg']['predict_opt']['cm_tmp_repo_uoa']}
03777 r=cm_kernel.access(ii)
03778 if r['cm_return']>0: return r
03779
03780 p=r['cm_path']
03781 p=r['cm_path']
03782 tmp_uid=r['cm_uid']
03783
03784
03785 ext='.c'
03786 compiler_type=''
03787 xcompiler_type=''
03788
03789 cl=a1.get('codelet_lang','').lower()
03790 if cl=='c': ext='.c'; compiler_type='CC'; xcompiler_type='CC'
03791 elif cl=='c++': ext='.cpp'; compiler_type='CPP'; xcompiler_type='CXX'
03792 elif cl=='fortran 77': ext='.f'; compiler_type='FC'; xcompiler_type='FC'
03793 elif cl=='fortran 90': ext='.f90'; compiler_type='F90'; xcompiler_type='F90'
03794
03795 codelet_name='tmp'+ext
03796
03797
03798 cur_dir=os.getcwd()
03799 os.chdir(p)
03800
03801
03802 cs=a1.get('codelet_source','')
03803 f=open(codelet_name, 'wt')
03804 f.write(cs)
03805 f.close()
03806
03807
03808 ii={}
03809
03810 ii['build_target_os_uoa']=dcsl['target_os_uoa']
03811 ii['run_host_os_uoa']=dcsl['host_os_uoa']
03812
03813 ctuning_compiler_plugins_code_uoa=dcsl.get('ctuning_compiler_plugins_code_uoa','')
03814 milepost_compiler_code_uoa=dcsl.get('milepost_compiler_code_uoa','')
03815
03816 milepost_compiler_extract_after_pass=dcsl.get('milepost_compiler_extract_after_pass','')
03817 if a1.get('milepost_compiler_extract_after_pass','')!='':
03818 milepost_compiler_extract_after_pass=a1['milepost_compiler_extract_after_pass']
03819
03820 optimization=dcsl.get('milepost_compiler_flags','')
03821 if a1.get('milepost_compiler_flags','')!='':
03822 optimization=a1['milepost_compiler_flags']
03823
03824 if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
03825 ii['run_set_env2']['CT_CMD']=' '+optimization+' '+codelet_name
03826 ii['run_set_env2']['CT_COMPILER_TYPE']=compiler_type
03827 ii['run_set_env2']['CT_COMPILER_TYPEX']=xcompiler_type
03828 ii['run_set_env2']['CT_MILEPOST_COMPILER_BIN_DIR']='CM_'+milepost_compiler_code_uoa+'_BIN'
03829
03830 if milepost_compiler_code_uoa=='' or ctuning_compiler_plugins_code_uoa=='':
03831 return {'cm_return':1,'cm_error':'"milepost_compiler_code_uoa" or "ctuning_compiler_plugins_code_uoa" is not defined'}
03832
03833 if len(ii.get('code_deps',[]))==0: ii['code_deps']=[]
03834 ii['code_deps'].append({"CM_CODE_DEP_MILEPOST_GCC":milepost_compiler_code_uoa})
03835 ii['code_deps'].append({"CM_CODE_DEP_CTUNING_CC_PLUGIN":ctuning_compiler_plugins_code_uoa})
03836
03837 if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
03838 if milepost_compiler_extract_after_pass!='': ii['run_set_env2']['ICI_PROG_FEAT_PASS']=milepost_compiler_extract_after_pass
03839
03840 ici_structure='ici_passes_function.'
03841 ici_properties='ici_features_function.'
03842
03843
03844 dirList=os.listdir('.')
03845 for fn in dirList:
03846 if os.path.isfile(fn) and (fn.startswith(ici_structure) or fn.startswith(ici_properties)):
03847 os.remove(fn)
03848
03849
03850
03851
03852 ii['ignore_script_error']='no'
03853 ii['skip_target_file_check']='yes'
03854
03855 ii['run_script_uoa']=ini['cfg']['predict_opt']['run_script_uoa']
03856 ii['run_script']=ini['cfg']['predict_opt']['run_script']
03857
03858 ii['run_cmd_out1']='tmp-build-output1.tmp'
03859 ii['run_cmd_out2']='tmp-build-output2.tmp'
03860
03861 ii['cm_verbose']='no'
03862
03863 ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['code.source']
03864 ii['cm_action']='build'
03865
03866 r=cm_kernel.access(ii)
03867 if r['cm_return']>0: return r
03868
03869 ici_structure='ici_passes_function.'
03870 ici_properties='ici_features_function.'
03871
03872 structure={}
03873
03874
03875 dirList=os.listdir('.')
03876 for fn in dirList:
03877 if os.path.isfile(fn):
03878 if fn.startswith(ici_structure):
03879 i1=fn.find('.')
03880 if i1>0:
03881 i2=fn.find('.',i1+1)
03882 if i2>0:
03883 func=fn[i1+1:i2]
03884 if func not in structure:
03885 structure[func]={}
03886 structure[func]['compiler_passes']=[]
03887 f=open(fn,'rt')
03888 l='x'
03889 while (l != "" ):
03890 l = f.readline().strip()
03891 structure[func]['compiler_passes'].append(l)
03892 f.close()
03893 if i.get('keep_all_files','')!='yes':
03894 os.remove(fn)
03895 elif fn.startswith(ici_properties):
03896 i1=fn.find('.')
03897 if i1>0:
03898 i2=fn.find('.',i1+1)
03899 if i2>0:
03900 func=fn[i1+1:i2]
03901 i3=fn.find('.',i2+1)
03902 if i3>0:
03903 ps=fn[i2+1:i3]
03904 if func not in structure:
03905 structure[func]={}
03906 if 'program_static_properties' not in structure[func]:
03907 structure[func]['program_static_properties']={}
03908 if ps not in structure[func]['program_static_properties']:
03909 structure[func]['program_static_properties'][ps]={}
03910 f=open(fn,'rt')
03911 l = f.readline().strip()
03912 y=-1
03913 while True:
03914 y=l.find('ft', y+1)
03915 if y>=0:
03916 i1=l.find('=',y+1)
03917 if i1>0:
03918 n=l[y+2:i1]
03919 i2=l.find(',',i1+1)
03920 if i2==-1: i2=len(l)
03921 v=l[i1+1:i2]
03922 structure[func]['program_static_properties'][ps][n]=v
03923 else: break
03924 else: break
03925 f.close()
03926 if i.get('keep_all_files','')!='yes':
03927 os.remove(fn)
03928
03929 if 'program_milepost_properties' not in of: of['program_milepost_properties']={}
03930 of['program_milepost_properties'].update(structure)
03931 changed=True
03932
03933
03934 if os.path.isfile('tmp-build-output2.tmp'):
03935 f=open('tmp-build-output2.tmp','rt')
03936 er=f.read()
03937 f.close()
03938
03939 cm_kernel.print_for_con('<div align="left"><pre>'+er+'</pre></div>')
03940
03941
03942 os.chdir(cur_dir)
03943 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
03944 'cm_action':'delete',
03945 'cm_admin':'yes',
03946 'cm_data_uoa':tmp_uid,
03947 'cm_repo_uoa':ini['cfg']['predict_opt']['cm_tmp_repo_uoa']}
03948 r=cm_kernel.access(ii)
03949 if r['cm_return']>0: return r
03950
03951 a2=of
03952
03953 cm_kernel.print_for_con('</small></i></td>')
03954
03955 cm_kernel.print_for_con('</tr></table>')
03956
03957 cm_kernel.print_for_con('<input type="submit" class="cm-button" name="cm_submit_extract_st_properties" value="Extract program\'s static properties (features)">')
03958 cm_kernel.print_for_con('<input type="submit" class="cm-button" name="cm_reset" value="Reset">')
03959
03960 cm_kernel.print_for_con('</td>')
03961
03962 if i.get('cm_console')=='web' or len(i.get('program_milepost_properties',{}))>0:
03963 a2f={}
03964 if len(i.get('program_milepost_properties',{}))>0:
03965 a2={}
03966 a2f=i['program_milepost_properties']
03967
03968 if len(a2)>0 or len(a2f)>0:
03969
03970 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-module'],
03971 'cm_action':'load',
03972 'cm_data_uoa':'ctuning.scenario.program.features.milepost'}
03973 r=cm_kernel.access(ii)
03974 if r['cm_return']>0: return r
03975
03976 dx=r['cm_data_obj']['cfg']['milepost_features_description']
03977
03978 key=ini['cfg']['predict_opt']['key_milepost_desc']
03979 ar=ini['cfg']['predict_opt']['cm_data_desc2']
03980
03981 for qx in dx:
03982 kx=qx.keys()[0]
03983 v=qx[kx]
03984
03985 kx1=kx[2:]
03986
03987 key1=key+'#'+kx1
03988 ar[key1]={'desc_text':kx.upper()+': '+v['desc'], 'sort_index':kx1, 'type':'float'}
03989
03990 if i.get('cm_console')=='web':
03991 cm_kernel.print_for_con('<hr><B>Semantic program properties (features):</B><BR>')
03992
03993 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
03994 'cm_action':'visualize_data',
03995 'cm_array':a2,
03996 'cm_data_desc':cm_data_desc2,
03997 'cm_form_commands':cm_form_commands2,
03998 'cm_separator':'#',
03999 'cm_separator_form':'#form12#',
04000 'cm_forms_exists':forms_exists,
04001 'cm_support_raw_edit':'no',
04002 'hide_add_new_object':'yes',
04003 'cm_mode':'add'}
04004 if 'cm_raw_edit' in i: ii['cm_raw_edit']=i['cm_raw_edit']
04005 if 'cm_back_json' in i: ii['cm_back_json']=i['cm_back_json']
04006 r=cm_kernel.access(ii)
04007 if r['cm_return']>0: return r
04008 cm_kernel.print_for_web(r['cm_string'])
04009
04010
04011 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
04012 'cm_action':'convert_field_to_html',
04013 'cm_value':'',
04014 'cm_key':'opt_predictor',
04015 'cm_mode':'add',
04016 'cm_data_desc':ini['cfg']['desc_opt_predictor'],
04017 'cm_disable_coloring':'yes'}
04018 r1=cm_kernel.access(ii)
04019 if r1['cm_return']>0: return r1
04020 vh=r1['cm_value_html']
04021 cm_kernel.print_for_con('Predict optimizations using '+vh+'<BR>')
04022
04023 cm_kernel.print_for_con('<input type="submit" class="cm-button" name="cm_predict_opts" value="Predict optimizations">')
04024 cm_kernel.print_for_con('<input type="submit" class="cm-button" name="cm_reset" value="Reset">')
04025
04026
04027 if i.get('cm_predict_opts','')!='' or \
04028 i.get('cm_predict_opts_state','')=='yes' or \
04029 i.get('cm_submit_prune','')!='' or \
04030 i.get('cm_submit_prune_state','')=='yes':
04031
04032 if i.get('cm_console')=='web':
04033 cm_kernel.print_for_con(' <input type="hidden" name="cm_predict_opts" value="yes">')
04034
04035 cm_kernel.print_for_con('<br>')
04036 cm_kernel.print_for_con('<hr>')
04037
04038
04039 op=i.get('opt_predictor','')
04040 if op=='nearest_neighbour':
04041
04042
04043
04044 ii=copy.deepcopy(i)
04045
04046 if len(a2f)>0:
04047 pf=a2f
04048 if len(i.get('pruning_similar_programs',''))>0:
04049 ii['pruning_similar_programs']=i['pruning_similar_programs']
04050 else:
04051 ii['pruning_similar_programs']={}
04052
04053 ii['pruning_similar_programs']['exploration_scenario_module_uoa']=ini['cfg']['cm_modules']['ctuning.scenario.compiler.optimizations']
04054
04055 cru=i.get('cm_repo_uoa','')
04056 if cru!='':
04057 ii['pruning_similar_programs']['tuning_repo_uoa']=cru
04058
04059 else:
04060
04061 ii['web_title']='Find most similar program based on semantic program features (properties), system, OS, etc:'
04062
04063 pf=a2.get('program_milepost_properties',{}).get('main',{}).get('program_static_properties',{}).get(ini['cfg']['default_pass_to_extract'],{})
04064 if ii.get('#form1##exploration_scenario_module_uoa','')=='':
04065 ii['#form1##exploration_scenario_module_uoa']=ini['cfg']['cm_modules']['ctuning.scenario.compiler.optimizations']
04066
04067 if i.get('number_of_returned_items','')!='':
04068 ii['number_of_returned_items']=i['number_of_returned_items']
04069
04070 ii['program_features']=pf
04071 ii['calculate_program_similarities']='yes'
04072
04073 rr=web_view(ii)
04074
04075 if i.get('cm_console')=='web':
04076 cm_kernel.print_for_con('</div>');
04077
04078 cm_kernel.print_for_con('</FORM><br>')
04079 cm_kernel.print_for_con('</td>')
04080
04081 return rr
04082
04083
04084 def run_stat_cmd(i):
04085
04086 """
04087 Run pipeline from CMD with user-friendly features
04088
04089 Input: {
04090 (host_os_uoa) - if not selected, take from kernel
04091 (target_os_uoa) - if not selected, take from kernel
04092 (target_processor_uoa) - if not selected, use generic-x86-64
04093
04094 compiler_code_uoa
04095 or
04096 compiler_package_uoa
04097 or
04098 compiler_package_name
04099 (compiler_fuzzy_match) - if 'compiler_package_name' is used, and 'yes' check for inclusion rather than exact match of names
04100
04101 program_uoa6
04102 run_cmd_key
04103 dataset_uoa
04104 compile_script
04105
04106 (kernel_repeat)
04107
04108 (repeat_for_statistics) - number of pipeline repetitions (for statistical analysis)
04109 (statistics_module) - module for statistical analysis
04110
04111 (update_state) - directly update state
04112
04113 (only_prepare_state) - if 'yes', do not run pipeline but only update state and description
04114 }
04115
04116 Output: {
04117 cm_return - if =0, success
04118 }
04119 """
04120
04121
04122 cur_dir=os.getcwd()
04123
04124
04125 dtp=ini['cfg']['default_target_processor_uoa']
04126
04127
04128 pipeline=ini['cfg']
04129 pss=pipeline['cm_p_uids']
04130 pipeline_desc=copy.deepcopy(pipeline['pipeline_desc'])
04131
04132
04133 ho=i.get('host_os_uoa','')
04134 to=i.get('target_os_uoa','')
04135 tp=i.get('target_processor_uoa',dtp)
04136
04137 if ho=='':
04138
04139 dos=cm_kernel.ini['dcfg'].get('cm_default_os_uoa','')
04140 if dos!='':
04141 ho=dos
04142 to=dos
04143
04144 if ho=='':
04145 return {'cm_return':1,'cm_error':'host_os_uoa is not defined'}
04146 if to=='':
04147 return {'cm_return':1,'cm_error':'target_os_uoa is not defined'}
04148 if tp=='':
04149 return {'cm_return':1,'cm_error':'target_processor_uoa is not defined'}
04150
04151
04152 zho=''
04153 zto=''
04154 ztp=''
04155
04156 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['os'],
04157 'cm_action':'load',
04158 'cm_data_uoa':ho})
04159 if r['cm_return']>0: return r
04160 zho=r['cm_data_obj']['cfg']['cm_display_as_alias']
04161
04162 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['os'],
04163 'cm_action':'load',
04164 'cm_data_uoa':to})
04165 if r['cm_return']>0: return r
04166 zto=r['cm_data_obj']['cfg']['cm_display_as_alias']
04167
04168 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['processor'],
04169 'cm_action':'load',
04170 'cm_data_uoa':tp})
04171 if r['cm_return']>0: return r
04172 ztp=r['cm_data_obj']['cfg']['cm_display_as_alias']
04173
04174
04175 cm_kernel.print_for_con('Initializing experimental pipeline "build and run" state (can take some time) ...')
04176 state={}
04177
04178 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
04179 'cm_action':'visualize_data',
04180 'cm_array':state,
04181 'cm_data_desc':pipeline_desc,
04182 'cm_separator':'#',
04183 'cm_mode':'add'}
04184 r=cm_kernel.access(ii)
04185 if r['cm_return']>0: return r
04186
04187
04188 us=i.get('update_state',{})
04189 if len(us)>0:
04190 cm_kernel.merge_arrays_ll(state, us)
04191
04192
04193 cm_kernel.print_for_con(' ')
04194 cm_kernel.print_for_con('Updating state with host and target OS, target processor and tuning repo (can take some time) ...')
04195 state[pss['init']]['input']['host_os_uoa']=ho
04196 state[pss['init']]['input']['target_os_uoa']=to
04197 state[pss['init']]['input']['target_processor_uoa']=tp
04198 ct=i.get('compilation_type','')
04199 if ct!='':
04200 state[pss['init']]['input']['compilation_type']=ct
04201
04202 tuning_repo_uoa=i.get('tuning_repo_uoa','')
04203 if tuning_repo_uoa!='':
04204 state[pss['init']]['input']['tuning_repo_uoa']=tuning_repo_uoa
04205 cm_kernel.print_for_con(' Tuning repository: '+tuning_repo_uoa)
04206
04207 cm_kernel.print_for_con(' Host OS: '+zho+' ('+ho+')')
04208 cm_kernel.print_for_con(' Target OS: '+zto+' ('+to+')')
04209 cm_kernel.print_for_con(' Target Processor: '+ztp+' ('+tp+')')
04210
04211
04212 ppc=i.get('prune_program_by_class','')
04213 if ppc!='': state[pss['init']]['input']['prune_program_by_class']=ppc
04214
04215 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
04216 'cm_action':'visualize_data',
04217 'cm_array':state,
04218 'cm_data_desc':pipeline_desc,
04219 'cm_separator':'#',
04220 'cm_mode':'update'}
04221 r=cm_kernel.access(ii)
04222 if r['cm_return']>0: return r
04223
04224
04225 ccu=i.get('compiler_code_uoa','')
04226
04227 if ccu=='':
04228 cpu=i.get('compiler_package_uoa','')
04229 cpn=i.get('compiler_package_name','')
04230 if cpu=='' and cpn=='':
04231 return {'cm_return':1, 'cm_error':'compiler_package_uoa is not defined'}
04232
04233 cm_kernel.print_for_con(' ')
04234 cm_kernel.print_for_con('Searching installed or registered compiler (can take some time) ...')
04235 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['package'],
04236 'cm_action':'installed',
04237 'prune_by_class_uoa':'72f81bd39f84ed89',
04238 'prune_by_host_os':ho,
04239 'prune_by_target_os':to,
04240 'prune_by_target_processor':tp}
04241 if cpu!='': ii['prune_by_name_uoa']=cpu
04242 elif cpn!='':
04243 ii['prune_by_name']=cpn
04244 if i.get('compiler_fuzzy_match','')=='yes':
04245 ii['fuzzy_match']='yes'
04246
04247 r=cm_kernel.access(ii)
04248 if r['cm_return']>0: return r
04249
04250 final=r['final']
04251
04252 if len(final)==0:
04253 return {'cm_return':1, 'cm_error':'can\'t find installed compilers'}
04254 elif len(final)>1:
04255 cm_kernel.print_for_con(' Installed or registered compilers:')
04256 for q in final:
04257 ccu=q['cm_uid']
04258 zccu=q['cm_display_as_alias']
04259 cm_kernel.print_for_con(' '+zccu+' ('+ccu+')')
04260 return {'cm_return':1, 'cm_error':'more than one installed compiler is returned'}
04261
04262 ccu=final[0]['cm_uid']
04263 zccu=final[0]['cm_display_as_alias']
04264
04265 cm_kernel.print_for_con(' '+zccu+' ('+ccu+')')
04266
04267
04268 programs=pipeline_desc['##'+pss['init']+'#input#program_uoa'].get('dynamic_choice',[])
04269
04270 prog=i.get('program_uoa','')
04271 if prog=='':
04272 cm_kernel.print_for_con('')
04273 cm_kernel.print_for_con('Available programs: ')
04274 for q in programs:
04275 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'], 'cm_action':'load', 'cm_data_uoa':q})
04276 if r['cm_return']>0: return r
04277 x=' '+r['cm_uid']
04278 if r['cm_alias']!='': x+=' - '+r['cm_alias']
04279 if r['cm_display_as_alias']!='': x+=' ('+r['cm_display_as_alias']+')'
04280 cm_kernel.print_for_con(x)
04281
04282 return {'cm_return':1,'cm_error':'program_uoa is not defined'}
04283
04284
04285 zprog=''
04286 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'],
04287 'cm_action':'load',
04288 'cm_data_uoa':prog})
04289 if r['cm_return']>0: return r
04290 zprog=r['cm_data_obj']['cfg']['cm_display_as_alias']
04291
04292 cm_kernel.print_for_con('')
04293 cm_kernel.print_for_con('Selected program:')
04294 cm_kernel.print_for_con(' '+zprog+' ('+prog+')')
04295
04296
04297 cm_kernel.print_for_con('')
04298 cm_kernel.print_for_con('Updating state with program and compiler to get possible command line, dataset and compilation type (it can take some time) ...')
04299 state[pss['init']]['input']['program_uoa']=prog
04300 state[pss['init']]['input']['compiler_code_uoa']=ccu
04301
04302 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
04303 'cm_action':'visualize_data',
04304 'cm_array':state,
04305 'cm_data_desc':pipeline_desc,
04306 'cm_separator':'#',
04307 'cm_mode':'update'}
04308 r=cm_kernel.access(ii)
04309 if r['cm_return']>0: return r
04310
04311
04312 run_cmd_keys=pipeline_desc['##'+pss['init']+'#input#run_cmd_key'].get('dynamic_choice',[])
04313
04314 cm_kernel.print_for_con('')
04315 cm_kernel.print_for_con('Available run command line choices for a selected program:')
04316 for q in run_cmd_keys:
04317 cm_kernel.print_for_con(' '+q)
04318
04319 rck=i.get('run_cmd_key','')
04320 if rck=='':
04321 return {'cm_return':1, 'cm_error':'run_cmd_key is not defined'}
04322
04323 cm_kernel.print_for_con('')
04324 cm_kernel.print_for_con('Selected run command line:')
04325 cm_kernel.print_for_con(' '+rck)
04326
04327
04328 cm_kernel.print_for_con('')
04329 cm_kernel.print_for_con('Updating state with a run command line choice for a selected program (can take some time) ...')
04330 state[pss['init']]['input']['run_cmd_key']=rck
04331
04332 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
04333 'cm_action':'visualize_data',
04334 'cm_array':state,
04335 'cm_data_desc':pipeline_desc,
04336 'cm_separator':'#',
04337 'cm_mode':'update'}
04338 r=cm_kernel.access(ii)
04339 if r['cm_return']>0: return r
04340
04341
04342 datasets=pipeline_desc['##'+pss['init']+'#input#dataset_uoa'].get('dynamic_choice',[])
04343
04344 if len(datasets)>0:
04345 du=i.get('dataset_uoa','')
04346
04347 if du=='':
04348 cm_kernel.print_for_con('')
04349 cm_kernel.print_for_con('Available datasets: ')
04350 for q in datasets:
04351 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['dataset'],
04352 'cm_action':'load',
04353 'cm_data_uoa':q})
04354 if r['cm_return']>0: return r
04355 x=' '+r['cm_uid']
04356 if r['cm_alias']!='': x+=' - '+r['cm_alias']
04357 cm_kernel.print_for_con(x)
04358
04359 return {'cm_return':1, 'cm_error':'dataset_uoa is not defined'}
04360
04361
04362 if du!='':
04363 zdu=''
04364 r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['dataset'],
04365 'cm_action':'load',
04366 'cm_data_uoa':du})
04367 if r['cm_return']>0: return r
04368 zdu=r['cm_data_obj']['cfg']['cm_display_as_alias']
04369
04370 cm_kernel.print_for_con('')
04371 cm_kernel.print_for_con('Number of available datasets for a selected program and run command line: '+str(len(datasets)))
04372
04373 cm_kernel.print_for_con('')
04374 cm_kernel.print_for_con('Selected dataset:')
04375 cm_kernel.print_for_con(' '+zdu+' ('+du+')')
04376
04377 cm_kernel.print_for_con('')
04378 cm_kernel.print_for_con('Updating state with dataset (can take some time) ...')
04379 state[pss['init']]['input']['dataset_uoa']=du
04380
04381 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
04382 'cm_action':'visualize_data',
04383 'cm_array':state,
04384 'cm_data_desc':pipeline_desc,
04385 'cm_separator':'#',
04386 'cm_mode':'update'}
04387 r=cm_kernel.access(ii)
04388 if r['cm_return']>0: return r
04389
04390
04391 if i.get('print_build_and_run_params','')=='yes':
04392 cm_kernel.print_for_con('')
04393 cm_kernel.print_for_con('Build parameters:')
04394 cm_kernel.print_for_con('')
04395
04396 qq='##'+pss['build_program']+'#input#program_build_properties#'
04397 for q in pipeline_desc:
04398 qx=pipeline_desc[q]
04399 if q.startswith(qq):
04400 x=' '+q[len(qq):]
04401 if qx.get('dataset_size','')=='yes':
04402 x+=' (dataset size)'
04403 cm_kernel.print_for_con(x)
04404
04405 cm_kernel.print_for_con('')
04406 cm_kernel.print_for_con('Run parameters:')
04407 cm_kernel.print_for_con('')
04408
04409 qq='##'+pss['run_program']+'#input#run_vars#'
04410 for q in pipeline_desc:
04411 qx=pipeline_desc[q]
04412 if q.startswith(qq):
04413 x=' '+q[len(qq):]
04414 if qx.get('dataset_size','')=='yes':
04415 x+=' (dataset size)'
04416 cm_kernel.print_for_con(x)
04417
04418 return {'cm_return':0}
04419
04420
04421 bp=i.get('build_params','')
04422 if len(bp)>0:
04423 if 'program_build_properties' not in state[pss['build_program']]['input']:
04424 state[pss['build_program']]['input']['program_build_properties']={}
04425 state[pss['build_program']]['input']['program_build_properties'].update(bp)
04426
04427
04428 rp=i.get('run_params','')
04429 if len(bp)>0:
04430 if 'run_vars' not in state[pss['run_program']]['input']:
04431 state[pss['run_program']]['input']['run_vars']={}
04432 state[pss['run_program']]['input']['run_vars'].update(rp)
04433
04434
04435 compile_scripts=pipeline_desc['##'+pss['init']+'#input#run_script'].get('dynamic_choice',[])
04436
04437 cm_kernel.print_for_con('')
04438 cm_kernel.print_for_con('Available compile scripts: ')
04439 for q in compile_scripts:
04440 cm_kernel.print_for_con(' '+q)
04441
04442 cs=i.get('compile_script','')
04443 if cs=='':
04444 return {'cm_return':1, 'cm_error':'dataset_uoa is not defined'}
04445
04446
04447 cm_kernel.print_for_con('')
04448 cm_kernel.print_for_con('Selected compile script:')
04449 cm_kernel.print_for_con(' '+cs)
04450
04451 cm_kernel.print_for_con('')
04452 cm_kernel.print_for_con('Updating state with compile script (can take some time) ...')
04453 state[pss['init']]['input']['run_script']=cs
04454
04455 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
04456 'cm_action':'visualize_data',
04457 'cm_array':state,
04458 'cm_data_desc':pipeline_desc,
04459 'cm_separator':'#',
04460 'cm_mode':'update'}
04461 r=cm_kernel.access(ii)
04462 if r['cm_return']>0: return r
04463
04464
04465 cf=i.get('force_compile_flags','')
04466 if cf!='':
04467 cm_kernel.print_for_con('')
04468 cm_kernel.print_for_con('Forcing compiler flags:')
04469 cm_kernel.print_for_con(' '+cf)
04470
04471 state[pss['prepare_compiler_flags']]['input']['external_flag_selection']=cf
04472
04473
04474 kr=i.get('kernel_repeat','')
04475 if kr!='':
04476 cm_kernel.print_for_con('')
04477 cm_kernel.print_for_con('Setting kernel repetition: '+kr)
04478
04479 state[pss['run_program']]['input']['calibrate']='no'
04480 if 'run_vars' not in state[pss['run_program']]['input']:
04481 state[pss['run_program']]['input']['run_vars']={}
04482 state[pss['run_program']]['input']['run_vars']['CT_REPEAT_MAIN']=kr
04483
04484
04485 if i.get('use_likwid','')=='yes':
04486 cm_kernel.print_for_con('')
04487 cm_kernel.print_for_con('Preparing likwid ...')
04488
04489 lpu=i.get('likwid_package_uoa','')
04490 lpn=i.get('likwid_package_name','')
04491 if lpu=='' and lpn=='':
04492 return {'cm_return':1, 'cm_error':'likwid_package_uoa is not defined'}
04493
04494 cm_kernel.print_for_con('')
04495 cm_kernel.print_for_con('Searching installed likwid tool (can take some time) ...')
04496 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['package'],
04497 'cm_action':'installed',
04498 'prune_by_class_uoa':'06922fb5dbed7297',
04499 'prune_by_host_os':ho,
04500 'prune_by_target_os':to,
04501 'prune_by_target_processor':tp}
04502 if lpu!='': ii['prune_by_name_uoa']=lpu
04503 elif lpn!='':
04504 ii['prune_by_name']=lpn
04505 if i.get('likwid_fuzzy_match','')=='yes':
04506 ii['fuzzy_match']='yes'
04507
04508 r=cm_kernel.access(ii)
04509 if r['cm_return']>0: return r
04510
04511 final=r['final']
04512
04513 if len(final)==0:
04514 return {'cm_return':1, 'cm_error':'can\'t find installed likwid tools'}
04515 elif len(final)>1:
04516 return {'cm_return':1, 'cm_error':'more than one installed likwid tool is returned'}
04517
04518 lcu=final[0]['cm_uid']
04519
04520 state[pss['use_likwid']]['enable']='yes'
04521 state[pss['use_likwid']]['input']['likwid_code_uoa']=lcu
04522
04523 lp=i.get('likwid_params',{})
04524 if len(lp)>0:
04525 state[pss['use_likwid']]['input'].update(lp)
04526
04527
04528 if i.get('use_vtune','')=='yes':
04529 cm_kernel.print_for_con('')
04530 cm_kernel.print_for_con('Preparing Intel vTune ...')
04531
04532 ivpu=i.get('vtune_package_uoa','')
04533 ivpn=i.get('vtune_package_name','')
04534 if ivpu=='' and ivpn=='':
04535 return {'cm_return':1, 'cm_error':'vtune_package_uoa is not defined'}
04536
04537 cm_kernel.print_for_con('')
04538 cm_kernel.print_for_con('Searching installed vtune tool (can take some time) ...')
04539 ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['package'],
04540 'cm_action':'installed',
04541 'prune_by_class_uoa':'0745ba964e97d0a4',
04542 'prune_by_host_os':ho,
04543 'prune_by_target_os':to,
04544 'prune_by_target_processor':tp}
04545 if ivpu!='': ii['prune_by_name_uoa']=ivpu
04546 elif ivpn!='':
04547 ii['prune_by_name']=ivpn
04548 if i.get('vtune_fuzzy_match','')=='yes':
04549 ii['fuzzy_match']='yes'
04550
04551 r=cm_kernel.access(ii)
04552 if r['cm_return']>0: return r
04553
04554 final=r['final']
04555
04556 if len(final)==0:
04557 return {'cm_return':1, 'cm_error':'can\'t find installed vtune tools'}
04558 elif len(final)>1:
04559 return {'cm_return':1, 'cm_error':'more than one installed vtune tool is returned'}
04560
04561 lcu=final[0]['cm_uid']
04562
04563 state[pss['prepare_vtune']]['enable']='yes'
04564 state[pss['prepare_vtune']]['input']['vtune_code_uoa']=lcu
04565
04566 vp=i.get('vtune_params',{})
04567 if len(vp)>0:
04568 state[pss['prepare_vtune']]['input'].update(vp)
04569
04570
04571 if i.get('use_perf','')=='yes':
04572 cm_kernel.print_for_con('')
04573 cm_kernel.print_for_con('Preparing perf ...')
04574
04575 state[pss['prepare_perf']]['enable']='yes'
04576
04577 pp=i.get('perf_params',{})
04578 if len(pp)>0:
04579 state[pss['prepare_perf']]['input'].update(pp)
04580
04581
04582 ii={'state':state, 'desc':pipeline_desc, 'additional_params':{}}
04583
04584
04585 rs=i.get('repeat_for_statistics','1')
04586 ii['additional_params']={'repeat_for_statistics':rs}
04587
04588 if i.get('skip_compile_output_redirect','')=='yes':
04589 state[pss['build_program']]['input']['skip_output_redirect']='yes'
04590
04591 cm_kernel.print_for_con('')
04592 cm_kernel.print_for_con('Numer of statistical repetition of experiment: '+rs)
04593
04594 sm=i.get('statistics_module','')
04595 if sm!='':
04596 ii['additional_params']['statistics_module']=sm
04597
04598 rs=i.get('record_state_before_running_pipeline_to_file','')
04599 if rs!='':
04600 rs1=os.path.join(cur_dir, rs)
04601 r4=cm_kernel.save_array_to_file_as_json({'cm_filename':rs1, 'cm_array':ii})
04602 if r4['cm_return']>0: return r4
04603
04604
04605 srcs=i.get('save_results_to_ctuning_space_uoa','')
04606 if srcs!='':
04607 cm_kernel.print_for_con('')
04608 cm_kernel.print_for_con('Experimental results will be saved in cTuning space UOA: '+srcs)
04609
04610 state[pss['save_in_ctuning_space']]['input']['save_in_existing_space_text']=srcs
04611
04612 if i.get('only_prepare_state','')=='yes':
04613 r={'cm_return':0}
04614 r.update(ii)
04615 else:
04616
04617 cm_kernel.print_for_con('')
04618 cm_kernel.print_for_con('Running experimental pipeline (can be very long) ...')
04619
04620 so=i.get('suppress_output_when_running_pipeline','')
04621 if so=='yes':
04622 cm_kernel.print_for_con('')
04623 cm_kernel.print_for_con(' Warning: output of experimental pipeline is suppressed (apart from CMD execution) ...')
04624 cm_kernel.set_print_mode({'mode':'off'})
04625 r=run_stat(ii)
04626 if so=='yes':
04627 cm_kernel.set_print_mode({'mode':'on'})
04628 if r['cm_return']>0: return r
04629
04630 if i.get('output_state_after_running_pipeline','')=='yes':
04631 print json.dumps(r, indent=2)
04632
04633 rs=i.get('record_state_after_running_pipeline_to_file','')
04634 if rs!='':
04635 rs1=os.path.join(cur_dir, rs)
04636 r4=cm_kernel.save_array_to_file_as_json({'cm_filename':rs1, 'cm_array':r})
04637 if r4['cm_return']>0: return r4
04638
04639 if i.get('output_known_characteristics','')!='':
04640 r=output_known_characteristics({'cm_characteristics':r['state'].get('cm_characteristics',{})})
04641
04642 return r
04643
04644
04645 def output_known_characteristics(i):
04646
04647 """
04648 Output known characteristics from the state
04649
04650 Input: {
04651 cm_characteristics
04652 }
04653
04654 Output: {
04655 cm_return - if =0, success
04656 }
04657 """
04658
04659 kc=i.get('cm_characteristics',{})
04660
04661 btmin=kc.get('global_build_time_min','')
04662 btmax=kc.get('global_build_time_max','')
04663 cmd5=kc.get('global_code_md5sum','')
04664 csize=kc.get('global_code_size','')
04665 cpi=kc.get('global_cpi','')
04666 ir=kc.get('global_instructions_retired','')
04667 krtmin=kc.get('global_run_kernel_time_min','')
04668 krtmax=kc.get('global_run_kernel_time_max','')
04669 kr=kc.get('kernel_repeat','')
04670
04671 cm_kernel.print_for_con('')
04672 cm_kernel.print_for_con('Global characteristics after pipeline execution:')
04673 cm_kernel.print_for_con(' Global build time (min): '+btmin+' sec.')
04674 cm_kernel.print_for_con(' Global build time (max): '+btmax+' sec.')
04675
04676 cm_kernel.print_for_con('')
04677 cm_kernel.print_for_con(' Global binary size: '+csize+' bytes')
04678 cm_kernel.print_for_con(' Global binary md5sum: '+cmd5)
04679
04680 cm_kernel.print_for_con('')
04681 cm_kernel.print_for_con(' Kernel repeat number: '+kr)
04682
04683 cm_kernel.print_for_con('')
04684 cm_kernel.print_for_con(' Global kernel run time (min): '+krtmin+' sec.')
04685 cm_kernel.print_for_con(' Global kernel run time (max): '+krtmax+' sec.')
04686
04687 cm_kernel.print_for_con('')
04688 cm_kernel.print_for_con(' Global CPI (cycles per instructions): '+cpi)
04689 cm_kernel.print_for_con(' Global instructions retired: '+ir)
04690
04691 return {'cm_return':0}
04692
04693
04694 def filter_special_str(i):
04695
04696 """
04697 Filter special string (convert to dict entry)
04698
04699 Input: {
04700 cm_string - string to filter
04701 }
04702
04703 Output: {
04704 cm_return - if =0, success
04705 cm_string - converted string
04706 }
04707 """
04708
04709 s=i['cm_string'].strip()
04710
04711 s=s.replace(' ','_')
04712 s=s.replace('[','_')
04713 s=s.replace(']','_')
04714 s=s.replace('(','_')
04715 s=s.replace(')','_')
04716 s=s.replace('\t','')
04717
04718 return {'cm_return':1, 'cm_string':s}
