module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import os
import json

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def convert_flags_from_ccc(i):

    """
    Convert flags from CCC framework

    Input:  {
              cm_filename            - CCC compiler configuration file
              (add_base_opt_upfront) - if 'yes', add base opts upfront (such as -O3, -O2, etc).
                                       One should edit final config!

              (create_cm_entry)      - if 'yes' add to cM
              (repo_uoa)             - repository to create entry
              (data_uoa)             - data uoa to create entry
            }
    """

    if 'cm_filename' not in i: return {'cm_return':1, 'cm_error':'"cm_filename" is not defined'}
    fn=i['cm_filename']

    if not os.path.isfile(fn):
       return {'cm_return':1, 'cm_error':'input CCC file not found'}

    f1=open(fn, 'r')

    l="x"

    final={}
    array=[]

    while (l != "" ):
       l = f1.readline().rstrip()
       cm_kernel.print_for_con(l)

       r=cm_kernel.gen_uid({})
       if r['cm_return']>0: return r
       uid=r['cm_uid']

       cm_kernel.print_for_con(uid)

       a={'cm_choice':'true',
          'cm_uoa':uid}

       x=l.split(',')
       if x[0]=='1':
          a['cm_type']='range'
          a['cm_prefix']=x[3].strip()
          a['cm_range_start']=x[1].strip()
          a['cm_range_stop']=x[2].strip()
          a['cm_range_step']='1'

       elif x[0]=='2':
          y=[x[1].strip(), '']
          a['cm_type']='one_of'
          a['cm_prefix']=''
          a['cm_list']=[y, '']

       elif x[0]=='3':
          y=[]
          for q in range(0, int(x[1])):
              y.append(x[q+2].strip())
          a['cm_type']='one_of'
          a['cm_prefix']=''
          a['cm_list']=y

       array.append(a)

    f1.close()

    final['compiler_flags']={}

    array1=array
    if i.get('add_base_opt_upfront','')=='yes':
       r=cm_kernel.gen_uid({})
       if r['cm_return']>0: return r
       uid1=r['cm_uid']
       r=cm_kernel.gen_uid({})
       if r['cm_return']>0: return r
       uid2=r['cm_uid']
       r=cm_kernel.gen_uid({})
       if r['cm_return']>0: return r
       uid3=r['cm_uid']

       array1={"cm_list":[{"cm_list": [
                                 "-O1", 
                                 "-O2", 
                                 "-O3", 
                                 "-Os", 
                                 "-Ofast"
                               ], 
                               "cm_type": "one_of", 
                               "cm_uoa": uid1, 
                               "cm_prefix": "", 
                               "cm_choice": "true"
                           }, 
                           {"cm_list":[
                              array
                             ],
                            "cm_type": "combine_without_order", 
                            "cm_prefix": "", 
                            "cm_uoa": uid2, 
                            "cm_choice": "true"
                           }
                          ],
               "cm_type": "select_all", 
               "cm_prefix": "", 
               "cm_uoa": uid3, 
               "cm_choice": "true"
              }


    r=cm_kernel.gen_uid({})
    if r['cm_return']>0: return r
    uid1=r['cm_uid']
    r=cm_kernel.gen_uid({})
    if r['cm_return']>0: return r
    uid2=r['cm_uid']
    array2={"cm_list": [
                         {
                           "cm_list": [
                             "-O0", 
                             "-O1", 
                             "-O2", 
                             "-O3", 
                             "-Os"
                           ], 
                           "cm_type": "one_of", 
                           "cm_uoa": uid1, 
                           "cm_prefix": "", 
                           "cm_choice": "true"
                         }, 
                       ], 
                       "cm_type": "combine_with_order", 
                       "cm_prefix": "", 
                       "cm_uoa": uid2, 
                       "cm_choice": "true"
                     }

    final['compiler_flags']['all_optimizations']=array1
    final['compiler_flags']['base_optimizations']=array2

    # Check where to write
    if i.get('create_cm_entry','')=='yes':
       ii={'cm_run_module_uoa':ini['cm_module_uoa'],
           'cm_action':'update',
           'cm_array':final}
       if 'repo_uoa' in i: ii['cm_repo_uoa']=i['repo_uoa']
       if 'data_uoa' in i: ii['cm_data_uoa']=i['data_uoa']
       r=cm_kernel.access(ii)
       if r['cm_return']>0: return r

       cm_kernel.print_for_con('Entry was created successfully: '+r['cm_uoa'])
       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Path: '+r['cm_path'])

    else:
       f2=open(fn+'.json', 'w')
       s = json.dumps(final, indent=2)
       f2.write(s+'\n')
       f2.close()

    return {'cm_return':0}

# ============================================================================
def prepare_flags(i):

    """
    Prepare flags from array

    Input:  {
              cm_data_uoa       - data with compiler description
              all_flags         - {} all flags
              (all_flags_order) - [] order of flags
            }

    Output: {
              cm_return       - if =0, success
              prepared_flags  - string with prepared flags
              all_flags_order - [] order of flags (if original was empty, 
                                prepare order according to sort_index in description)
            }
    """

    cdu=i.get('cm_data_uoa','')
    if cdu=='': return {'cm_return':1, 'cm_error':'"cm_data_uoa" is not defined in ctuning.compiler/prepare_flags'}

    # Load compiler description
    jj={'cm_run_module_uoa':ini['cm_module_uoa'],
        'cm_action':'load',
        'cm_data_uoa':cdu}
    r=cm_kernel.access(jj)
    if r['cm_return']>0: return r
    cfg=r['cm_data_obj']['cfg']

    d=cfg.get('all_compiler_flags_desc',{})
    
    all_flags=i.get('all_flags',{})
    all_flags_order=i.get('all_flags_order',[])

    afo1=[]

    pf=all_flags.get('base_flag','')

    # all other flags
#    all_flags_sorted=sorted(all_flags.keys(), key=lambda k: (int(d.get(k,{}).get('sort_index','999999')), d.get(k,{}).get('desc_text','').lower()))

    keys=all_flags.keys()
    if len(all_flags_order)>0: keys=all_flags_order

    for q in keys:
        if q in all_flags:
           afo1.append(q)

           if q!='base_flag':
              v=all_flags[q]
              if v!='':
                 if '##'+q not in d:
                    v=''
                 elif 'explore_prefix' in d['##'+q]:
                    v=d['##'+q]['explore_prefix']+v
              if v!='':
                 pf+=' '+v

    return {'cm_return':0, 'prepared_flags':pf, 'all_flags_order':afo1}

# ============================================================================
def compile_program(i):

    """
    Compile program

    Input:  {
              cm_unparsed
              (mode)
              (no_default_compiler_in_cmd)
              (default_compiler_name)
            }

    Output: {
              cm_return       - if =0, success
            }
    """

    # Check if there is a cTuning local setup 
    if ini['cfg']['env_ctuning_local_setup_uoa'] in os.environ.keys():
       x=os.environ[ini['cfg']['env_ctuning_local_setup_uoa']]
       if x!='':
          # Try to load setup
          ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.setup.local'],
              'cm_action':'load',
              'cm_data_uoa':x}
          r=cm_kernel.access(ii)
          if r['cm_return']>0: return r 
        
          d=r['cm_data_obj']['cfg']

          cm_kernel.merge_arrays({'cm_array':i, 'cm_array1':d})

    # Check if environment has additional parameter
    if ini['cfg']['env_cmd'] in os.environ.keys():
       x=os.environ[ini['cfg']['env_cmd']]
       if x!='':
          r=cm_kernel.str2cm('dummy_module dummy_action '+x) # we add dummy to be able to reuse cM kernel cmd parsing function
          if r['cm_return']>0: return r

          y=r['cm_array']
          if 'cm_run_module_uoa' in y: del(y['cm_run_module_uoa'])
          if 'cm_action' in y: del(y['cm_action'])

          cm_kernel.merge_arrays({'cm_array':i, 'cm_array1':y})

    # Get unparsed parameters (after --)
    ap=i.get('cm_unparsed',[])

    # Get mode (if any)
    mode=i.get('ct_mode','')
    if '--ct-mode' in ap:
      x=ap.index('--ct-mode')
      del(ap[x])
      if len(ap)>x:
         mode=ap[x]
         del(ap[x])

    # Get repo (if any)
    repo=i.get('ct_repo','')
    if '--ct-repo' in ap:
      x=ap.index('--ct-repo')
      del(ap[x])
      if len(ap)>x:
         repo=ap[x]
         del(ap[x])

    # Get verbosity
    vrb=i.get('ct_verbose','')
    if '--ct-verbose' in ap:
      x=ap.index('--ct-verbose')
      del(ap[x])
      if len(ap)>x:
         vrb=ap[x]
         del(ap[x])

    # Get verbosity
    out_file=i.get('ct_output_file','')
    if '--ct-output-file' in ap:
      x=ap.index('--ct-output-file')
      del(ap[x])
      if len(ap)>x:
         out_file=ap[x]
         del(ap[x])

    # Get verbosity
    clean_output_file=i.get('ct_clean_output_file','')
    if '--ct-clean-output-file' in ap:
      x=ap.index('--ct-clean-output-file')
      del(ap[x])
      clean_output_file='yes'

    # Get verbosity
    record_compilation_flow=i.get('ct_record_compilation_flow','')
    if '--ct-record-compilation-flow' in ap:
      x=ap.index('--ct-record-compilation-flow')
      del(ap[x])
      record_compilation_flow='yes'

    # If no repo, get default
    if repo=='':
       repo=ini['cfg']['ct_default_repo']

    # Check some host environment parameters
    xenv1='$';xenv2='';xenv3='.';xenv4='.sh';xenv5=';'
    if cm_kernel.is_windows():
       xenv1='%';xenv2='%';xenv3='call';xenv4='.bat';xenv5='&&'

    # Default compiler
    compiler_code_uoa=i.get('compiler_code_uoa','')

    # Check if help, otherwise continue processing ***********************************************
    if mode=='help' or '-h' in ap or '-help' in ap or '--help' in ap or '-?' in ap:
      cm_kernel.print_for_con("")
      cm_kernel.print_for_con("cM machine-learning based compiler wrapper")
      cm_kernel.print_for_con("")
      cm_kernel.print_for_con("  cm ctuning-compiler (cTuning parameters) ... @config.json -- <UNCHANGED COMMAND LINE for a user compiler>")
      cm_kernel.print_for_con("")
      cm_kernel.print_for_con("   cTuning parameters:")
      cm_kernel.print_for_con("     ct_mode=help                    - print this help")
      cm_kernel.print_for_con("             test-repo               - test current repository with characterized computer systems")
      cm_kernel.print_for_con("                                          and their profitable optimization cases")
      cm_kernel.print_for_con("             clean-output-file       - clean output file")
      cm_kernel.print_for_con("             record-compilation-flow - record compilation flow")
      cm_kernel.print_for_con("             <empty>                 - transparently call user compiler with user parameters")
      cm_kernel.print_for_con("     ct_verbose=yes/no               - print various Debug info")
      cm_kernel.print_for_con("     ct_repo=<REPO UOA>              - select current repo UOA (by default, cmind)")
      cm_kernel.print_for_con("     ct_out_file=<output file>       - output file (if needed)")
      cm_kernel.print_for_con("     ct_clean_out_file=yes/no        - cleaning output file")
      cm_kernel.print_for_con("")
      cm_kernel.print_for_con("")
      cm_kernel.print_for_con(" Alternative call:")
      cm_kernel.print_for_con("")
      cm_kernel.print_for_con("  ctuning-cc / ctuning-cpp / ctuning-fortran (cTuning flags) <UNCHANGED COMMAND LINE for a user compiler>")
      cm_kernel.print_for_con("   cTuning flags:")
      cm_kernel.print_for_con("     -h / -help / --help / -?       - print this help")
      cm_kernel.print_for_con("     --ct-mode help                 - print this help")
      cm_kernel.print_for_con("     --ct-verbose yes               - print various Debug info")
      cm_kernel.print_for_con("     --ct-out-file <filename>       - output file")
      cm_kernel.print_for_con("     --ct-clean-output-file         - clean output file")
      cm_kernel.print_for_con("     --ct-record-compilation-flow   - record compilation flow")

      exit(0)

    # Test repo ***********************************************
    elif mode=='test-repo' or '-tr' in ap or '--test-repo' in ap:
      cm_kernel.print_for_con('Testing repository with characterized computer systems and their profitable cases')

      r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-repo'],
                          'cm_action':'test',
                          'test_repo_uoa':repo})
      cm_kernel.print_for_con('')
      if r['cm_return']>0: return r

      cm_kernel.print_for_con('Test passed successfully!')

      exit(0)

    # Select collective optimization from repository ***************************************
    if mode=='clean_output_file' or clean_output_file=='yes':
      if vrb=='yes':
         cm_kernel.print_for_con('')
         cm_kernel.print_for_con('Removing output file '+out_file+' ...')
      if out_file!='':
         if os.path.isfile(out_file):
            os.remove(out_file)  
            if vrb=='yes':
               cm_kernel.print_for_con('')
               cm_kernel.print_for_con('Output file cleaned!')

    # ***************** Compile program, if needed *****************

    # Check various additional cTuning params 
    # and remove them from the unparsed cmd if needed

    # Check if default compiler name is specified
    dcn=i.get('default_compiler_name','')

    # Check if first parameter is compiler name
    # I.e. we can allow both to use a default compiler name and just prefix with cm
    # or fully substitute it from the start - it depends on user needs and configuration
    ndc=i.get('no_default_compiler_in_cmd','')

    # Compiler type
    compiler_type=i.get('compiler_type','')

    if ndc=='yes':
       if dcn=='': 
          if compiler_type!='':
             dcn=xenv1+'CM_'+compiler_type+xenv2
          else:
             return {'cm_return':1, 'cm_error':'default compiler is not defined (no "compiler_type" in CMD)'}
    else:
       if len(ap)>0:
          dcn=ap[0]
          del(ap[0])
       else:
          return {'cm_return':1, 'cm_error':'no default compiler in cmd'}

    # Now we have default cmd without compiler (need to add ndc)
    # FGG separated it since default compiler can have additional options so
    # we will not be able to change it when substituting with various ICI/OpenME
    # feature extractors ...
    cmd=''
    for x in ap: #i.get('cm_unparsed',[]):
        if ' ' in x: x='"'+x+'"' #      We need to add " to values with spaces otherwise will break into sub-values
        cmd+=x+' '

    xcmd=''
    ycmd=''

    # Check if need to remove flags
    if mode=='extract_milepost_properties' or mode=='predict_opt' or mode=='select_collective_opt':
       tccdu=i.get('target_ctuning_compiler_desc_uoa','')

       # Remove opts from original CMD
       r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                           'cm_action':'load',
                           'cm_data_uoa':tccdu})
       if r['cm_return']>0: return r
       da=r['cm_data_obj']['cfg'].get('all_compiler_flags_desc',{})

       if vrb=='yes':
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Removing old optimization flags from CMD:')
       ap1=[]
       for q in ap:
           add=True
           for k in da:
               v=da[k]
               ep=v.get('explore_prefix','')

               if ep!='':
                  if q.startswith(ep):
                     add=False
               else:
                  ch=v.get('choice',[])
                  for l in ch:
                      if q==l:
                         add=False
                         break

               if not add:
                  xcmd+=' '+q
                  break

           if add: ap1.append(q)

       xcmd=xcmd.strip()

       for x in ap1: #i.get('cm_unparsed',[]):
           if ' ' in x: x='"'+x+'"' #      We need to add " to values with spaces otherwise will break into sub-values
           ycmd+=x+' '

    # Read output file if needed
    of={}
    changed=False
    if out_file!='' and os.path.isfile(out_file):
       r=cm_kernel.load_json_file({'cm_filename':out_file})
       if r['cm_return']>0: return r
       of=r['cm_array']

    # If cTuning mode is not selected, simply call default cmd
    run_default_cmd=True

    # Record compilation flow *****************************************************************************************
    if mode=='record_compilation_flow' or record_compilation_flow=='yes':
       if vrb=='yes':
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('cTuning mode: recording compilation flow ...')

       if 'compilation_flow' not in of: of['compilation_flow']=[]
       of['compilation_flow'].append(dcn+' '+cmd)
       changed=True

    # Select collective optimization from repository ******************************************************************
    elif mode=='extract_milepost_properties' or mode=='predict_opt':
       if vrb=='yes':
          cm_kernel.print_for_con('')
          if mode=='extract_milepost_properties':
             cm_kernel.print_for_con('cTuning mode: extracting semantic program properties (features) ...')
          elif mode=='predict_opt':
             cm_kernel.print_for_con('cTuning mode: predict optimization using semantic program properties (features) ...')

          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Preparing cTuning CC / MILEPOST GCC ...')

       ii={}

       if vrb!='yes': ii['cm_verbose']='no'

       ii['build_target_os_uoa']=i['target_os_uoa']
       ii['run_host_os_uoa']=i['host_os_uoa']

       milepost_compiler_extract_after_pass=i.get('milepost_compiler_extract_after_pass','')
       milepost_compiler_flags=i.get('milepost_compiler_flags','')

       ctuning_compiler_plugins_code_uoa=i.get('ctuning_compiler_plugins_code_uoa','')
       milepost_compiler_code_uoa=i.get('milepost_compiler_code_uoa','')
       functions=i.get('ct_functions',[])

       xcompiler_type=i['compiler_type']
       if xcompiler_type=='CPP': xcompiler_type='CXX'

       if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
       ii['run_set_env2']['CT_CMD']=' '+milepost_compiler_flags+' '+ycmd
       ii['run_set_env2']['CT_COMPILER_TYPE']=compiler_type
       ii['run_set_env2']['CT_COMPILER_TYPEX']=xcompiler_type
       ii['run_set_env2']['CT_MILEPOST_COMPILER_BIN_DIR']='CM_'+milepost_compiler_code_uoa+'_BIN'

       if milepost_compiler_code_uoa=='' or ctuning_compiler_plugins_code_uoa=='':
          return {'cm_return':1,'cm_error':'milepost_compiler_code_uoa or ctuning_compiler_plugins_code_uoa is not defined'}

       if len(ii.get('code_deps',[]))==0: ii['code_deps']=[]
       ii['code_deps'].append({"CM_CODE_DEP_MILEPOST_GCC":milepost_compiler_code_uoa})              # first MILEPOST GCC
       ii['code_deps'].append({"CM_CODE_DEP_CTUNING_CC_PLUGIN":ctuning_compiler_plugins_code_uoa})  # then cTuning CC plugins
       if compiler_code_uoa!='':
          ii['code_deps'].append({"CM_CODE_DEP_MAIN_COMPILER":compiler_code_uoa})                   # then real compiler if not default

       if len(ii.get('run_set_env2',{}))==0: ii['run_set_env2']={}
       if milepost_compiler_extract_after_pass!='': ii['run_set_env2']['ICI_PROG_FEAT_PASS']=milepost_compiler_extract_after_pass

       ici_structure='ici_passes_function.'
       ici_properties='ici_features_function.'

       # Clean old files
       if i.get('keep_all_files','')!='yes':
          dirList=os.listdir('.')
          for fn in dirList:
              if os.path.isfile(fn) and (fn.startswith(ici_structure) or fn.startswith(ici_properties)):
                 os.remove(fn)

       # Check if functions are selected
       if len(functions)>0:
          rx=cm_kernel.gen_uid({})
          if rx['cm_return']>0: return rx
          file_with_funcs=os.path.join(os.getcwd(), 'tmp-funcs-'+rx['cm_uid']+'.txt')
          f=open(file_with_funcs,'wt')
          for q in functions:
              f.write(q+'\n')
          f.close()

          ii['run_set_env2']['ICI_FILE_SELECT_FUNCTIONS']=file_with_funcs

       # Check which flags to use!
       #ii['run_set_env2']['CM_COMPILER_FLAGS_CC_OPTS']=''

       if i.get('keep_all_files','')!='':
          ii['keep_all_files']=i['keep_all_files']

       ii['ignore_script_error']='no'
       ii['skip_target_file_check']='yes'

       ii['run_script_uoa']=ini['cfg']['run_script_uoa']
       ii['run_script']=ini['cfg']['run_script']

       ii['run_cmd_out1']='tmp-build-output1.tmp'
       ii['run_cmd_out2']='tmp-build-output2.tmp'

       ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['code.source']
       ii['cm_action']='build'

       # Building program to get program structure and extract features
       if vrb=='yes':
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Building program ...')

       r=cm_kernel.access(ii)
       if r['cm_return']>0: return r

       ici_structure='ici_passes_function.'
       ici_properties='ici_features_function.'

       structure={}

       # Read program structure
       dirList=os.listdir('.')
       for fn in dirList:
           if os.path.isfile(fn):
              if fn.startswith(ici_structure):
                 i1=fn.find('.')
                 if i1>0:
                    i2=fn.find('.',i1+1)
                    if i2>0:
                       func=fn[i1+1:i2]
                       if func not in structure:
                          structure[func]={}
                       structure[func]['compiler_passes']=[]
                       f=open(fn,'rt')
                       l='x'
                       while (l != "" ):
                          l = f.readline().strip()
                          structure[func]['compiler_passes'].append(l)
                       f.close()
                       if i.get('keep_all_files','')!='yes':
                          os.remove(fn)
              elif fn.startswith(ici_properties):
                 i1=fn.find('.')
                 if i1>0:
                    i2=fn.find('.',i1+1)
                    if i2>0:
                       func=fn[i1+1:i2]
                       i3=fn.find('.',i2+1)
                       if i3>0:
                          ps=fn[i2+1:i3]
                          if func not in structure:
                             structure[func]={}
                          if 'program_static_properties' not in structure[func]:
                             structure[func]['program_static_properties']={}
                          if ps not in structure[func]['program_static_properties']:
                             structure[func]['program_static_properties'][ps]={}
                          f=open(fn,'rt')
                          l = f.readline().strip()
                          y=-1
                          while True:
                            y=l.find('ft', y+1)
                            if y>=0:
                               i1=l.find('=',y+1)
                               if i1>0:
                                  n=l[y+2:i1]
                                  i2=l.find(',',i1+1)
                                  if i2==-1: i2=len(l)
                                  v=l[i1+1:i2]
                                  structure[func]['program_static_properties'][ps][n]=v
                               else: break
                            else: break
                          f.close()
                          if i.get('keep_all_files','')!='yes':
                             os.remove(fn)

       if 'program_milepost_properties' not in of: of['program_milepost_properties']={}
       of['program_milepost_properties'].update(structure)
       changed=True

       if len(functions)>0:
          if os.path.isfile(file_with_funcs): os.remove(file_with_funcs)

       # Clean the file mess
       for q in ini['cfg']['clean_files']:
           if os.path.isfile(q): os.remove(q)

    # Select collective optimization from repository ******************************************************************
    elif mode=='select_collective_opt':
       if vrb=='yes':
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('cTuning mode: selecting collective optimization from a repository ...')

       opt_uid=i.get('ct_opt_uid','')

       if opt_uid!='':
          return {'cm_return':1, 'cm_error':'"opt_uid" is not defined'}

       if vrb=='yes':
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Obtaining compiler flags from the repository ('+repo+') ...')

       ocdu=i.get('opt_class_data_uoa','')
       ocu=i.get('opt_class_uid','')
       to=i.get('tuning_objective','')
       
       if ocdu!='' and ocu!='' and to!='' and tccdu!='':
          r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.compiler.optimizations'],
                              'cm_action':'load',
                              'cm_data_uoa':ocdu,
                              'cm_repo_uoa':repo})
          if r['cm_return']>0: return r

          d=r['cm_data_obj']['cfg']

          dx=d.get('opt_classes',{}).get(to,{})

          found=False
          for q in dx:
              if q.get('cm_uid','')==ocu:
                 found=True
                 break

          opt=q.get('prepared_compiler_flags','')

          cm_kernel.print_for_con('')
          if xcmd=='':
             cm_kernel.print_for_con('No optimization flags were detected for a target compiler ...')
          else:
             cm_kernel.print_for_con('Original flags:')
             cm_kernel.print_for_con(xcmd)

          if opt!='' and vrb=='yes':
             cm_kernel.print_for_con('')
             cm_kernel.print_for_con('Adding new flags ...')

          cmd=opt+' '+ycmd

    # Predict optimization based on similarities between programs *****************************************************
    if mode=='predict_opt':
       # Find first encountered features (for now for the first function - later add per function)
       ft={}

       q1=of.get('program_milepost_properties',{})
       qq={}
       for qf in q1:
           qx=q1[qf].get('program_static_properties',{})
           if len(qx)>0:
              kx=qx.keys()[0]
              ft=qx[kx]
              qq=qx
 
       if len(ft)>0:
          if vrb=='yes':
             cm_kernel.print_for_con('')
             cm_kernel.print_for_con('Extracted following features:')
             cm_kernel.print_for_con(json.dumps(ft))

             cm_kernel.print_for_con('')
             cm_kernel.print_for_con('Searching most close programs with speedups ...')

          pmp={'program_milepost_properties':{qf:qq}}
          r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['ctuning.pipeline.build_and_run_program'],
                              'cm_action':'predict_opts',
                              'program_milepost_properties':ft,
                              'cm_submit_prune':'yes',
                              'opt_predictor':'nearest_neighbour', # FGG TBD: move outside
                              'pruning_similar_programs':i.get('pruning_similar_programs',{}),
                              'number_of_returned_items':i.get('number_of_returned_items',''),
                              'cm_repo_uoa':repo})
          if r['cm_return']>0: return r

#          print json.dumps(r, indent=2)

          sp=r.get('similar_programs',[])
          if len(sp)==0:
             if vrb=='yes':
                cm_kernel.print_for_con('')
                cm_kernel.print_for_con('Close programs not found in the repository!')
 
          else:
             if vrb=='yes':
                cm_kernel.print_for_con('')
                cm_kernel.print_for_con('Found similar program(s) in the repository!')

             most_close_program=sp[0]
             if vrb=='yes':
                cm_kernel.print_for_con('')
                cm_kernel.print_for_con('Most close program: '+most_close_program['program_name']+' (distance='+most_close_program['milepost_distance']+')')
             
                # Get only first speed up point
                # FGG TBD: get/print more points with different speedups
                spp=r['similar_programs_points']
                sppi=r['similar_programs_points_improvements']

                if len(spp)<1 or len(sppi)<1:
                   cm_kernel.print_for_con('')
                   cm_kernel.print_for_con('Solutions are not found for the most close program!')
                else:
                   xspp=spp[1]
                   xsppi=sppi[1]

                   cm_kernel.print_for_con('Execution time speedup for the most close program: '+xsppi.get('#75581c4a4e4072c8#output#global_execution_time_div_by_repeat',''))

                   opt=xspp.get('#1048eba38df5b240#output#prepared_compiler_flags','')

                   cm_kernel.print_for_con('')
                   if xcmd=='':
                      cm_kernel.print_for_con('No optimization flags were detected for a target compiler ...')
                   else:
                      cm_kernel.print_for_con('Original flags:')
                      cm_kernel.print_for_con(xcmd)

                   if opt!='' and vrb=='yes':
                      cm_kernel.print_for_con('')
                      cm_kernel.print_for_con('Reusing new flags from most similar program from the repository ...')

                   cmd=opt+' '+ycmd

    # Write output file if needed
    if changed:
       if out_file!='':
          r=cm_kernel.save_array_to_file_as_json({'cm_filename':out_file, 'cm_array':of})
          if r['cm_return']>0: return r

    # Call default 
    if run_default_cmd:
       # Default cmd
       dcmd=dcn+' '+cmd

       if vrb=='yes':
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con('Executing the following command:')
          cm_kernel.print_for_con('')
          cm_kernel.print_for_con(dcmd)
          cm_kernel.print_for_con('')

       if compiler_code_uoa!='':
          dcmd=xenv3+' cm_code_env_'+compiler_code_uoa+xenv4+' '+xenv5+' '+dcmd 

       r=os.system(dcmd)

       # Process return code
       if r>0:
          if r>255: r=r >> 8 # use only first 8 bits; the rest is usually the signal that killed process on Linux
          return{'cm_return':r, 'cm_error':'compiler returned non-zero code ('+str(r)+')'}

    return {'cm_return':0}

# ============================================================================
def prepare_environment(i):

    """
    Prepare environment

    Input:  {
            }

    Output: {
              cm_return       - if =0, success
            }
    """


    return {'cm_return':0}