module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import string
import os
import random
import copy
import json

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def explore(i):

    """
    Explore choices

    Input:  {
              cm_choices                - TODO: describe format, see ct-compiler gcc-4.6.2
              cm_exploration_strategy   - random
              (choice_rnd_seed)         - random seed
              *concatenate_output)      - if 'yes', concatenate list to string
              (output_key)              - use this value to set output key instead of cm_choice.
                                          Mainly used for command line exploration.
            }

    Output: {
              cm_return                        - if =0, success
              cm_choice or value of output_key - list of selected choices
            }
    """

    if 'cm_choices' not in i:
       return {'cm_return':1, 'cm_error':'"cm_choices" are not defined'}
    if 'cm_exploration_strategy' not in i:
       return {'cm_return':1, 'cm_error':'"cm_exploration_strategy" is not defined'}

    output_key='cm_choice'
    if 'output_key' in i: output_key=i['output_key']

    choices=i['cm_choices']
    choice=[]

    ii={'cm_choices':choices, 
        'cm_exploration_strategy':i['cm_exploration_strategy'],
        'cm_choice':choice}
    if 'choice_rnd_seed' in i: ii['choice_rnd_seed']=i['choice_rnd_seed']

    r=select(ii)

    if r['cm_return']==0:
       # Check if concatenate
       if i.get('concatenate_output','')=='yes':
          s=''
          for x in r['cm_choice']:
              if i.get('concatenate_output','')=='yes':
                 s+=' '+x
          r['cm_choice']=s.strip()

       if i.get('cm_console','')=='txt':
          if i.get('concatenate_output', '')=='yes': 
             cm_kernel.print_for_con(s.strip())
          else:
             for x in r['cm_choice']:
                 cm_kernel.print_for_con(x)

       if 'cm_choice' in r and 'output_key' in i:
          rx=cm_kernel.set_value_by_flattened_key({'cm_array':r, 'cm_key':output_key, 'cm_value':r['cm_choice']})
          if rx['cm_return']>0: return rx
          r=rx['cm_array']
          if 'cm_choice' in r: del(r['cm_choice'])

    return r

# ============================================================================
def select(i):
    """
    Select from the choice

    Input:  {
              cm_choices              - TODO: describe format, see ct-compiler gcc-4.6.2
              cm_exploration_strategy - random
              (choice_rnd_seed)       - random seed
              (cm_choice)             - previous cm_choice state (can be updated using various methods including probabilistic)
            }

    Output: {
              cm_return - return code = 0, if successful
              cm_choice
            }
    """

    if 'cm_choices' not in i:
       return {'cm_return':1, 'cm_error':'"cm_choices" are not defined'}
    if 'cm_exploration_strategy' not in i:
       return {'cm_return':1, 'cm_error':'"cm_exploration_strategy" is not defined'}

    choices=i['cm_choices']
    choice=[]
    if 'cm_choice' in i: choice=i['cm_choice']
    es=i['cm_exploration_strategy']

    if 'choice_rnd_seed' in i:
       random.seed(int(i['choice_rnd_seed']))

    if 'cm_choice' in choices and choices['cm_choice']=='true':
       if es=='random':
          cm_prefix=''
          if 'cm_prefix' in choices: cm_prefix=choices['cm_prefix']
          if choices['cm_type']=='combine_without_order':
             # Select random length!

             ln=random.randrange(0, len(choices['cm_list']))
             choice1=[]
             for z in range(0, ln):
                 y=random.randrange(0, len(choices['cm_list'])-1)
                 while y in choice1:
                       y=random.randrange(0, len(choices['cm_list'])-1)
                 choice1.append(y)

                 x=choices['cm_list'][y]
                 if type(x) is dict and 'cm_choice' in x and x['cm_choice']=='true':
                    r=select({'cm_choices':x, 
                              'cm_exploration_strategy':i['cm_exploration_strategy'],
                              'cm_choice':choice})
                    if r['cm_return']>0: return r
                 else:
                    choice.append(cm_prefix+x)

#             for x in choices['cm_list']:
#                 y=random.randrange(0,2)
#                 if y==0: 
#                    if type(x) is dict and 'cm_choice' in x and x['cm_choice']=='true':
#                       r=select({'cm_choices':x, 
#                                 'cm_exploration_strategy':i['cm_exploration_strategy'],
#                                 'cm_choice':choice})
#                       if r['cm_return']>0: return r
#                    else:
#                       choice.append(cm_prefix+x)
          elif choices['cm_type']=='combine_with_order':
             for x in choices['cm_list']:
                 y=random.randrange(0,2)
                 if y==0: 
                    if type(x) is dict and 'cm_choice' in x and x['cm_choice']=='true':
                       r=select({'cm_choices':x, 
                                 'cm_exploration_strategy':i['cm_exploration_strategy'],
                                 'cm_choice':choice})
                       if r['cm_return']>0: return r
                    else:
                       choice.append(cm_prefix+x)
          elif choices['cm_type']=='select_all':
             for x in choices['cm_list']:
                 if type(x) is dict and 'cm_choice' in x and x['cm_choice']=='true':
                    r=select({'cm_choices':x, 
                              'cm_exploration_strategy':i['cm_exploration_strategy'],
                              'cm_choice':choice})
                    if r['cm_return']>0: return r
                 else:
                    choice.append(cm_prefix+x)
          elif choices['cm_type']=='one_of':
               list=choices['cm_list']
               y=random.randrange(0,len(list))
               x=list[y]
               if type(x) is dict and 'cm_choice' in x and x['cm_choice']=='true':
                  r=select({'cm_choices':x, 
                            'cm_exploration_strategy':i['cm_exploration_strategy'],
                            'cm_choice':choice})
                  if r['cm_return']>0: return r
               else:
                  choice.append(cm_prefix+x)
          elif choices['cm_type']=='range':
               r1=int(choices['cm_range_start'])
               r2=int(choices['cm_range_stop'])
               rs=int(choices['cm_range_step'])
               rx=(r2-r1+1)/rs
               y=random.randrange(0,rx)
               choice.append(cm_prefix+str(r1+(y*rs)))
       else:
           return {'cm_return':1, 'cm_error':'choice type "'+es+'"is not recognized'}

    return {'cm_return':0, 'cm_choice':choice}

# ============================================================================
def prepare_tuning_dimension_for_web(i):

    """
    Prepare tuning dimension for web

    Input:  {
              key1 
              key2 
              key_ext
              form_name
              form_explore
              cm_data_desc - data description
            }

    Output: {
              cm_return  - if =0, success
              cm_string  - html for choices, explore strategy and tuning parameters
            }
    """
    key1=i.get('key1','')
    key2=i.get('key2','')
    key_ext=i.get('key_ext','')
    form_name=i.get('form_name','')
    form_explore=i.get('form_explore',{})
    d=i.get('cm_data_desc',{})
    t=d.get('type','')

    s='<TD><B><center>Explore<BR>choices:</center></b></TD><TD>'

    v1=''
    vstart=''
    vstop=''
    vstep=''
    if key1 in form_explore:
       v1=form_explore[key1].get('type','')
       vstart=form_explore[key1].get('start','')
       vstop=form_explore[key1].get('stop','')
       vstep=form_explore[key1].get('step','')

    if v1=='': v1=d.get('explore_type','')
    if vstart=='': vstart=d.get('explore_start','')
    if vstop=='': vstop=d.get('explore_stop','')
    if vstep=='': vstep=d.get('explore_step','')

    s+='<select class="cm-select" style="width:170px" name="@explore_type@'+key2+'"'+key_ext+'>\n'
#       ' onchange="document.'+form_name+'.submit();">\n'

    for xx in ini['cfg']['cm_explore_types']:
        yy=xx.keys()[0]
        x=xx[yy]

        z=''
        if yy==v1: z=' SELECTED '

        s+='<option value="'+yy+'"'+z+'>'+x+'</option>\n'
    s+='</TD>'

    if d.get('has_choice','')!='yes' and (t=='integer' or t=='float'):
       s+='<TD><B><center>Start:</center></b></TD><TD>'
       s+='<input type="text" size="7" spellcheck="false" name="@explore_start@'+key2+'"'+key_ext+' value="'+vstart+'"></TD>'

       s+='<TD><B><center>Stop:</center></b></TD><TD>'
       s+='<input type="text" size="7" spellcheck="false" name="@explore_stop@'+key2+'"'+key_ext+' value="'+vstop+'"></TD>'

       s+='<TD><B><center>Step:</center></b></TD><TD>'
       s+='<input type="text" size="7" spellcheck="false" name="@explore_step@'+key2+'"'+key_ext+' value="'+vstep+'"></TD>'

    return {'cm_return':0, 'cm_string':s}

# ============================================================================
def prepare_choices(i):

    """
    Prepare tuning dimension

    Input:  {
              cm_array         - pipeline array
              cm_data_desc     - pipeline data description
              cm_explore       - exploration parameters 
              cm_explore_state - exploration state
              cm_explore_level - exploration level ('' - coarse-grain, '1' - next level, ...)
              random_seed      - random seed
            }

    Output: {
              cm_return               - if =0, success
              cm_explore_state        - exploration state
              cm_exploration_finished - if 'yes', exploration finished (for nested loops)
            }
    """

    a=i.get('cm_array',{})
    d=i.get('cm_data_desc',{})
    de=i.get('cm_explore',{})

    if i.get('random_seed','')!='': random.seed(int(i['random_seed']))

    # FGG: Need to save data description otherwise choices created at run-time for different dimensions 
    #  mix up/slow down logic - it's not a problem but should be cleaned up/optimized one day
    d_copy=copy.deepcopy(d)

    # Get initial state with dimensions and choices
    e=i.get('cm_explore_state',[])
    init=False
    if len(e)==0:
       # Init state
       init=True
       ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
           'cm_action':'visualize_data',
           'cm_array':a,
           'cm_data_desc':d,
           'cm_form_explore':de,
           'cm_separator':'#',
           'cm_mode':'add',
           'cm_explore':'yes',
           'cm_explore_state':e,
           'cm_explore_level':i.get('cm_explore_level','')}
       r=cm_kernel.access(ii)
       if r['cm_return']>0: return r

       e=r['cm_explore_state']

    # Iteratively prepare pipeline (each time expanding dependant dimensions)
    ef='no'
    li='no'
    l=len(e)

    # Check nested loops from highest to lowest
    loop_finished=False
    has_random=False
    has_loop=False
    other_loop_reset=False
    for t in range(16,0,-1):
        # Scan to check that there is a loop
        loop='loop_'+str(t)
        found_loop=False

        for n in e:
            if n.get('type','')==loop:
               found_loop=True
               break

        if found_loop or t==1:
           cm_kernel.print_for_con('')
           if found_loop: 
               cm_kernel.print_for_con('Processing nested loop: '+str(t))
               cm_kernel.print_for_con('')

           for j in range(0,l):
               n=e[j]

               # Prepare some vars
               desc=n.get('desc',{})
               dt=desc.get('type','')

               x=n.get('index','')
               t=n.get('type','')
               v=n.get('value','')
               c=n.get('choice',[])

               vstart=n.get('start','')
               vstop=n.get('stop','')
               vstep=n.get('step','')
               if vstart=='': vstart='0'
               if vstop=='': vstop='0'
               if vstep=='': vstep='0'

               if dt=='uoa' or len(c)>0 or desc.get('has_choice','')=='yes': 
                  isc=True
               else:
                  isc=False
                  if dt=='integer':
                     rstart=int(vstart)
                     rstop=int(vstop)
                     rstep=int(vstep)
                  else:
                     rstart=float(vstart)
                     rstop=float(vstop)
                     rstep=float(vstep)

               h=''

               # Prepare according exploration type
               if t=='disabled':
                  v=''
                  h=' (disabled)'
               elif t=='fixed':
                  # in some cases we fix empty selection (dataset depending on program)
                  # if v=='': return {'cm_return':1, 'cm_error':'dimension '+x+' is fixed, but value is not set'}
                  h=' (fixed)'
               elif t=='fixed_first':
                  if isc:
                     v=''
                     if len(c)>0: v=c[0]
                  h=' (fixed first)'
               elif t=='random':
                  # first check if enable or disable
                  u=int(random.randint(0,1))
                  if n.get('desc',{}).get('forbid_disable_at_random','')=='yes' or u==0:
                     has_random=True
                     v=''
                     if isc:
                        if len(c)>0: 
                           if len(c)==1: v=c[0]
                           else: v=c[random.randint(0, len(c)-1)]
                        h=' (randomly selected from range {'+str(len(c))+'})'
                     else:
                        rx=(rstop-rstart)/rstep
                        y=random.randint(0,rx)
                        v=str(rstart+(y*rstep))
                        h=' (randomly selected from range {'+str(rstart)+', '+str(rstop)+', '+str(rstep)+'})'
                  else:
                     v=''
                     h=' (randomly disabled)'

               elif t.startswith('loop_'):
                  has_loop=True
                  if init:
                     v=''
                     if isc:
                        if len(c)>0: v=c[0]
                     else:
                        v=str(rstart)
                     h=' (loop initialized)'
                  else:
                     if other_loop_reset:
                        v=''
                        if isc:
                           if len(c)>0: v=c[0]
                        else:
                           v=str(rstart)
                        h=' (loop reset)'
                     elif t==loop:
                        if isc:
                           if len(c)>0:
                              index=c.index(v)
                              index+=1
                              if index<len(c): 
                                 v=c[index]
                                 if loop_finished:
                                    loop_finished=False
                                    other_loop_reset=True
                                 h=' (loop iterated)'
                              else: 
                                 v=c[0]
                                 loop_finished=True
                                 h=' (loop finished)'
                           else: 
                              v=''
                              loop_finished=True
                              h=' (loop finished)'
                        else:
                           if dt=='integer':
                              vv=int(v)
                           else:
                              vv=float(v)
                           vv+=rstep
                           if vv<=rstop:
                              if loop_finished:
                                 loop_finished=False
                                 other_loop_reset=True
                              h=' (loop iterated)'
                           else: 
                              vv=rstart
                              loop_finished=True
                              h=' (loop finished)'
                           v=str(vv)
                           
               else:
                  h=' (exploration type empty or unknown - skipping)'
                  continue

               cm_kernel.print_for_con('Dimension: '+x+' = '+v+h)

               # Set value
               ro=cm_kernel.set_value_by_flattened_key({'cm_array':a,
                                                        'cm_key':x,
                                                        'cm_value':v})
               if ro['cm_return']>0: return ro

               # Refresh pipeline if needed
               if desc.get('skip_form_refresh','')=='yes':
                  e[j]['value']=v
               else:
                  e1=[]
                  ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                      'cm_action':'visualize_data',
                      'cm_array':a,
                      'cm_data_desc':d,
                      'cm_form_explore':de,
                      'cm_separator':'#',
                      'cm_mode':'add',
                      'cm_forms_exists':'yes',
                      'cm_explore':'yes',
                      'cm_explore_state':e1,
                      'cm_explore_level':i.get('cm_explore_level','')}
                  r=cm_kernel.access(ii)
                  if r['cm_return']>0: return r

                  e1=r['cm_explore_state']

                  # Get value (after pipeline refresh)
                  ro=cm_kernel.get_value_by_flattened_key({'cm_array':a,
                                                           'cm_key':x})
                  if ro['cm_return']>0: return ro
                  e[j]['value']=ro['cm_value']

                  # Update dependent choices
                  for g in range(j+1, l):
                      e[g]['choice']=e1[g]['choice']

                  d=copy.deepcopy(d_copy)

        if found_loop and not loop_finished: break


    if loop_finished or (not has_loop and not has_random):
       cm_kernel.print_for_con('')

       ef='yes'
       if (not has_loop and not has_random): 
          li='yes'
          cm_kernel.print_for_con('One fixed point in space during exploration!')
       else:
          cm_kernel.print_for_con('Exploration finished!')

    return {'cm_return':0, 'cm_explore_state':e, 'cm_exploration_finished':ef, 'cm_last_iteration':li}