module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import os
import string
import copy
import json

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def gen_docs(i):

    """
    Generate cM documentation

    Input:  {
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    # Set default web style (either from kernel or from user)
    cm_default_web_uoa=''
    if 'cm_web_uoa' in i and i['cm_web_uoa']!='':
       cm_default_web_uoa=i['cm_web_uoa']
    else:
       cm_default_web_uoa=''
       if 'cm_default_web_uoa' in cm_kernel.ini['dcfg'] and cm_kernel.ini['dcfg']['cm_default_web_uoa']!='': 
          cm_default_web_uoa=cm_kernel.ini['dcfg']['cm_default_web_uoa']

    # Set and load
    if cm_default_web_uoa=='': cm_default_web_uoa=var_cm_cfg_default

    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                        'cm_action':'load',
                        'cm_data_uoa':cm_default_web_uoa})
    if r['cm_return']>0: return r

    web=r['cm_data_obj']['cfg']

    # Get documentation UOA
    du=cm_kernel.ini['dcfg'].get('cm_documentation_uoa','')
    if du=='':
       return {'cm_return':1, 'cm_error':'cm_documentation_uoa is not set in kernel configuration'}

    lp=[]

    # Find path to core
    ii={}
    ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-module']
    ii['cm_action']='load'
    ii['cm_data_uoa']=ini['cfg']['cm_modules']['cm-core']
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    p=r['cm_path_module']
    lp.append(p)

    # Find path to internal data
    ii={}
    ii['cm_run_module_uoa']=ini['cm_module_uid']
    ii['cm_action']='load'
    ii['cm_data_uoa']=du
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    pt=r['cm_path']

    # Find path to source libs
    for q in ini['cfg']['source_libs']:
        ii={}
        ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['code.source']
        ii['cm_action']='load'
        ii['cm_data_uoa']=q
        r=cm_kernel.access(ii)
        if r['cm_return']==0: 
           p+=' '+r['cm_path']
           lp.append(r['cm_path'])

    p='"'+p+'"'

    # Generating docs for different directories
    cm_kernel.print_for_con('Path with modules to prepare documentation:')
    cm_kernel.print_for_con('   '+p)
    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Path for output:')
    cm_kernel.print_for_con('   '+pt)
    cm_kernel.print_for_con('')

    # Run script
    ii={}
    ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['code']
    ii['cm_action']='run'

    ii['work_dir_module_uoa']=ini['cfg']['cm_modules']['os.script']
    ii['work_dir_data_uoa']=i['script_uoa_to_generate_doc']

    ii['run_script_uoa']=i['script_uoa_to_generate_doc']
    ii['run_script']='gen_docs'

    os.putenv('CM_DOC_PATH_INPUT','"'+p+'"')
    os.putenv('CM_DOC_PATH_OUTPUT',pt)

    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    # Process all files (change url)
    pr='html'
    ext='.html'
    ext1='.out'

    s=du+'&cm_web_filename=html/'

    url0=web['http_prefix']+'cm_web_module_uoa=web&cm_web_action=show&cid=internal:'+s
    url=web['http_prefix']+'cm_menu=scenarios&cm_submenu=browse_cm_api&cid=internal:'+s

    pt1=os.path.join(pt,pr)
    dirList=os.listdir(pt1)
    for fn in dirList:
        p=os.path.join(pt1,fn)
        p1=os.path.join(pt1,fn+ext1)

        url1=url+fn+'#'

        print p

        if os.path.isfile(p) and p.endswith(ext):
           cm_kernel.print_for_con('Processing file '+p+' ...')
           f=file(p,'r')
           f1=file(p1,'wb')

           while True:
               line = f.readline()
               if not line: break

               line=line.replace(' href="#', ' href="#'+url1)

               if string.find(line, ' href="http')==-1:
                  line=line.replace(' href="', ' href="'+url)
               if string.find(line, ' src="http')==-1:
                  line=line.replace(' src="', ' src="'+url0)
                
               f1.write(line)

           f1.close()
           f.close()

           os.remove(p)

           os.rename(p1,p)

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Documentation was generated successfully!')

    return {'cm_return':0}

# ============================================================================
def gen_stats(i):

    """
    Generate cM statistics

    Input:  {
              (cm_repo_uoa) - repo where to create entry
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    # Get statistics UOA
    su=cm_kernel.ini['dcfg'].get('cm_internal_statistics_uoa','')
    if su=='':
       return {'cm_return':1, 'cm_error':'cm_internal_statistics_uoa is not set in kernel configuration'}

    cru=i.get('cm_repo_uoa','')

    # Load statistics data with lock
    lock_uid=''
    d={}
    ii={}
    ii['cm_run_module_uoa']=ini['cm_module_uid']
    ii['cm_data_uoa']=su
    if cru!='': ii['cm_repo_uoa']=cru
    ii_copy=copy.deepcopy(ii)
    ii['cm_action']='load'
    ii['lock_acquire']='yes'
    r=cm_kernel.access(ii)
    if r['cm_return']==0: 
       d=r['cm_data_obj']['cfg']
       lock_uid=r['lock_uid']

    # Number of classes:
    dc=0

    r1=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['class'], 'cm_action':'list'})
    if r1['cm_return']==0: 
       dc=len(r1['cm_mixed'])

    d['number_of_classes']=str(dc)

    # Number of benchmarks:
    db=0
    db_unique_uid=[]
   
    rb=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['code.source'], 
                         'cm_action':'list',
                         'cm_classes_uoa':[i['benchmark_class_uoa']]})
    if rb['cm_return']==0: 
       # Check unique
       for q in rb['cm_mixed']:
           if q['cm_uid'] not in db_unique_uid:
              db_unique_uid.append(q['cm_uid'])
              db+=1

    d['number_of_benchmarks']=str(db)

    # Number of datasets:
    dd=0

    rd=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['dataset'], 
                         'cm_action':'list'})
    if rd['cm_return']==0: 
       dd=len(rd['cm_mixed'])

    d['number_of_datasets']=str(dd)

    # Number of combinations of benchmarks and datasets:
    dbd=0
    db_unique_uid=[]

    for q in rb['cm_mixed']:
        if q['cm_uid'] not in db_unique_uid:
           db_unique_uid.append(q['cm_uid'])

           di=q['cm_data_obj_cfg']

           found=False
           if len(di.get('run_cmds',{}))>0:
              for j in di['run_cmds']:
                  jj=di['run_cmds'][j]
                  if len(jj.get('dataset_classes_uoa',[]))>0:
                     for k in jj['dataset_classes_uoa']:
                         if k!='':
                            for l in rd['cm_mixed']:
                                dx=l['cm_data_obj_cfg']
                                if len(dx.get('cm_classes_uoa',[]))>0:
                                   dx1=dx['cm_classes_uoa']
                                   if k in dx1: 
                                      Found=True
                                      dbd+=1

           if not found: 
              # Count at least 1
              dbd+=1

    d['number_of_combinations_bench_dataset']=str(dbd)

    # Number of packages:
    dp=0

    r1=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['package'], 
                         'cm_action':'list'})
    if r1['cm_return']==0: 
       dp=len(r1['cm_mixed'])

    d['number_of_packages']=str(dp)

    diter=int(d.get('number_of_total_explored_points','0'))
    diter1=int(d.get('number_of_latest_explored_points','0'))

    if diter1!=0:
       diter+=diter1
       diter1=0

    d['number_of_total_explored_points']=str(diter)
    d['number_of_latest_explored_points']=str(diter1)

    # Update statistics
    ii=ii_copy
    ii['cm_array']=d
    ii['cm_action']='add'
    if cru!='': ii['cm_repo_uoa']=cru
    if lock_uid!='':
       ii['lock_uid']=lock_uid
       ii['cm_action']='update'

    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Statistics was generated successfully!')

    return {'cm_return':0}

# ============================================================================
def view_stats(i):

    """
    View cM statistics (through web)

    Input:  {
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    # Get web style
    if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
    else:
       return {'cm_return':1, 'cm_error':'web style is not defined'}

    # Get statistics UOA
    su=cm_kernel.ini['dcfg'].get('cm_internal_statistics_uoa','')
    if su=='':
       return {'cm_return':1, 'cm_error':'cm_internal_statistics_uoa is not set in kernel configuration'}

    # Load data
    ii={}
    ii['cm_run_module_uoa']=ini['cm_module_uid']
    ii['cm_action']='load'
    ii['cm_data_uoa']=su
    ii['cm_admin']='yes'
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r
    d=r['cm_data_obj']['cfg']

#    c1='<span style="color: red; font-size: 14pt"><i>'
    c1='<span style="color: red;"><i>'
    c2='<i></span>'

    href='<a href="'+web['http_prefix']

    cm_kernel.print_for_web('<small><table border="0" cellpadding="0" cellspacing="0">')

    cm_kernel.print_for_web('</td></tr><tr><td><B>Number of shared codelets and benchmarks:</B></td><td>')
    cm_kernel.print_for_web(href+'&cm_menu=browse&browse_module_uoa='+ini['cfg']['cm_modules']['code.source']+'" target="_blank">')
    cm_kernel.print_for_web(c1+d.get('number_of_benchmarks','')+c2)
    cm_kernel.print_for_web('</a>')

    cm_kernel.print_for_web('</td></tr><tr><td><B>Number of shared datasets:</B></td><td>')
    cm_kernel.print_for_web(href+'&cm_menu=browse&browse_module_uoa='+ini['cfg']['cm_modules']['dataset']+'" target="_blank">')
    cm_kernel.print_for_web(c1+d.get('number_of_datasets','')+c2)
    cm_kernel.print_for_web('</a>')

    cm_kernel.print_for_web('</td></tr><tr><td><B>Number of shared tools:</B></td><td>')
    cm_kernel.print_for_web(href+'&cm_menu=browse&browse_module_uoa='+ini['cfg']['cm_modules']['package']+'" target="_blank">')
    cm_kernel.print_for_web(c1+d.get('number_of_packages','')+c2)
    cm_kernel.print_for_web('</a>')
#    cm_kernel.print_for_web('</td></tr><tr><td></td><td>')
#    cm_kernel.print_for_web('&nbsp;')

    cm_kernel.print_for_web('<tr><td><B>Number of classes:</B></td><td>')
    cm_kernel.print_for_web(href+'&cm_menu=browse&browse_module_uoa='+ini['cfg']['cm_modules']['class']+'" target="_blank">')
    cm_kernel.print_for_web(c1+d.get('number_of_classes','')+c2)
    cm_kernel.print_for_web('</a>')


    cm_kernel.print_for_web('</td></tr><tr><td><B>Number of combinations of benchmarks and datasets:</B></td><td>')
    cm_kernel.print_for_web(c1+d.get('number_of_combinations_bench_dataset','')+c2)
#    cm_kernel.print_for_web('</td></tr><tr><td></td><td>')
#    cm_kernel.print_for_web('&nbsp;')

    cm_kernel.print_for_web('</td></tr><tr><td><B>Number of total explored points in space:</B></td><td>')
    cm_kernel.print_for_web(c1+d.get('number_of_total_explored_points','')+c2)

    cm_kernel.print_for_web('</td></tr><tr><td><B>Number of latest explored points in space:</B></td><td>')
    cm_kernel.print_for_web(c1+d.get('number_of_latest_explored_points','')+c2)
    cm_kernel.print_for_web('</td></tr><tr><td></td><td>')


    cm_kernel.print_for_web('</td></tr></table></small>')

    return {'cm_return':0}