cm.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

import imp
import sys
import os
import json
import uuid
import random
import re
import datetime
import string
import urllib2
import json
import shutil
import tempfile
import subprocess
import time
import signal
import hashlib
import cgi
import base64
import copy
import getpass
import zipfile
import shlex
import platform

# Core variables that has to be defined here for bootstrapping.
# All other paramaters can be moved to configuration file so
# that cM library written in other languages can easily reuse them.
# We do not use classes again to be able to reuse functionality
# in other languages.

env_cm_root='CM_ROOT'
env_cm_bin='CM_BIN'
env_cm_tmp='CM_TMP'
env_cm_cfg='CM_DEFAULT_CFG'
env_cm_cfg_path='CM_DEFAULT_CFG_PATH'
env_cm_default_repo='CM_DEFAULT_REPO'
env_cm_local_repo='CM_LOCAL_REPO'
env_cm_user_uoa='CM_USER_UOA'
env_cm_user_password='CM_USER_PASSWORD'
env_cm_user_password1='CM_USER_PASSWORD1'
env_cm_remote_repo_proxy_http='CM_REMOTE_REPO_PROXY_HTTP'
env_cm_remote_repo_proxy_https='CM_REMOTE_REPO_PROXY_HTTPS'

var_cm_info='.cm'
var_cm_data='data.json'
var_cm_cfg_local='local'
var_cm_cfg_default='default'

cm_module_default='core'
cm_module_default_access='access'
cm_action_default='default'
cm_unparsed_sep='--'

cm_json_with_text_sep='*** ### --- CM JSON SEPARATOR --- ### ***'

ini={
     # This variable describes initialization of cM framework
     # -1 - not initalized
     # 0 - initialized 
     # 1 - error during initialization
     'init':-1,
     'loaded_module_by_path':{},
     'loaded_module_by_path_last_modification':{},
     'loaded_data_by_path':{},
     'loaded_data_by_path_last_modification':{},
     'pcfg':'',
     'cfg':{},
     'dcfg':{},
     'path':'',
     'web':'no',
     'web_auth':{},
     'web_style':{},
     'remote_repo':{}
    }

# Local variables to simplify referencing
cm_kernel=None

print_on=True

# ============================================================================
def init(i):

    """
    Module init

    Input:  {}

    Output: {
              cm_return       - return code = 0, if successful
              (cm_error)      - error text if return code > 0
            }
    """

    global cm_kernel, ini

    # Save first kernel (bootsrapping)
    if cm_kernel==None:
       cm_kernel=sys.modules[__name__]

    # Load configuration for boostrapping (first check direct cfg, then check local and default entries)
    if not env_cm_cfg in os.environ.keys() and not env_cm_cfg_path in os.environ.keys():
       return {'cm_return':1, 'cm_error':'neither of environment variables '+env_cm_cfg+' and '+env_cm_cfg_path+' are defined!'}

    pcfg=''
    if env_cm_cfg in os.environ.keys() and os.environ[env_cm_cfg]!='':
       pcfg=os.environ[env_cm_cfg]
    elif env_cm_cfg_path in os.environ.keys() and os.environ[env_cm_cfg_path]!='':
       p=os.path.join(os.environ[env_cm_cfg_path], var_cm_cfg_local, var_cm_info, var_cm_data)
       if os.path.isfile(p): pcfg=p
       else:
          p=os.path.join(os.environ[env_cm_cfg_path], var_cm_cfg_default, var_cm_info, var_cm_data)
          if os.path.isfile(p): pcfg=p

    if pcfg=='':
       return {'cm_return':1, 'cm_error':'can\'t detect default kernel config file'}

    r=load_json_file({'cm_filename':pcfg})
    if r['cm_return']>0: return r

    ini['dcfg']=r['cm_array']
    ini['pcfg']=pcfg

    # Check cM root
    if env_cm_root in os.environ.keys():
       ini[env_cm_root]=os.environ[env_cm_root]
    else:
       return {'cm_return':1, 'cm_error':'environment variable '+env_cm_root+' is not defined!'}

    # Check cM root
    if env_cm_bin in os.environ.keys():
       ini[env_cm_bin]=os.environ[env_cm_bin]
    else:
       ini[env_cm_bin]=os.path.normpath(os.path.join(ini[env_cm_root], ini['dcfg']['bin_dir']))

    # Check cM root
    if env_cm_tmp in os.environ.keys():
       ini[env_cm_tmp]=os.environ[env_cm_tmp]
    else:
       return {'cm_return':1, 'cm_error':'environment variable '+env_cm_tmp+' is not defined!'}

    # Check local repository
    if env_cm_local_repo in os.environ.keys():
       ini['cm_local_repo']=os.path.normpath(os.environ[env_cm_local_repo])

    # Check cM default repo
    if env_cm_default_repo in os.environ.keys():
       ini['cm_default_repo']=os.path.normpath(os.environ[env_cm_default_repo])
    else:
       ini['cm_default_repo']=os.path.normpath(os.path.join(ini[env_cm_root], ini['dcfg']['cm_default_repo_path'], ini['dcfg']['cmr_ext']))

    return {'cm_return':0}

# ============================================================================
def is_windows():
    """
    Is Windows?

    Input: None

    Output: True if Windows
    """

    win=False

    if platform.system().lower().startswith('win'):
       win=True

    return win

# ============================================================================
def load_data(i):
    """
    Load and cache cM data

    Input:  {
              cm_module_uoa       - UOA of the module
              cm_data_uoa         - UOA of the data
              (cm_path)           - path to repository  
              (cm_path_array)     - array of paths to search
              (lock_acquire)      - if 'yes', acquire lock
              (lock_retry_num)    - number of retries to acquire lock (default=6)
              (lock_retry_delay)  - delay in sec before retrying to acquire lock (default=11)
              (lock_expire)       - for how long to acquire lock (secs) 
              (unlock_uid)        - unlock data (FGG note: I changed lock_uid to unlock_uid 
                                                 not to interfere with update locking)
              (cm_admin)          - if 'yes', override access control - only internal use,
                                    can not be used during web access (explicitly removed)
            }

    Output: {
              cm_return           - return code = 0, if successful
              (cm_error)          - error text if return code > 0
              cm_data_obj         - cM data object
              cm_path             - path to data entry
              cm_path_module      - path to module entry with this entry
              cm_path_repo        - path to the repository of this entry
              cm_uid              - uid (from UOA)
              cm_alias            - alias (from UOA)
              cm_uoa              - cm_data_uoa (alias or uid if alias=='')
              cm_module_uoa       - module UOA 
              cm_module_uid       - module UID
              cm_display_as_alias - taken from the data
              cm_display_html     - prepare display: "display as alias" or <I>alias</I> or UID
              (lock_uid)          - UID of the aquired lock
            }
    """

    if 'cm_module_uoa' not in i or i['cm_module_uoa']=='':
       return {'cm_return':1, 'cm_error':'parameter "cm_module_uoa" is not defined'}

    if 'cm_data_uoa' not in i or i['cm_data_uoa']=='':
       return {'cm_return':1, 'cm_error':'parameter "cm_data_uoa" is not defined'}

    # Find path to data
    if 'cm_path_array' in i: path_repo=i['cm_path_array']
    else: path_repo=[]

    if 'cm_path' in i and i['cm_path']!='': path_repo.append(i['cm_path'])

    if len(path_repo)==0:
       r=find_path_to_work_repositories({})
       if r['cm_return']>0: return r
       for x in r['cm_array']:
           if x not in path_repo: path_repo.append(x)

    module_uoa=i['cm_module_uoa']
    module_uid=''
    uoa=i['cm_data_uoa']

    # Continue
    found=False
    for p in path_repo:
        y=find_path_to_entry({'cm_path':p, 'cm_data_uoa':i['cm_module_uoa']})
        if y['cm_return']>0: return y
        elif y['cm_return']==0:
           module_uoa=y['cm_uoa']
           module_uid=y['cm_uid']
           pm=y['cm_path']
           y1=find_path_to_entry({'cm_path':pm, 'cm_data_uoa':i['cm_data_uoa']})
           if y1['cm_return']>0: return y1
           elif y1['cm_return']==0:
              found=True
              uid=y1['cm_uid']
              alias=y1['cm_alias']
              uoa=y1['cm_uoa']
              if alias!='': uoa=alias
              break

    if not found:
       return {'cm_return':16, 'cm_error':'can\'t find path to data "'+module_uoa+':'+uoa+'"'}

    pd=y1['cm_path']

    p0=os.path.join(pd, ini['dcfg']['dcm'])
    p1=os.path.join(p0, ini['dcfg']['fconfig'])
    p5=os.path.join(p0, ini['dcfg']['flock'])

    # Check if acquire coarse-grain lock or release it
    locks={}
    if i.get('lock_acquire','')=='yes' or i.get('unlock_uid','')!='':

       luid=''
       if os.path.isfile(p5):

          try:
             f=file(p5)
             luid=f.readline().strip()
             exp=f.readline().strip()
             f.close()
          except Exception as e:
             return {'cm_return':1, 'cm_error':'problem reading locking file'}

          # Check if lock expired
          if i.get('lock_acquire','')=='yes':
             # Retry if locked
             retry=int(ini['dcfg'].get('flock_retry_num','6'))
             if i.get('lock_retry_num','')!='': retry=int(i['lock_retry_num'])
             retryd=float(ini['dcfg'].get('flock_retry_delay','11'))
             if i.get('lock_retry_delay','')!='': retry=float(i['lock_retry_delay'])

             if exp=='': exp=ini['dcfg'].get('flock_expire_max','60')

             dt=os.path.getmtime(p5)+float(exp)-time.time()
             if dt>0: 
                while retry>0 and os.path.isfile(p5) and dt>0:
                   retry-=1
                   time.sleep(retryd)
                   if os.path.isfile(p5): dt=os.path.getmtime(p5)+float(exp)-time.time()

                if retry==0 or os.path.isfile(p5):
                   return {'cm_return':32, 'cm_error':'data is locked'}

             luid=''
             if os.path.isfile(p5): os.remove(p5)

       if luid!='' and i.get('unlock_uid','')!='':
          if luid!=i['unlock_uid']:
             return {'cm_return':32, 'cm_error': 'data is locked with another UID'}
          luid=''
          os.remove(p5)

       if i.get('lock_acquire','')=='yes':
          # (Re)acquire lock
          r=gen_uid({})
          if r['cm_return']>0: return r
          luid=r['cm_uid']
          exp1=i.get('lock_expire','')
          if exp1=='' or float(exp1)<1: exp1=ini['dcfg'].get('flock_expire_max','60')

          f=open(p5,'w')
          f.write(luid+'\n')
          f.write(exp1+'\n')
          f.close()

          locks['lock_uid']=luid

    # Check if data has been already loaded
    cached=None
    cm_display_as_alias=''
    cm_display_html=''
    if pd in ini['loaded_data_by_path'] and ini['loaded_data_by_path_last_modification'][pd]==os.path.getmtime(p1):
       # Code already loaded 
       cached=ini['loaded_data_by_path'][pd]
       # FGG HACK - sometimes on intensive updates (test)
       # path changes. Here I explicitly change it
       cached['cm_path']=pd
       xcfg=cached['cm_data_obj']['cfg']
    else:
       # Load config file
       r1=load_json_file({'cm_filename':p1})
       if r1['cm_return']>0: return r1
       xcfg=r1['cm_array']
       if 'cm_display_as_alias' in xcfg and xcfg['cm_display_as_alias']!='': 
           cm_display_as_alias=xcfg['cm_display_as_alias']
           cm_display_html=cm_display_as_alias[0].upper()+cm_display_as_alias[1:]
       elif alias!='':
           cm_display_html=alias
       else:
           cm_display_html=uid

    # Check fine-grain access
    ac=ini['dcfg'].get('access_control',{})
    if i.get('cm_admin','')!='yes' and ini['web']=='yes' and ac.get('use','')=='yes':
       rx=access_control({'dcfg':xcfg, 'module_uoa':module_uoa, \
                          'module_uid':module_uid, 'data_uoa':uoa, 'key':'read'})
       if rx['cm_return']>0: return rx

    # If cached, quit
    if cached!=None: 
       cached.update(locks)
       return cached

    # Data object
    d={}
    d['cm_path']=pd
    d['cfg']=xcfg


    # Cache module
    o={'cm_return':0, 'cm_data_obj':d, \
                      'cm_path':pd, \
                      'cm_path_module':pm, \
                      'cm_path_repo':p, \
                      'cm_uid':uid, \
                      'cm_alias':alias, \
                      'cm_uoa':uoa, \
                      'cm_module_uoa':module_uoa, \
                      'cm_module_uid':module_uid, \
                      'cm_display_as_alias':cm_display_as_alias, \
                      'cm_display_html':cm_display_html}
    ini['loaded_data_by_path'][pd]=o
    # Cache time of change of json description
    ini['loaded_data_by_path_last_modification'][pd]=os.path.getmtime(p1)

    # Update info about locks
    o.update(locks)

    return o

# ============================================================================
def load_module(i):
    """
    Load and cache cM module

    Input:  {
              cm_module_uoa - UOA of the module
              (cm_path)
            }

    Output: {
              cm_return       - return code = 0, if successful
              cm_code         - python code object
              external_module - if 'yes', module is external
              cm_path         - path to data entry
              cm_uid          - uid (from UOA)
              cm_alias        - alias (from UOA)

            }
    """

    if 'cm_module_uoa' not in i:
       return {'cm_return':1, 'cm_error':'parameter "cm_module_uoa" is not defined'}

    external=False
    if i['cm_module_uoa'].startswith('cme.'): external=True

    if external:
       # Module in current directory (or in search path)
       p2=''
       p3=i['cm_module_uoa']
       if p3.endswith('.py'): p3=p3[:-3]

    else:
       # Find path to module
       ii={'cm_module_uoa':ini['dcfg']['cmr_module'],
           'cm_data_uoa':i['cm_module_uoa']}
       if 'cm_path' in i: ii.update({'cm_path':i['cm_path']})

       r=load_data(ii)
       if r['cm_return']>0: return r
       elif r['cm_return']<0: 
          return {'cm_return':16, 'cm_error':'can\'t find path to module "'+i['cm_module_uoa']+'"'}
       path=r['cm_path']
       uid=r['cm_uid']
       alias=r['cm_alias']

       p1=os.path.join(path, ini['dcfg']['dcm'], ini['dcfg']['fconfig'])

       # Check if code has been already loaded
       if path in ini['loaded_module_by_path'] and ini['loaded_module_by_path_last_modification'][path]==os.path.getmtime(p1):
          # Code already loaded 
          return ini['loaded_module_by_path'][path]

       # Load config file
       r=load_json_file({'cm_filename':p1})
       if r['cm_return']>0: return r
       xcfg=r['cm_array']

       # Load code
       subdir=xcfg['cm_module_py_dir']
       p2=os.path.join(path, subdir)
       p3=xcfg['cm_module_py_ext']

    r=load_code({'cm_path':p2, 'cm_code_name':p3})
    if r['cm_return']>0: return r
    c=r['code']

    # FGG: I move various initializations here to simplify kernel and modules
    c.cm_kernel=cm_kernel

    if not external:
       c.ini['cm_code']=c
       c.ini['cm_module_uoa']=i['cm_module_uoa']
       c.ini['cm_module_uid']=uid
       c.ini['cm_module_alias']=alias
       c.ini['cfg']=xcfg
       c.ini['path']=path

       r=c.init(i)
       if r['cm_return']>0: return r

       # Cache module
       o={'cm_return':0, 'cm_code':c, 'cm_path':path, 'cm_uid':uid, 'cm_alias':alias}
       ini['loaded_module_by_path'][path]=o
       # Cache time of change of json description
       ini['loaded_module_by_path_last_modification'][path]=os.path.getmtime(p1)
    else:
       o={'cm_return':0, 'cm_code':c, 'external_module':'yes'}   

    return o

# ============================================================================
def load_code(i):
    """
    Load python code

    Input:  {
              cm_path      - path
              cm_code_name - name of the python native module
            }

    Output: {
              cm_return  - return code = 0, if successful
              code       - python code object
            }
    """

    # Load code
    try:
       x=imp.find_module(i['cm_code_name'], [i['cm_path']])
    except ImportError as e:
       return {'cm_return':1, 'cm_error':'can\'t find code (path='+i['cm_path']+', name='+i['cm_code_name']+', err='+format(e)+')'}

    # Generate uid for the run-time extension of the loaded module 
    # otherwise modules with the same extension (key.py for example) 
    # will be reloaded ...

    r=gen_uid({})
    if r['cm_return']>0: return r
    ruid='runtime-'+r['cm_uid']

    try:
       c=imp.load_module(ruid, x[0], x[1], x[2])
    except ImportError as e:
       return {'cm_return':1, 'cm_error':'can\'t load code (path='+i['cm_path']+', name='+i['cm_code_name']+', err='+format(e)+')'}

    x[0].close()

    return {'cm_return':0, 'code':c}

# ============================================================================
def gen_uid(i):
    """
    Generate cM UID

    Input:  {}

    Output: {
              cm_return - return code = 0, if successful
              cm_uid    - cM UID in string format (16 characters 0..9,a..f)
            }
    """

    uid=str(uuid.uuid4().hex)

    if len(uid)!=32:
       return {'cm_return':1, 'cm_error':'problem generating UID : len='+str(len(uid))+' !=32'}

    random.seed

    x=random.randrange(0,16)
    return {'cm_return':0, 'cm_uid':uid[x:x+16]}

# ============================================================================
def is_uid(str):
    """
    Check if string is cM UID

    Input: string to check

    Output: True if UID, otherwise False
    """

    if len(str)!=16:
       return False

    pattern = r'[^\.a-f0-9]'
    if re.search(pattern, str.lower()):
        return False

    return True

# ============================================================================
def cid2array(i):
    """
    Convert cid to array

    Input:  {
              cid      - in format (REPO_UOA:)KEY_UOA:DATA_UOA
              (prefix) - add prefic to output variables (<prefix>_repo_uoa:...)
            }

    Output: {
              cm_return - return code = 0 if successful
              cm_array  - array
            }
    """

    prefix='cm'
    if 'prefix' in i and i['prefix']!='': prefix=i['prefix']

    array={}
    if 'cid' in i and i['cid']!='':
        id=i['cid'].split(":")
        if (len(id)<2): 
           return {'cm_return':1, 'cm_error':'unknown cid format'}
        elif (len(id)<3):
            array[prefix+'_module_uoa']=id[0]
            array[prefix+'_data_uoa']=id[1]
        elif (len(id)<4):
            array[prefix+'_repo_uoa']=id[0]
            array[prefix+'_module_uoa']=id[1]
            array[prefix+'_data_uoa']=id[2]
    return {'cm_return':0, 'cm_array':array}


# ============================================================================
def web_print_array_for_debug(i):
    """
    Print array for debug purposes (use <pre> and <div align="left")

    Input:  {
              some array
            }

    Output: {
              cm_return - 0
            }
    """

    print '<pre><div align="left">'+json.dumps(i, indent=2, sort_keys=True)+'</div></pre>'


    return {'cm_return':0}

# ============================================================================
def find_path_to_entry(i):
    """
    Find path to an UOA entry (check UID or alias).

    Input:  {
              cm_path     - path to a repository
              cm_data_uoa - data UOA
            }

    Output: {
              cm_return      - return code =  0 if found path to entry
                                           = -1 if not found path to entry
                                           >  0 if error

              cm_path_orig   - i['cm_path']
              cm_path        - path to data entry
              cm_uid         - uid (from UOA)
              cm_alias       - alias (from UOA)
              cm_uoa         - alias or uid, if alias==''
            }
    """

    if ('cm_path' not in i):
       return {'cm_return':1, 'cm_error':'parameter "'+'cm_path'+'" is not defined'}
    if ('cm_data_uoa' not in i):
       return {'cm_return':1, 'cm_error':'parameter "cm_data_uoa" is not defined'}

    # Disambiguate UOA
    alias=''
    if is_uid(i['cm_data_uoa']):
       # If UID
       uid=i['cm_data_uoa']

       # Check if alias exists
       p1=os.path.join(i['cm_path'], ini['dcfg']['dcm'], ini['dcfg']['falias-u'] + uid)
       found_alias=False
       if os.path.isfile(p1):
          try:
             f=file(p1)
             alias=f.readline().strip()
             f.close()
             found_alias=True
          except Exception as e:
             None

       # If alias exist, check directory with alias
       if found_alias:
          p2=os.path.join(i['cm_path'], alias)
          if not os.path.isdir(p2):
             #auto-repairment
             try:
                os.mkdir(p2)
             except Exception as e:
                return {'cm_return':1, 'cm_error':format(e)}

          return {'cm_return':0, 'cm_path':p2, 'cm_uid':uid, 'cm_alias':alias, 'cm_uoa':alias}

       p2=os.path.join(i['cm_path'], uid)
       if os.path.isdir(p2):
          return {'cm_return':0, 'cm_path':p2, 'cm_uid':uid, 'cm_alias':'', 'cm_uoa':uid}

       return {'cm_return':-1}

    # If alias
    alias=i['cm_data_uoa']

    p1=os.path.join(i['cm_path'], alias)
    if os.path.isdir(p1):
       # Check uid for this alias
       p2=os.path.join(i['cm_path'], ini['dcfg']['dcm'], ini['dcfg']['falias-a'] + alias)
       try:
          f=file(p2)
          uid=f.readline().strip()
          f.close()
       except Exception as e:
          return {'cm_return':10, 'cm_error':'inconsistent entry: alias "'+alias+'" exists, but not the uid in file '+p2, 'cm_path_orig':i['cm_path'], 'cm_path':p1, 'cm_alias':alias}

       return {'cm_return':0, 'cm_path_orig': i['cm_path'], 'cm_path':p1, 'cm_uid':uid, 'cm_alias':alias, 'cm_uoa':alias}

    return {'cm_return':-1}

# ============================================================================
def create_entry(i):
    """
    Create an UOA entry directory structure.

    Input:  {
              cm_path       - path to a repository
              (cm_data_uoa) - data UOA
              (cm_data_uid) - if cm_data_uoa==alias, we can force a given UID through this variable
              (lock_uid)    - UID of the aquired lock
            }

    Output: {
              cm_return - return code =  0 if created entry
                                      = -1 if already exist
                                      >  0 if error

              cm_path   - path to data entry
              cm_uid    - uid (from UOA)
              cm_alias  - alias (from UOA)
              cm_uoa    - alias or uid if alias==''
            }
    """

    if ('cm_path' not in i):
       return {'cm_return':1, 'cm_error':'parameter "'+'cm_path'+'" is not defined'}

    # If no cm_data_uoa, generate one
    alias=''
    uid=''
    if ('cm_data_uoa' not in i or i['cm_data_uoa']==''):
       if 'cm_data_uid' not in i or i['cm_data_uid']=='':
          r=gen_uid({})
          if r['cm_return']>0: return r
          uid=r['cm_uid']
       else:
          uid=i['cm_data_uid']

          # Check if already exist
          ii={}
          ii.update(i)
          ii['cm_data_uoa']=uid
          r=find_path_to_entry(ii)
          if r['cm_return']>0: return r
          elif r['cm_return']==0:
             r1={}; r1.update(r); r1['cm_return']=-1
             return r1

       alias=''
    else:
       # Check if already exist
       r=find_path_to_entry(i)
       if r['cm_return']>0: return r
       elif r['cm_return']==0:
          r1={}; r1.update(r); r1['cm_return']=-1
          return r1

       if is_uid(i['cm_data_uoa']):
          uid=i['cm_data_uoa']; alias=''
       else:
          alias=i['cm_data_uoa']
          if 'cm_data_uid' in i: uid=i['cm_data_uid']
          else: 
             r=gen_uid({})
             if r['cm_return']>0: return r
             uid=r['cm_uid']

    if alias!='':
       p=os.path.join(i['cm_path'], alias)
    else:
       p=os.path.join(i['cm_path'], uid)

    # Check alias disamibugation
    if alias!='':
       p1=os.path.join(i['cm_path'], ini['dcfg']['dcm'])
       if not os.path.isdir(p1):
          # Create .cm directory
          try:
             os.mkdir(p1)
          except Exception as e:
             return {'cm_return':1, 'cm_error':format(e)}

       # Check if alias->uid exist
       p3=os.path.join(p1, ini['dcfg']['falias-a'] + alias)
       if os.path.isfile(p3):
          try:
             fx=file(p3)
             uid1=fx.readline().strip()
             fx.close()
          except Exception as e:
             None

          if uid1!=uid:
             return {'cm_return':1, 'cm_error':'different alias->uid disambiguator already exists ('+p3+')'}

       try:
          f=file(p3, 'w')
          f.write(uid+'\n')
          f.close()
          found_alias=True
       except Exception as e:
          None

       # Check if uid->alias exist
       p2=os.path.join(p1, ini['dcfg']['falias-u'] + uid)
       if os.path.isfile(p2):
          try:
             fx=file(p2)
             alias1=fx.readline().strip()
             fx.close()
          except Exception as e:
             None

          if alias1!=alias:
             return {'cm_return':1, 'cm_error':'different uid->alias disambiguator already exists ('+p2+')'}

       try:
          f=file(p2, 'w')
          f.write(alias+'\n')
          f.close()
          found_alias=True
       except Exception as e:
          None

    # Create directory
    if not os.path.exists(p):
       try:
          os.mkdir(p)
       except Exception as e:
          return {'cm_return':1, 'cm_error':format(e)}

    uoa=uid
    if alias!='': uoa=alias

    return {'cm_return':0, 'cm_path':p, 'cm_uid':uid, 'cm_alias':alias, 'cm_uoa':uoa}

# ============================================================================
def check_global_writing(i):
    """
    Check global writing to repository

    Input:  {
            }

    Output: {
              cm_return  - return code >0 if writing is not allowed
            }
    """

    if cm_kernel.ini['dcfg'].get('forbid_write','')=='yes':
       return {'cm_return':1, 'cm_error':'any writing is forbidden by configuration'}

    return {'cm_return':0}

# ============================================================================
def delete_entry(i):
    """
    Delete entry

    Input:  {
              cm_path        - path to the entry
              cm_path_module - part of the path with module
              cm_uid         - uid (from UOA), needed to delete alias disambiguator if exists
              cm_alias       - alias (from UOA), needed to delete alias disambiguator if exists
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    r=check_global_writing({})
    if r['cm_return']>0: return r

    # Remove alias if exists
    if i['cm_alias']!='':
       # Delete alias
       r=cm_kernel.delete_alias({'cm_path':i['cm_path_module'], 'cm_alias':i['cm_alias'], 'cm_uid':i['cm_uid']})
       if r['cm_return']>0: return r

    # Delete directory
    return cm_kernel.delete_directory({'cm_path':i['cm_path']})

# ============================================================================
def delete_alias(i):
    """
    Delete alias

    Input:  {
              cm_path    - path to the entry
              cm_uid     - uid (from UOA)
              cm_alias   - alias (from UOA)
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    p=i['cm_path']
    uid=''

    if 'cm_alias' in i:
       alias=i['cm_alias']
       if alias!='' and os.path.isdir(p):
          p1=os.path.join(p, ini['dcfg']['dcm'], ini['dcfg']['falias-a'] + alias)
          if os.path.isfile(p1):
             try:
                f=file(p1)
                uid=f.readline().strip()
                f.close()
             except Exception as e:
                None
             os.remove(p1)

          if uid=='': uid=i['cm_uid']

          if uid!='':
             p1=os.path.join(p, ini['dcfg']['dcm'], ini['dcfg']['falias-u'] + uid)
             if os.path.isfile(p1):
                os.remove(p1)

    return {'cm_return':0}

# ============================================================================
def substitute_str_in_file(i):
    """
    Substitute string in file

    Input:  {
              cm_file    - file
              cm_string1 - string to be replaced
              cm_string2 - replace string
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    import codecs

    fil=i['cm_file']
    s1=i['cm_string1']
    s2=i['cm_string2']

    f=open(fil, 'r')
    x=f.read().decode('utf-8')
    f.close()

    x=x.replace(s1,s2)
   
    f=codecs.open(fil,'w','utf-8')
    f.write(x)
    f.close()

    return {'cm_return':0}

# ============================================================================
def create_data_entry(i):
    """
    Create data entry

    Input:  {
              cm_path                  - path to the repository
              cm_module_uoa            - module UOA 
              cm_data_uoa              - data UOA 
              (cm_data_uid)            - if cm_data_uoa==alias, we can force a given UID through this variable
              (cm_array)               - array to add to the entry
                                         If not set, the whole array will be added
                                         including all working variables - we do not suggest to do that
              (cm_add_default)         - if 'yes', add default params from descriptions (module/kernel)
              (cm_update)              - if 'yes', update entry
              (cm_description)         - description of the data entry
              (cm_display_as_alias)    - user readable alias to display instead of module:data
              (cm_skip_update_info)    - if == 'yes', do not write 'cm_updated'
              (cm_note)                - note about this data entry
              (cm_like)                - if 'yes' add +1 for this user to the entry

              (cm_file_upload_tmp_uid) - normally generated automatically by web server
                                         when uploading files
              (cm_file_upload_name)    - preffered uploaded file name 
              (cm_file_upload_type)    - type of file (if .zip, it will be automatically unziped)
              (cm_archive)             - if file should be kept as archive

              ...                      - if 'cm_array' not present, add everything else from this array
                                         to the entry 
              (lock_uid)               - unlock data
            }

    Output: {
              cm_return  - return code >0 if error
                                       -1 if entry exists
              cm_path    - path to created data entry
              cm_uid     - uid (from UOA)
              cm_alias   - alias (from UOA)
            }
    """

    r=check_global_writing({})
    if r['cm_return']>0: return r

    # Check if file upload
    upload_file_uid=i.get('cm_file_upload_tmp_uid','')
    upload_file_name=i.get('cm_file_upload_name','')
    upload_file_type=i.get('cm_file_upload_type','')
    upload_archive=i.get('cm_archive','')
    if upload_file_uid!='': 
       del(i['cm_file_upload_tmp_uid'])
       if 'upload_file_type' in i: del(i['cm_file_upload_type'])
       if upload_file_name!='' and i.get('cm_archive','')!='already_archived_with_user_name': del(i['cm_file_upload_name'])

    if upload_file_name=='' and 'cm_file_upload_name' in i: del(i['cm_file_upload_name'])
    if upload_file_type=='' and 'cm_file_upload_type' in i: del(i['cm_file_upload_type'])

    # Check user
    cm_user_uid=''
    cm_user_uoa=''
    if 'cm_user_uid' in ini['web_auth']: 
       cm_user_uid=ini['web_auth']['cm_user_uid']
       cm_user_uoa=ini['web_auth']['cm_user_uoa']

    p=i['cm_path']

    # Create first level entry (module)
    # Find path to module
    r=load_data({'cm_module_uoa':ini['dcfg']['cmr_module'],
                 'cm_data_uoa':i['cm_module_uoa']})
    if r['cm_return']>0: return r
    elif r['cm_return']<0: 
       return {'cm_return':16, 'cm_error':'can\'t find path to module "'+i['cm_module_uoa']+'"'}
    uid=r['cm_uid']
    alias=r['cm_alias']
    module_desc=r['cm_data_obj']['cfg']

    if alias=='': alias=uid

    r=create_entry({'cm_path':p, 'cm_data_uoa':alias, 'cm_data_uid':uid})
    if r['cm_return']>0: return r
    p1=r['cm_path']

    # Create second level entry (data)
    i1={}; i1['cm_path']=p1
    if 'cm_data_uoa' in i: i1['cm_data_uoa']=i['cm_data_uoa']
    if 'cm_data_uid' in i: i1['cm_data_uid']=i['cm_data_uid']
    r=create_entry(i1)
    if r['cm_return']>0: return r

    # Configuration
    p2=r['cm_path']
    p3=os.path.join(p2, ini['dcfg']['dcm'])
    p4=os.path.join(p3, ini['dcfg']['fconfig'])
    p5=os.path.join(p3, ini['dcfg']['flock'])

    a={}
    if r['cm_return']<0:
       if i.get('cm_update','')=='yes':
          # Entry exists, check locks
          if os.path.isfile(p5):
             try:
                f=file(p5)
                luid=f.readline().strip()
                exp=f.readline().strip()
                f.close()
             except Exception as e:
                return {'cm_return':1, 'cm_error':'problem reading locking file'}

             # Check if lock expired
             if exp=='': exp=ini['dcfg'].get('flock_expire_max','60')

             dt=os.path.getmtime(p5)+float(exp)-time.time()
             if dt<0:
                # Expired
                if i.get('lock_uid','')=='':
                   os.remove(p5)
                else:
                   return {'cm_return':32, 'cm_error':'data lock UID is not matching'}

             else:   
                if i.get('lock_uid','')=='':
                   return {'cm_return':32, 'cm_error':'data is locked'}
                elif i.get('lock_uid','')!=luid:
                   return {'cm_return':32, 'cm_error':'data is locked with different UID'}
             
          elif i.get('lock_uid','')!='':
             return {'cm_return':32, 'cm_error':'lock removed or expired'}

          # Entry exists, load configuration if update
          r['cm_return']=0
          r2=load_json_file({'cm_filename':p4})
          if r2['cm_return']>0: return r2
          a=copy.deepcopy(r2['cm_array'])

    else:
       # Create configuration directory
       if not os.path.isdir(p3):
          try:
             os.mkdir(p3)
          except Exception as e:
             return {'cm_return':1, 'cm_error':format(e)}

    # Check if cm_array in input - then use only this array as input
    cma={}

    # Check if need to add default data from description
    if i.get('cm_add_default','')=='yes':
       df={}

       if 'cm_common_data_description7' in ini['dcfg']:
          rz=create_default_array_from_description({'cm_data_desc':ini['dcfg']['cm_common_data_description7']})
          if rz['cm_return']>0: return rz
          merge_arrays({'cm_array':df, 'cm_array1':rz['cm_array']})

       if 'cm_common_data_description8' in ini['dcfg']:
          rz=create_default_array_from_description({'cm_data_desc':ini['dcfg']['cm_common_data_description8']})
          if rz['cm_return']>0: return rz
          merge_arrays({'cm_array':df, 'cm_array1':rz['cm_array']})

       if 'cm_data_description' in module_desc:
          rz=create_default_array_from_description({'cm_data_desc':module_desc['cm_data_description']})
          if rz['cm_return']>0: return rz
          merge_arrays({'cm_array':df, 'cm_array1':rz['cm_array']})

       if len(df)>0:
          merge_arrays({'cm_array':cma, 'cm_array1':df})

    if 'cm_array' in i: 
       merge_arrays({'cm_array':cma, 'cm_array1':i['cm_array']})

    # Update array
    merge_arrays({'cm_array':a, 'cm_array1':cma})

    if 'cm_description' in i: a['cm_description']=i['cm_description']
    if 'cm_display_as_alias' in i: a['cm_display_as_alias']=i['cm_display_as_alias']

    if 'powered_by' not in a:
       a['powered_by']={}
       a['powered_by']['name']='Collective Mind Engine'
       rq=get_cm_version({})
       if rq['cm_return']>0: return rq
       a['powered_by']['version']=rq['cm_string']

    # Add record time to configuration
    if i.get('cm_skip_update_info','')!='yes':
       u={}
       if cm_user_uid!='': 
          u['cm_user_uoa']=cm_user_uid
          u['cm_person_uoa']=cm_user_uid
       if 'cm_note' in i: u['cm_note']=i['cm_note']
       if 'cm_validated' in i: u['cm_validated']=i['cm_validated']
       if 'cm_like' in i and i['cm_like']=='yes': 
          # Check that only 1 rank
          rank=1
          if 'cm_updated' in a:
             found=False
             for x in a['cm_updated']:
                 if x.get('cm_like','')=='yes':
                    if 'cm_user_uoa' in x and (cm_user_uid!='' or cm_user_uoa!='') and \
                       (x['cm_user_uoa']==cm_user_uid or x['cm_user_uoa']==cm_user_uoa):
                       found=True
                    else:
                       rank+=1
             if not found:
                u['cm_like']=i['cm_like']

          a['cm_rank']=rank

       r5=get_current_date_time({})
       u['cm_iso_datetime']=r5['cm_iso_datetime']
       u['cm_module_uid']=uid

       if 'cm_updated' not in a: 
          a['cm_updated']=[]
       a['cm_updated'].append(u)

    # Check if no cm_access_control - use default
    if 'cm_access_control' in a:
       pass
    elif 'cm_access_control' in i:
       a.update({'cm_access_control':i['cm_access_control']})
    elif 'cm_access_control_default' in cm_kernel.ini['dcfg']:
       a.update({'cm_access_control':cm_kernel.ini['dcfg']['cm_access_control_default']})

    r4=save_array_to_file_as_json({'cm_filename':p4, 'cm_array':a})
    if r4['cm_return']>0: return r4

    # Finish upload
    pp1=''
    if upload_file_uid!='':
       pp1=os.path.join(ini[cm_kernel.env_cm_tmp], ini['dcfg']['cm_tmp_prefix']+upload_file_uid+ini['dcfg']['cm_tmp_postfix'])
    else:
       # If upload made through direct call from other modules
       if ('cm_file_upload' in i and len(i['cm_file_upload'])>0) or \
          ('cm_file_upload_base64' in i and len(i['cm_file_upload_base64'])>0):

          if ('cm_file_upload' in i and len(i['cm_file_upload'])>0):
             xn=i['cm_file_upload']
          elif ('cm_file_upload_base64' in i and len(i['cm_file_upload_base64'])>0):
             xn=base64.urlsafe_b64decode(str(i['cm_file_upload_base64'])) # convert from unicode to str since base64 works on strings

          px=os.path.join(p2,i.get('cm_file_upload_name','default_name'))
          fx=file(px, 'wb')
          fx.write(xn)
          fx.close()

          pp1=px
    
    if pp1!='':
       if upload_file_type=='.zip' and upload_archive!='already_archived_with_user_name': 
          upload_file_name=''

       if upload_file_type=='' or upload_archive!='':
          # regular file
          if os.path.isfile(pp1):
             x=upload_file_name
             if x=='': x=ini['dcfg']['cm_default_upload_filename']
             if upload_archive=='already_archived_with_internal_name': x=ini['dcfg']['cm_archive_filename']
             pp2=os.path.join(p2, x)
             shutil.move(pp1,pp2)
       elif upload_file_type=='.zip':
          # Archive
          f=open(pp1,'rb')
          z=zipfile.ZipFile(f)
          for d in z.namelist():
              if not d.startswith('.') and not d.startswith('/') and not d.startswith('\\'):
                 if d.endswith('/'): 
                    # create directory 
                    pp2=os.path.join(p2,d)
                    if not os.path.exists(pp2): os.makedirs(pp2)
                 else:
                    # extract file
                    pp2=os.path.join(p2, d)
                    fo=open(pp2, 'wb')
                    fo.write(z.read(d))
                    fo.close()
          f.close()
          os.remove(pp1)   

    # Remove lock after update
    if i.get('lock_uid','')!='' and os.path.isfile(p5):
       os.remove(p5)

    return r

# ============================================================================
def create_default_array_from_description(i):
    """
    Create array with default parameters from data description

    Input:  {
              cm_data_desc    - cM data description
            }

    Output: {
              cm_return       - return code =0 if successful
                                            >0 if error
              cm_array        - array with default parameters
            }
    """

    a={}

    for x in i['cm_data_desc']:
        y=i['cm_data_desc'][x]

        if '$' not in x:
           d=''

           t=y.get('type','')
           if t=='list': d=[]
           elif t=='dict': d={}

           if 'default_value' in d: d=y['default_value']

           set_value_by_flattened_key({'cm_array':a, 'cm_key':x, 'cm_value':d})

    return {'cm_return':0, 'cm_array':a}

# ============================================================================
def convert_file_to_upload_string(i):
    """
    Convert file to upload string

    Input:  {
              cm_full_filename - file name to convert
            }

    Output: {
              cm_return            - return code =0 if successful
                                                 >0 if error
              cm_file_upload_base64 - string that can be used to upload file
            }
    """

    fn=i['cm_full_filename']

    if not os.path.isfile(fn):
       return {'cm_return':1, 'cm_error':'file '+fn+' not found'}

    s=''
    try:
       f=open(fn, 'rb')
       while True:
          x = f.read(32768);
          if not x: break
          s+=x
       f.close()
    except Exception as e:
       return {'cm_return':1, 'cm_error':'error reading file ('+format(e)+')'}

    r={'cm_return':0}
    r['cm_file_upload_base64']=base64.urlsafe_b64encode(s)

    return r

# ============================================================================
def get_current_date_time(i):
    """
    Get current date and time

    Input:  {}

    Output: {
              cm_return       - return code =0 if successful
                                            >0 if error
              cm_array        - array with date and time
              cm_iso_datetime - date and time in ISO format
            }
    """

    a={}

    now1=datetime.datetime.now()
    now=now1.timetuple()

    a["cm_date_year"]=str(now[0])
    a["cm_date_month"]=str(now[1])
    a["cm_date_day"]=str(now[2])
    a["cm_time_hour"]=str(now[3])
    a["cm_time_minute"]=str(now[4])
    a["cm_time_second"]=str(now[5])

#    return {'cm_return':0, 'cm_array':a, 'cm_iso_datetime':now1.isoformat(' ')}
    return {'cm_return':0, 'cm_array':a, 'cm_iso_datetime':now1.isoformat()}

# ============================================================================
def convert_iso_time_to_str(i):
    """
    Get current date and time

    Input:  {
              cm_iso_datetime    - date and time in ISO format
              (cm_custom_format) - customize view
            }

    Output: {
              cm_return       - return code =0 if successful
                                            >0 if error
              cm_string       - user friendly date/time
            }
    """

    if 'cm_iso_datetime' not in i: return {'cm_return':1, 'cm_error':'"cm_iso_datetime" is not set in kernel/convert_iso_time_to_str'}

    d=datetime.datetime(*map(int, re.split('[^\d]', i['cm_iso_datetime'])[:-1]))

    format=i.get('cm_custom_format','')
    if format=='':
       format='%Y-%B-%d (%A) %H:%M:%S'
    s=d.strftime(format)

    return {'cm_return':0, 'cm_string':s}

# ============================================================================
def find_index_in_array(i):
    """
    Find index in array of formart [{cm_index:'a', ...}, {cm_index='b', ...}, ...]
    We use such array instead of simple associative array 
    when we need to keep an order such as for actions ... 

    Input:  {
              cm_array - array of format
              cm_index - index
            }

    Output: {
              cm_return - return code =  0 if successful
                                        -1 if not found
              cm_array  - sub-array with given index
            }
    """

    for x in i['cm_array']:
        if 'cm_index' in x and x['cm_index']==i['cm_index']:
           return {'cm_return':0, 'cm_array':x}

    return {'cm_return':-1}


# ============================================================================
def find_path_to_current_repository(i):
    """
    Finds local repository (recursively checks 
    all upper directories until it finds the repository)

    Input:  {
              cm_get_uid      - if 'yes' load data to get UID besides UOA
            }

    Output: {
              cm_return       - return code >0 if error
                                            -1 if doesn't exist
              cm_path         - path to the local repository
              (cm_module_uoa) - module UOA of the current path
              (cm_module_uid) - module UID of the current path
              (cm_data_uoa)   - data UOA of the current path
              (cm_data_uid)   - data UID of the current path
            }
    """

    pe=ini['dcfg']['cmr_ext']

    px=os.path.join(os.getcwd())
    py="dummy"

    paths=[]

    p=os.path.join(px,pe)
    if os.path.isdir(p)==True:
       return {'cm_return':0, 'cm_path':p}

    while py!="":
       px0=os.path.split(px)
       px=px0[0]
       px1=px0[1]
       py=os.path.split(px)[1]
       p=os.path.join(px,pe)
       if os.path.isdir(p):
          r={'cm_return':0, 'cm_path':p}
          if len(paths)==1:
             r['cm_module_uoa']=paths[0]
          elif len(paths)>=2:
             l=len(paths)
             r['cm_module_uoa']=paths[l-1]
             r['cm_data_uoa']=paths[l-2]

          if i.get('cm_get_uid','')=='yes' and r.get('cm_data_uoa','')!='':
             r1=copy.deepcopy(r)
             rx=load_data(r1)
             if rx['cm_return']>0: return rx

             r['cm_module_uid']=rx['cm_module_uid']
             r['cm_data_uid']=rx['cm_uid']
          return r
       else:
          paths.append(px1)

    return {'cm_return':-1}

# ============================================================================
def find_path_to_work_repositories(i):
    """
    Find path to work and local repositories

    Input:  {
            }

    Output: {
              cm_return  - return code >0 if error
              cm_array   - array of paths
            }
    """

    a=[]

    r=find_path_to_current_repository({})
    if r['cm_return']>0: return r
    elif r['cm_return']==0:
       a.append(r['cm_path'])

    # Check local repository
    if 'cm_local_repo' in cm_kernel.ini:
       p=cm_kernel.ini['cm_local_repo']
       found=False
       for q in a:
           if os.stat(q)==os.stat(p):
              found=True
              break
       if not found: a.append(p)

    # Check default repository
    p=cm_kernel.ini['cm_default_repo']
    found=False
    for q in a:
        if os.stat(q)==os.stat(p):
           found=True
           break
    if not found: a.append(p)

    return {'cm_return':0, 'cm_array':a}

# ============================================================================
def load_json_file(i):
    """
    Load json from file

    Input:  {
              cm_filename - file name with json
            }

    Output: {
              cm_return - return code = 0 if successful
              cm_array  - array from json file
            }
    """

    if 'cm_filename' not in i:
       return {'cm_return':1, 'cm_error':'parameter "cm_filename" is not defined'}

    try:
       f=file(i['cm_filename'])
    except Exception as e:
       return {'cm_return':1, 'cm_error':format(e)}

    try:
       array=json.loads(f.read())
    except Exception as e:
       f.close()
       return {'cm_return':1, 'cm_error':'problem loading array from file='+i['cm_filename']+' ('+format(e)+')'}

    f.close()

    return {'cm_return':0, 'cm_array': array}

# ============================================================================
def save_array_to_file_as_json(i):
    """
    Save array to file

    Input:  {
              cm_filename - file name with json
              cm_array    - array to save
            }

    Output: {
              cm_return - return code = 0 if successful
            }
    """

    if 'cm_filename' not in i:
       return {'cm_return':1, 'cm_error':'parameter "cm_filename" is not defined'}

    if 'cm_array' not in i:
       return {'cm_return':1, 'cm_error':'parameter "cm_array" is not defined'}

    s = json.dumps(i['cm_array'], indent=2, sort_keys=True, encoding="utf-8")

    try:
       f=file(i['cm_filename'], 'w')
    except Exception as e:
       return {'cm_return':1, 'cm_error':format(e)}

    try:
       f.write(s+'\n')
    except Exception as e:
       f.close()
       return {'cm_return':1, 'cm_error':'problem writing array to file='+i['cm_filename']+' ('+format(e)+')'}

    f.close()

    return {'cm_return':0}

# ============================================================================
def load_array_from_file(i):
    """
    Load array from file

    Input:  {
              cm_filename - file name
            }

    Output: {
              cm_return  - return code = 0 if successful
              cm_array   - array
            }
    """

    if 'cm_filename' not in i:
       return {'cm_return':1, 'cm_error':'parameter "cm_filename" is not defined'}

    array=[]

    try:
       f=file(i['cm_filename'])
    except Exception as e:
       return {'cm_return':1, 'cm_error':format(e)}

    try:
       for line in f:
           array.append(line)
    except Exception as e:
       f.close()
       return {'cm_return':1, 'cm_error':'problem loading array from file='+i['cm_filename']+' ('+format(e)+')'}

    f.close()

    return {'cm_return':0, 'cm_array': array}

# ============================================================================
def save_flat_array_to_file(i):
    """
    Load array from file

    Input:  {
              cm_array       - array to save (only flat one)
              cm_filename    - file name
              separate_lines - if 'yes', put key and value on separate lines
            }

    Output: {
              cm_return  - return code = 0 if successful
            }
    """

    if 'cm_filename' not in i: return {'cm_return':1, 'cm_error':'parameter "cm_filename" is not defined'}
    if 'cm_array' not in i: return {'cm_return':1, 'cm_error':'parameter "cm_array" is not defined'}

    separate=False
    if i.get('separate_lines','')=='yes': separate=True

    a=i['cm_array']

    try:
       f=file(i['cm_filename'], 'w')
       for x in a:
           if separate:
              f.write(str(x)+'\n')
              f.write(str(a[x])+'\n')
           else:
              f.write(str(x)+'='+str(a[x])+'\n')
       f.close()
    except Exception as e:
       return {'cm_return':1, 'cm_error':format(e)}

    return {'cm_return':0}

# ============================================================================
def flatten_array(i):
    """
    Flatten array
    Any list item is converted to @number=value
    Any dict item is converted to #key=value
    # is always added at the beginning 

    Input:  {
              cm_array        - full array
              prefix          - prefix (for recursion)
            }

    Output: {
              cm_return - if =0, success
              cm_array  - flattened array
            }
    """

    # Check vars
    if 'cm_array' not in i: return {'cm_return':1, 'cm_error':'parameter "cm_array" is not defined in "kernel flatten_array"'}

    prefix='#'
    if 'prefix' in i: prefix=i['prefix']

    a=i['cm_array']
    aa={}

    flatten_array_internal(a, aa, prefix)

    return {'cm_return':0, 'cm_array': aa}

# ============================================================================
def flatten_array_internal(a, aa, prefix):
    # Start flattening
    if type(a) is dict or type(a) is list:
       i=0
       for x in a:
           if type(a) is dict: 
              v=a[x] 
              prefix1=prefix+'#'+x
           else: 
              prefix1=prefix+'@'+str(i)
              v=x
           if type(v) is dict or type(v) is list:
              flatten_array_internal(v, aa, prefix1)
           else:
              aa[prefix1]=v
           i+=1
    else:
       aa[prefix]=a

    return {'cm_return':0, 'cm_array': a}

# ============================================================================
def get_value_by_flattened_key(i):
    """
    Get value from array using flattened key

    Input:  {
              cm_array - array
              cm_key   - flattened key
            }

    Output: {
              cm_return - if =0, success
              cm_value  - value or None
            }
    """
    # Check vars
    if 'cm_array' not in i: return {'cm_return':1, 'cm_error':'"cm_array" is not defined in "kernel get_value_by_flattened_key"'}
    if 'cm_key' not in i: return {'cm_return':1, 'cm_error':'"cm_key" is not defined in "kernel get_value_by_flattened_key"'}

    v=None

    a=i['cm_array']
    k=i['cm_key']
 
    # Remove leading # if there 
    if len(k)>0 and k[0:1]=='#': k=k[1:]

    k1=''
    kt='' # type '#' or '@'
    x=0
    finish=False

    while not finish:
        y=k[x]
        x+=1

        if y=='#' or y=='@':
           if kt=='#':
              if k1 not in a: break
              a=a[k1]
           elif kt=='@':
              if len(a)<=long(k1): break
              a=a[long(k1)]
           k1=''
           kt=y
        else:
           k1+=y

        if x>=len(k): break

    if k1!='' and kt!='':
       if kt=='#':   
          if k1 in a: v=a[k1]
       else:         
          if len(a)>long(k1): v=a[long(k1)]

    return {'cm_return':0, 'cm_value': v}

# ============================================================================
def set_value_by_flattened_key(i):
    """
    Set value in array using flattened key

    Input:  {
              cm_array           - array (it will be directly changed!)
              cm_key             - flattened key (or not if doesn't start with #)
              cm_value           - value to set
              cm_replace_in_keys - array with strings to replace in keys
                                   (used in forms to process # and @ through ^35^ and ^64^ for example)
            }

    Output: {
              cm_return - if =0, success
              cm_array  - modified array
            }
    """
    # Check vars
    if 'cm_array' not in i: return {'cm_return':1, 'cm_error':'"cm_array" is not defined in "kernel get_value_by_flattened_key"'}
    if 'cm_key' not in i: return {'cm_return':1, 'cm_error':'"cm_key" is not defined in "kernel get_value_by_flattened_key"'}
    if 'cm_value' not in i: return {'cm_return':1, 'cm_error':'"cm_value" is not defined in "kernel get_value_by_flattened_key"'}

    rep={}
    if 'cm_replace_in_keys' in i: rep=i['cm_replace_in_keys']

    a=i['cm_array']
    k=i['cm_key']
    v=i['cm_value']

    if len(k)>0 and k[0]!='#':
       k=replace_from_dict(k, rep)
       a[k]=v
    else:
       # Remove leading # if there 
       if len(k)>0 and k[0:1]=='#': k=k[1:]

       k1=''
       kt='' # type '#' or '@'
       x=0
       finish=False

       while not finish:
           y=k[x]
           x+=1

           if y=='#' or y=='@':
              if kt=='#':
                 k1=replace_from_dict(k1, rep)
                 if k1 not in a: 
                    if y=='#': a[k1]={}
                    else: a[k1]=[]
                 a=a[k1]
              elif kt=='@':
                 if len(a)<=long(k1): 
                    for q in range(len(a)-1,long(k1)):
                        if y=='#': a.append({})
                        else: a.append([])
                 a=a[long(k1)]
              k1=''
              kt=y
           else:
              k1+=y

           if x>=len(k): break

       if k1!='' and kt!='':
          if kt=='#':   
             k1=replace_from_dict(k1, rep)
             a[k1]=v
          else:         
             if len(a)<=long(k1): 
                for q in range(len(a)-1,long(k1)):
                    if y=='#': a.append({})
                    else: a.append([])
             a[long(k1)]=v

    return {'cm_return':0, 'cm_array': i['cm_array']}

# ============================================================================
def replace_from_dict(s, rep):
    """
    Replace substrings from array rep in string s

    Input:  s   - string
            rep - array with substrings to replace

    Output: new string
    """

    for x in rep:
        s=s.replace(x, rep[x])

    return s

# ============================================================================
def restore_flattened_array(i):
    """
    Restore flattened array

    Input:  {
              cm_array             - flattened array
              (cm_replace_in_keys) - array with strings to replace in keys
                                     (used in forms to process # and @ through ^35^ and ^64^ for example)
            }

    Output: {
              cm_return - if =0, success
              cm_array  - restored array
            }
    """
    # Check vars
    if 'cm_array' not in i: return {'cm_return':1, 'cm_error':'"cm_array" is not defined in "kernel restore_flattened_key"'}

    rep={}
    if 'cm_replace_in_keys' in i: rep=i['cm_replace_in_keys']

    a={} # default
    b=i['cm_array']
    first=True
    for x in b:
        if first: 
           first=False
           y=x[1:2]
           if y=='@': a=[]
           else: a={}

        cm_kernel.set_value_by_flattened_key({'cm_array':a, 'cm_key':x, 'cm_value':b[x], 'cm_replace_in_keys':rep})

    return {'cm_return':0, 'cm_array': a}
    
# ============================================================================
def delete_directory(i):
    """
    Delete a given directory with subdirectories (be careful)

    Input:  {
              cm_path - path to delete
            }

    Output: {
              cm_return  - return code = 0 if successful
            }
    """

    r=check_global_writing({})
    if r['cm_return']>0: return r

    if 'cm_path' not in i:
       return {'cm_return':1, 'cm_error':'parameter "cm_path" is not defined'}
    p=i['cm_path']

    if not os.path.isdir(p):
       return {'cm_return':1, 'cm_error':p+' is not directory'}

    forbid=False
    if os.path.isdir(ini['cm_default_repo']) and os.stat(p)==os.stat(ini['cm_default_repo']): forbid=True
    if os.path.isdir(ini[env_cm_root]) and os.stat(p)==os.stat(ini[env_cm_root]): forbid=True

    if 'forbid_deletion_of_dirs' in ini['dcfg']:
       for x in ini['dcfg']['forbid_deletion_of_dirs']:
           if os.path.isdir(x) and os.stat(p)==os.stat(x): forbid=True; break

    if forbid:
       return {'cm_return':1, 'cm_error': 'forbidden to delete '+p}

    shutil.rmtree(p)

    return {'cm_return':0}

# ============================================================================
def convert_string_to_html(i):
    """
    Convert string to html (remove special characters)

    Input:  {
              cm_string  - string to process
              cm_skip_br - if 'yes', do not add <BR>
            }

    Output: {
              cm_return - return code = 0 if successful
              cm_string - processed string
            }
    """

    if 'cm_string' not in i: return {'cm_return':1, 'cm_error':'parameter "cm_string" is not defined in kernel/convert_string_to_html'}

    s=i['cm_string']
    s=s.replace('&', '&amp;')
    s=s.replace('<', '&lt;')
    s=s.replace('>', '&gt;')
    s=s.replace('"', '&quot;')
    s=s.replace('\r', '')
#    s=s.replace(' ', '&nbsp;')

    if i.get('cm_skip_br', '')!='yes': 
       s=s.replace('\n', '<BR>')

    return {'cm_return':0, 'cm_string':s}

# ============================================================================
def flatten_classification(i):
    """
    Flatten classification ["class", "subclass", ..."] to class.subclass. ...

    Input:  {
              cm_classification=[]
            }

    Output: {
              cm_return - return code = 0 if successful
              cm_string - classification as string
            }
    """

    if 'cm_classification' not in i: return {'cm_return':1, 'cm_error':'parameter "cm_classification" is not defined in kernel/flatten_classification'}

    cc=i['cm_classification']
    cs=''

    first=True; y=''
    for x in cc:
        if first: first=False
        else: y+='.'
        y+=x

    return {'cm_return':0, 'cm_string':y}

# ============================================================================
def set_print_mode(i):
    """
    Set print mode

    Input:  {
              'mode' = 'on' | 'off'
            }

    Output: 
    """
    global print_on

    mode=i.get('mode','on')
    if mode=='on': print_on=True
    else: print_on=False

    return {'cm_return':0}

# ============================================================================
def print_for_web(s):
    """
    Print string in UTF-8

    Input:  s   - string

    Output: 
    """

    if print_on:
       print(s.encode("utf-8", 'ignore'))

    return

# ============================================================================
def print_for_con(s):
    """
    Print string

    Input:  s   - string

    Output: 
    """

    if print_on:
       print(s)

    return

# ============================================================================
def get_all_uoa(i):
    """
    Get a list of all entries in the repository

    Input:  {
              cm_path                      - path to repository
              (cm_module_uoa)              - module UOA. If =='', return available modules in the repository
                                                         (or use cm_only_available_modules explicitly)
              cm_only_uid                  - 'yes' to show only UIDs
              (cm_classes_uoa)             - prune by classes UOA
              (prune_data_uoa)             - prune by data UOA
              (cm_only_available_modules)  - if 'yes', return only available modules in a given repository
              (skip_data_cfg)              - if 'yes', do not add data cfg
              (cm_admin)                   - if 'yes', override access control - only internal use,
                                             can not be used during web access (explicitly removed)
            }

    Output: {
              cm_return     - return code = 0 if successful
              cm_array      - list of UOA or only UID
              cm_mixed      - list of {cm_uid, cm_alias, cm_uoa, cm_display_as_alias, cm_display_html, cm_data_obj_cfg}
            }
    """

    if 'cm_path' not in i:
       return {'cm_return':1, 'cm_error':'parameter "cm_path" is not defined'}

    sdc=False
    if i.get('skip_data_cfg','')=='yes': sdc=True

    array=[]
    mixed=[]

    pp=i['cm_path']

    module_uoa=i.get('cm_module_uoa','')
    if i.get('cm_only_available_modules','')!='yes' and module_uoa!='':
       r=find_path_to_entry({'cm_path':pp, 'cm_data_uoa':module_uoa})
       if r['cm_return']>0: return r
       elif r['cm_return']<0:
          # entries do not exist in this repository, quit
          return {'cm_return':0, 'cm_array':array, 'cm_mixed':mixed}

       p=r['cm_path']
    else:
       module_uoa=ini['dcfg']['cm_module_module_uoa']
       p=pp
       pp='' # To search in all repos

    only_uid=False
    if i.get('cm_only_uid','')=='yes': only_uid=True

    if os.path.isdir(p):
       dirList=os.listdir(p)
       for fn in dirList:
           p1=os.path.join(p,fn)
           if fn!=ini['dcfg']['dcm'] and fn not in ini['dcfg']['ignore_entries'] and os.path.isdir(p1):
              # Attempt to load
              ii={'cm_path':pp,
                  'cm_module_uoa':module_uoa,
                  'cm_data_uoa':fn}
              if i.get('cm_admin','')!='': ii['cm_admin']=i['cm_admin']
              r1=load_data(ii)
              if r1['cm_return']==0:
                 d=r1['cm_data_obj']['cfg']

                 if i.get('prune_data_uoa','')!='': 
                    if (r1['cm_uid']!=i['prune_data_uoa'] and r1['cm_uoa']!=i['prune_data_uoa']):
                       break

                 if 'cm_classes_uoa' in i:
                    add=True
                    if 'cm_classes_uoa' not in d: 
                       add=False
                    else:
                       for q in i['cm_classes_uoa']:
                           if q not in d['cm_classes_uoa']:
                              add=False
                              break
                    if not add: continue
                 
                 if only_uid:
                    x=r1['cm_uid']
                 else:
                    x=fn

                 array.append(x)

                 jj={'cm_uid':r1['cm_uid'], \
                     'cm_alias':r1['cm_alias'], \
                     'cm_uoa':r1['cm_uoa'], \
                     'cm_module_uid':r1['cm_module_uid'], \
                     'cm_module_uoa':r1['cm_module_uoa'], \
                     'cm_display_as_alias':r1['cm_display_as_alias'], \
                     'cm_display_html':r1['cm_display_html'], \
                    }
                 if not sdc: jj['cm_data_obj_cfg']=d
                 mixed.append(jj)

    return {'cm_return':0, 'cm_array':array, 'cm_mixed':mixed}

# ============================================================================
def get_list_of_all_files(i):
    """
    Get a list of all files in a given directory recursively

    Input:  {
              cm_path      - path
              cm_path_ext  - path extension
              limit        - limit number of files (if huge directories)
              number       - cur. number of files
              cm_all_files - if 'yes' do not ignore special files (like .cm)
            }

    Output: {
              cm_return - return code = 0 if successful
              cm_array  - list of files
            }
    """

    number=0
    if 'number' in i: number=i['number']

    limit=-1
    if 'limit' in i: limit=i['limit']

    a=[] 

    pe=''
    if 'cm_path_ext' in i: pe=i['cm_path_ext']

    try:
       dirList=os.listdir(i['cm_path'])
    except Exception as e:
        None
    else:
        for fn in dirList:
            # FGG: should move to cfg ...
            if fn!='.svn':
               p=os.path.join(i['cm_path'], fn)
               if i.get('cm_all_files','')=='yes' or (fn!=ini['dcfg']['dcm'] and fn not in ini['dcfg']['ignore_entries']):
                  if os.path.isdir(p):
                     r=get_list_of_all_files({'cm_path':os.path.join(p), 'cm_path_ext':os.path.join(pe, fn), 'number':number})
                     if r['cm_return']>0: return r
                     a.extend(r['cm_array'])
                  else:
                     a.append(os.path.join(pe, fn))

                  number=len(a)
                  if limit!=-1 and number>limit: break
 
    return {'cm_return':0, 'cm_array':a, 'number':number}

# ============================================================================
def get_data_description(i):

    """
    Get data entry description

    Input:  {
              cm_run_module_uoa | cm_module_uoa - module UOA
              (cm_desc_key)                     - if this key is specified, take description from this key
              (cm_flat_desc_key)                - if this flat key is specified, take description from this flat key
              (cm_action_key)                   - action key
              cm_which_action                   - action
              (cm_key)                          - if action is specified, use this key instead of default one
            }

    Output: {
              cm_return         - return code >0 if error
              cm_data_desc      - data description
              cm_params_default - data description
              ...               - other parameters from load_data
            }
    """

    # Module command params description
    a={}
    
    # Module default params for this action
    b={}

    # Load module
    r1={}
    if 'cm_repo_uoa' in i: r1['cm_repo_uoa']=i['cm_repo_uoa']
    r1['cm_data_uoa']=i['cm_module_uoa']
    r1['cm_module_uoa']=cm_kernel.ini['dcfg']['cmr_module']
    r=load_data(r1)
    if r['cm_return']>0: return r
    d=r['cm_data_obj']

    if i.get('cm_desc_key','')!='':
       a.update(d['cfg'].get(i['cm_desc_key'],{}))
    elif i.get('cm_flat_desc_key','')!='':
       rx=cm_kernel.get_value_by_flattened_key({'cm_array':d['cfg'], 'cm_key':i['cm_flat_desc_key']})
       if rx['cm_return']==0 and rx['cm_value']!=None: 
          a.update(rx['cm_value'])
    else:
       ak=i.get('cm_action_key','')
       if ak=='': ak='cm_actions'

       if 'cm_which_action' in i:
          if ak in d['cfg']: 
             r1=cm_kernel.find_index_in_array({
                                               'cm_array':d['cfg'][ak],
                                               'cm_index':i['cm_which_action']
                                              })
             if r1['cm_return']==0:
                key=i.get('cm_key','')
                if key=='' and 'params_desc' in r1['cm_array']: a.update(r1['cm_array']["params_desc"])
                elif key!='' and key in r1['cm_array']: a.update(r1['cm_array'][key])
                if 'params_default' in r1['cm_array']: b.update(r1['cm_array']["params_default"])
       else:
          # Check standard cm_data_description and redirect

          red=d['cfg'].get('cm_data_description_from_other_module',{})
          if red.get('cm_module_uoa','')!='':
             # Load another module
             ii={'cm_run_module_uoa':cm_kernel.ini['dcfg']['cmr_module'],
                 'cm_action':'load',
                 'cm_data_uoa':red['cm_module_uoa']}
             r=cm_kernel.access(ii)
             if r['cm_return']>0: return r
             d=r['cm_data_obj']
             
          if 'cm_data_description' in d['cfg']:
             a.update(d['cfg']['cm_data_description'])

    ii={}; ii.update(r)
    ii['cm_return']=0
    ii['cm_data_desc']=a
    ii['cm_params_default']=b
    return ii

# ============================================================================
def system(i):

    """
    System call with timeout (careful, may cause overheads)

    Input:  {
              cmd       - command line
              (timeout) - timeout in seconds (granularity 0.01 sec) - may cause overheads ...
            }

    Output: {
              cm_return      - return code =0 if success
                                           =1 if timedout and killed
              (cm_error)     - error text  if return code > 0
              cm_return_code - return code of a program
            }
    """

    rc=0
    if 'timeout' not in i or i['timeout']=='' or float(i['timeout'])==0:
       rc=os.system(i['cmd'])
    else:
       tstart=time.time()
       t=0
       tx=float(i['timeout'])

       p=subprocess.Popen(i['cmd'], shell=True)

       while p.poll() == None and t<tx:
          time.sleep(0.01)
          t=time.time()-tstart

       if t>=tx and p.poll()==None:
          p.kill()

          return {'cm_return':1, 'cm_error':'process timed out and was killed'}

       rc=p.returncode

    return {'cm_return':0, 'cm_return_code':rc}

# ============================================================================
def gen_cm_tmp_file(i):

    """
    Generate tmp file name

    Input:  {}

    Output: {
              cm_return  - return code >0 if error
              cm_path    - path to file
              cm_path1   - path to file without extension
                           (for example, a script will create many sub-files with different extensions)
              cm_uid     - generated uid
            }
    """

    r=gen_uid({})
    if r['cm_return']>0: return r
    uid=r['cm_uid']

    p1=os.path.join(ini[env_cm_tmp], ini['dcfg']['cm_tmp_prefix']+uid)
    p=p1+ini['dcfg']['cm_tmp_postfix']

    return {'cm_return':0, 'cm_path':p, 'cm_uid':uid, 'cm_path1':p1}

# ============================================================================
def array2cm(a):
    """
    Convert list to cm array

    cm_module_uoa cm_action A=B C=D ... @file1.json E=F ... -- <unparsed arguments>

    if extension of @file is .tmp, it will be deleted after read!

    Input:  input array

    Output: {
              cm_return  - return code = 0 if successful
              cm_array   - array
              cm_console - if 'cm_console' in array, set it to the value of 'cm_console' parameter
            }
    """
    obj={}
    con=''

    la=len(a)

    obj['cm_run_module_uoa']=cm_module_default
    obj['cm_action']=cm_action_default

    if la>0: obj['cm_run_module_uoa']=a[0]

    # Parsing
    for x in range(1, len(a)):
        p=a[x].rstrip()
        if p==cm_unparsed_sep:

           obj['cm_unparsed']=a[x+1:]
           break

        elif p.startswith("@@"):
           jd=p[2:]
           if len(jd)<2:
              return {'cm_return':1, 'cm_error':'can\'t parse command line option '+p, 'cm_console':con}
           je=check_cm_json({'cm_json':jd})
           obj.update(je)

        elif p.startswith("@"):
           name=p[1:]
           if len(name)<2:
              return {'cm_return':1, 'cm_error':'can\'t parse command line option '+p, 'cm_console':con}

           y=load_json_file({'cm_filename':name})
           if y['cm_return']>0: 

              ii={}; ii.update(y); ii.update({'cm_console':con})
              return ii

           if name.endswith('.tmp'):
              os.remove(name)   

           merge_arrays({'cm_array':obj, 'cm_array1':y['cm_array']})
        else:
           if x==1:
              obj['cm_action']=p
           else:
              p1=p 
              p2=''
              q=p.find("=")
              if q>0:
                 p1=p[0:q]
                 if len(p)>q:
                   p2=p[q+1:]

              obj[p1]=p2
              if p1=='cm_console': con=p2

    return {'cm_return':0, 'cm_array':obj, 'cm_console':con}

# ============================================================================
def convert_text_to_value(i):
    """
    Convert cm text to cm value

    Input:  {
              cm_text - text to convert
              cm_type - type: {float, integer, string, uoa}

    Output: {
              cm_return - return code = 0 if successful
              cm_value  - value in python format
            }
    """

    v=None

    if 'cm_text' not in i: return {'cm_return':1, 'cm_error':'"cm_text" is not defined in kernel/convert_text_to_value'}
    if 'cm_type' not in i: return {'cm_return':1, 'cm_error':'"cm_type" is not defined in kernel/convert_text_to_value'}

    t=i['cm_text']
    tp=i['cm_type']

    if tp=='float':
       try: 
          v=float(t)
       except ValueError:
          return {'cm_return':16, 'cm_error':'can\'t convert '+str(t)+' to float'}
    elif tp=='integer':
       try: 
          v=int(t)
       except ValueError:
          return {'cm_return':16, 'cm_error':'can\'t convert '+str(t)+' to int'}
    elif tp=='uoa' or tp=='string' or tp=='text':
       v=str(t)
    else:
       return {'cm_return':16, 'cm_error':'can\'t recognize type '+tp}

    return {'cm_return':0, 'cm_value':v}

# ============================================================================
def find_keys_by_prefix(i):
    """
    Find keys by prefix

    Input:  {
              cm_array         - array to search
              cm_prefix        - prefix to search
              cm_sort_key_int  - sort key (int)
              cm_sort_key_text - sort key (text)
            }

    Output: {
              cm_return - return code = 0 if successful
              keys      - list of keys
            }
    """

    prefix=i.get('cm_prefix','')
    ski=i.get('cm_sort_key_int','')
    skt=i.get('cm_sort_key_text','')

    keys=[]

    a=i.get('cm_array',{})
    if ski!='':
       b=sorted(a.keys(), key=lambda k: (int(a[k].get(ski,'999999'))))
    elif skt!='':
       b=sorted(a.keys(), key=lambda k: (a[k].get(skt,'').lower()))
    else:
       b=a

    for k in b:
        if k.startswith(prefix): keys.append(k)

    return {'cm_return':0, 'cm_keys':keys}

# ============================================================================
def merge_arrays_ll(a, b):
    """
    Intelligently merge cm_array with cm_array1

    Input:  a  - merge this array with cm_array1 (will be directly modified!)
            b - cM array

    Output: {
              cm_return - return code = 0 if successful
            }
    """
    return merge_arrays({'cm_array':a, 'cm_array1':b})

# ============================================================================
def merge_arrays(i):
    """
    Intelligently merge cm_array with cm_array1

    Input:  {
              cm_array  - merge this array with cm_array1 (will be directly modified!)
              cm_array1 - cM array

    Output: {
              cm_return - return code = 0 if successful
              cm_array  - output array
            }
    """

    if 'cm_array' not in i: return {'cm_return':1, 'cm_error':'"cm_array" is not defined in "cm_kernel merge_arrays"'}
    if 'cm_array1' not in i: return {'cm_return':1, 'cm_error':'"cm_array1" is not defined in "cm_kernel merge_arrays"'}

    a=i['cm_array']
    b=i['cm_array1']

    for k in b:
        v=b[k]
        if type(v) is dict:
           if k not in a:
              a.update({k:b[k]})
           elif type(a[k])==dict:
              merge_arrays({'cm_array':a[k], 'cm_array1':b[k]})
           else:
              a[k]=b[k]
        elif type(v) is list:
#           if k not in a or type(a[k]) is not list: 
#              a[k]=[]
           a[k]=[]
           for y in v:
               a[k].append(y)
        else:
           a[k]=b[k]

    return {'cm_return':0, 'cm_array':a}

# ============================================================================
def str2cm(i):
    """
    Convert string to cM array

    Input:  string

    Output: {
              cm_return - return code = 0 if successful
              cm_array  - array
            }
    """
    # FGG: I changed it to shlex to be able to ignore spaces in quotes
#    a=i.split(' ')
    a=shlex.split(i)
    return array2cm(a)

# ============================================================================
def array2str(i):
    """
    Convert array to string with space

    Input:  array

    Output: {
              cm_return - 0
              cm_string - string
            }
    """
    s=string.join(i, ' ')
    return {'cm_return':0, 'cm_string':s}

# ============================================================================
def get_var(a,p):
    """
    Check if parameter 'p' is in array 'a' and return a[p] or '' otherwise

    Input:  a - array
            p - parameter

    Output: a[p], if 'p' in 'a'
            '', if 'p' not in 'a'

    """

    if p not in a: return ''
    return a[p]

# ============================================================================
def prepare_final_out(i, j):
    """
    Prepare final output of cM access function depending on cm_console

    Input:  i= {
                 cm_return    - return code (check "access" function for full description)
                 (cm_error)   - error text  (check "access" function for full description)
                 ...
               }

            j= {
                 (cm_console) - cm_console  (check "access" function for full description)
                 (cm_output)  - if !='', save output to file
                 ...
               }

    Output: {}

    """

    # Prepare output if error
    if i['cm_return']>0:
       if j.get('cm_console','')=='txt': print_for_con('cM error: '+i['cm_error']+'!')
       if j.get('cm_console','')=='web': print_for_web('<br><pre>cM error: '+i['cm_error']+'!</pre><br>')

    # Check if record to file
    if j.get('cm_output','')!='':
       rx=save_array_to_file_as_json({'cm_filename':j['cm_output'], 'cm_array':i})
       if rx['cm_return']>0: 
          print_for_con('cM error: '+rx['cm_error']+'!')

    # In any case, if json, print json
    if j.get('cm_console','')=='json': 
       print_for_con(json.dumps(i))
    if j.get('cm_console','')=='json_after_text': 
       print_for_con(cm_json_with_text_sep)
       print_for_con(json.dumps(i))
    elif j.get('cm_console','')=='json_with_indent': 
       print_for_con(json.dumps(i, indent=2, sort_keys=True))

    return {}

# ============================================================================
def check_cm_json(i):
    """
    Check if 'cm_json' exist in array and decode it

    Input:  {
              cm_json - array to check
            } 

    Output: {...} - returned array where cm_json is decoded
    """

    ii={}
    ii.update(i)

    if 'cm_json' in i:
       s=i['cm_json']
       s=s.replace('^22^', '"')
       s=s.replace('%22', '"')
       s=s.replace('%26quot;', '"')
       s=s.replace('&quot;', '"')
       success=False
       try:
          j=json.loads(s)
          success=True
       except:
          pass
       if success:
          del (ii['cm_json'])
          ii.update(j)

    return ii

# ============================================================================
def convert_cm_array_to_uri(i):
    """
    Convert cM array to uri (convert various special parameters)

    Input:  {
              cm_array   - array to convert to uri
            }

    Output: {
              cm_return  - return code (check "access" function for full description)
              (cm_error) - error text  (check "access" function for full description)
              cm_string  - converted string
              cm_string1 - converted string (for urlopen)
            }
    """

    if 'cm_array' not in i:
       return {'cm_return':1, 'cm_error':'"cm_array" is not set in kernel/convert_cm_array_to_uri'}

    ca=i['cm_array']
    cca=copy.deepcopy(ca)

    if 'cm_user_uoa' in cca: del(cca['cm_user_uoa'])

    s=json.dumps(cca)
    s=s.replace('"', '^22^')
    s=s.replace('#', '%23')

    s=s.replace('+', '%2B')

    s1=s.replace('>', '%3E')
    s1=s1.replace('<', '%3C')

    s1=s1.replace('[', '%5B')
    s1=s1.replace(']', '%5D')
    s1=s1.replace(' ', '%20')

    s=cgi.escape(s)

    return {'cm_return':0, 'cm_string':s, 'cm_string1':s1}

# ============================================================================
def copy_vars(i):
    """
    Convert cM array to uri (convert various special parameters)

    Input:  {
              cm_input_array  - input array
              cm_output_array - output array
              cm_vars         - list of variables to copy
            }

    Output: {
              cm_return  - return code (check "access" function for full description)
            }
    """

    if 'cm_input_array' not in i: return {'cm_return':1, 'cm_error':'"cm_input_array" is not defined in kernel/copy_vars'}
    if 'cm_output_array' not in i: return {'cm_return':1, 'cm_error':'"cm_output_array" is not defined in kernel/copy_vars'}
    if 'cm_vars' not in i: return {'cm_return':1, 'cm_error':'"cm_vars" is not defined in kernel/copy_vars'}

    inp=i['cm_input_array']
    out=i['cm_output_array']

    vr=i['cm_vars']

    inp1=inp.keys()
    if len(vr)>0: inp1=vr

    for x in inp1:
        if x in inp: out[x]=inp[x]

    return {'cm_return':0}

# ============================================================================
def convert_str_to_special(s):
    """
    Convert string to a special cM string that is used to substitute variables in text

    Input:  string

    Output: special string 
    """

    return '$#'+s+'#$'

# ============================================================================
def get_cm_version(i):

    """
    Get cM version

    Input:  {
            }

    Output: {
              cm_return  - return code >0 if not authentificated
              cm_array   - array with version
              cm_string  - version
            }
    """

    a={}
    x=''

    if 'cm_version' in ini['dcfg']:
       a=ini['dcfg']['cm_version']
 
    if 'major' in a: x+=str(a['major'])
    if 'minor' in a: x+='.'+str(a['minor'])
    if 'build' in a: x+='.'+str(a['build'])
    if 'extra' in a: x+='.'+str(a['extra'])

    return {'cm_return':0, 'cm_array':a, 'cm_string':x}

# ============================================================================
def convert_str_to_sha1(i):

    """
    Convert string to sha1

    Input:  {
              cm_string       - string
            }

    Output: {
              cm_return             - return code >0 if not authentificated
              cm_string_sha1        - password in SHA1 (digest)
              cm_string_sha1_hex    - password in SHA1 (digest in hex)
              cm_string_sha1_base64 - BASE64 (SHA1 digest) - compatible with htpasswd format
            }
    """

    if 'cm_string' not in i: 
       return {'cm_return':1, 'cm_error':'"cm_string" is not set'}

    x=hashlib.sha1()
    x.update(i['cm_string'])
    y=x.digest().strip()
    z=x.hexdigest().strip()
    b=base64.encodestring(y).strip()

    return {'cm_return':0, 'cm_string_sha1':y, 'cm_string_sha1_hex':z, 'cm_string_sha1_base64':b}

# ============================================================================
def prepare_cid(i):

    """
    Prepare cid 

    Input:  {
              (cm_repo_uoa)                     - repository UOA
              cm_run_module_uoa | cm_module_uoa - module UOA
              cm_data_uoa                       - data UOA
            }

    Output: {
              cm_return               - 0
              cm_string_uid           - cid with only uid
              cm_string_with_repo_uid - cid with only uid
              cm_string_uoa           - cid with alias or uid
              cm_string_with_repo_uoa - cid with alias or uid
              cm_repo_uid             - cm_repo_uid
              cm_repo_uoa             - cm_repo_uoa
              cm_module_uid           - cm_module_uid
              cm_module_uoa           - cm_module_uoa
              cm_data_uid             - cm_data_uid
              cm_data_uoa             - cm_data_uoa
            }
    """

    cm_repo_uid=''
    cm_repo_uoa=''
    cm_module_uid=''
    cm_module_uoa=''
    cm_data_uid=''
    cm_data_uoa=''

    # Check repository
    if 'cm_repo_uoa' in i and i['cm_repo_uoa']!='':
       ii={'cm_run_module_uoa':ini['dcfg']['cmr_repo'],
           'cm_data_uoa':i['cm_repo_uoa'],
           'cm_action':'load'}
       r=cm_kernel.access(ii)
       if r['cm_return']==0:
          cm_repo_uid=r['cm_uid']
          cm_repo_uoa=cm_repo_uid
          if r['cm_alias']!='': cm_repo_uoa=r['cm_alias']
       else:
          cm_repo_uoa=i['cm_repo_uoa']

    # Check module
    if 'cm_module_uoa' in i and i['cm_module_uoa']!='':
       ii={'cm_run_module_uoa':ini['dcfg']['cmr_module'],
           'cm_data_uoa':i['cm_module_uoa'],
           'cm_action':'load'}
       r=cm_kernel.access(ii)
       if r['cm_return']==0:
          cm_module_uid=r['cm_uid']
          cm_module_uoa=cm_module_uid
          if r['cm_alias']!='': cm_module_uoa=r['cm_alias']
       else:
          cm_module_uoa=i['cm_module_uoa']

    # Check data
    if 'cm_data_uoa' in i and i['cm_data_uoa']!='':
       ii={'cm_run_module_uoa':i['cm_module_uoa'],
           'cm_data_uoa':i['cm_data_uoa'],
           'cm_action':'load'}
       r=cm_kernel.access(ii)
       if r['cm_return']==0:
          cm_data_uid=r['cm_uid']
          cm_data_uoa=cm_data_uid
          if r['cm_alias']!='': cm_data_uoa=r['cm_alias']
       else:
          cm_data_uoa=i['cm_data_uoa']

    # Prepare cid
    r={'cm_return':0}
    r['cm_string_uid']=cm_module_uid+':'+cm_data_uid
    if cm_repo_uid!='': r['cm_string_with_repo_uid']=cm_repo_uid+':'+r['cm_string_uid']
    r['cm_string_uoa']=cm_module_uoa+':'+cm_data_uoa
    if cm_repo_uoa!='': r['cm_string_with_repo_uoa']=cm_repo_uoa+':'+r['cm_string_uoa']

    r['cm_repo_uid']=cm_repo_uid
    r['cm_repo_uoa']=cm_repo_uoa
    r['cm_module_uid']=cm_module_uid
    r['cm_module_uoa']=cm_module_uoa
    r['cm_data_uid']=cm_data_uid
    r['cm_data_uoa']=cm_data_uoa

    return r

# ============================================================================
def access_control(i):

    """
    Control access 

    Input:  {
              dcfg                    - cfg of the current entry
              module_uoa              - module UOA of the entry
              module_uid              - module UID of the entry
              data_uoa                - data UOA of the entry
              key                     - 'read' or 'write'
            }

    Output: {
              cm_return - 0, if Ok; ==32 if access denied
              cm_error  - if access is denied
            }
    """

    cm_user_uid=ini['web_auth'].get('cm_user_uid','')
    cm_user_uoa=ini['web_auth'].get('cm_user_uoa','')
    xcfg=i.get('dcfg',{})

    module_uoa=i.get('module_uoa','')
    module_uid=i.get('module_uid','')
    data_uoa=i.get('data_uoa','')

    key=i.get('key','')

    ac=xcfg.get('access_control',{})

    aread=xcfg.get('cm_access_control',{}).get(key+'_groups','')

    access=False

    if aread=='all':
       access=True
    elif cm_user_uoa!='' and cm_user_uid!='':
       if cm_user_uoa in ini['dcfg'].get('admin_users',[]) or \
          cm_user_uid in ini['dcfg'].get('admin_users',[]):
          access=True
       else:
          access=False

          # Check if module=user or organization, i.e. accessing own user params
          if module_uid==ini['dcfg']['cm_user_module_uoa'] and \
             data_uoa==cm_user_uoa or data_uoa==cm_user_uid:
             access=True
          else:   
             # Check if allowed for registered
             if aread=='registered':
                access=True
             elif aread=='owner':
                # Otherwise check if owner
                cmu=xcfg.get('cm_updated',[])
                if len(cmu)>0:
                   owner_uoa=cmu[0].get('cm_user_uoa','')
                   if owner_uoa==cm_user_uoa or owner_uoa==cm_user_uid:
                      access=True

    else:
       # If no user
       if aread=='' or aread=='registered' or aread=='owner':
          access=False
    
    if not access:
       if ini['web_auth']['cm_user_status']=='failed':
          return {'cm_return':32, 'cm_error':key+' access denied to "'+module_uoa+':'+data_uoa+'" (user authentication failed: '+ini['web_auth']['cm_error']+')!'}
       else:
          return {'cm_return':32, 'cm_error':key+' access denied to "'+module_uoa+':'+data_uoa+'" (user likely have to be registered, be administrator or owner of this data)!'}

    return {'cm_return':0}

# ============================================================================
def auth_user(i):

    """
    Authenticate user

    Input:  {
              cm_user_uoa       - user UOA
              cm_user_password  - user password in plain text 
                                  (not secure, usually used for web form authentication)
              cm_user_password1 - user password in SHA1
                                  (more secure, can be stored in environment, session or cookies)
                                  cM authenicates by taking SHA1(cm_user_password1)...
              cm_user_password2 - only should be set here by web environment!
            }   

    Output: {
              cm_return            - return code = 0 if authenticated
              (cm_error)           - error text, if cm_return > 0
              cm_user_cfg          - user config if authenticated (to check groups, etc)
              cm_user_password2    - user password in SHA1(SHA1)
              cm_user_uid          - user UID
              cm_user_uoa          - user UOA
              cm_user_alias        - user alias
              cm_user_path         - user path
            }
    """

    # Unsafe to use environment for passwords but may be still needed
    #  for automatic experimentation on cloud/grid/supercomputers 
    # combined with remote aggregation of experiments

    # If no user/pass, try from environment
    if 'cm_user_uoa' not in i or i['cm_user_uoa']=='':
       if env_cm_user_uoa in os.environ.keys():
          i['cm_user_uoa']=os.environ[env_cm_user_uoa].strip()

    if 'cm_user_password' not in i or i['cm_user_password']=='':
       if env_cm_user_password in os.environ.keys():
          i['cm_user_password']=os.environ[env_cm_user_password].strip()

    if 'cm_user_password1' not in i or i['cm_user_password1']=='':
       if env_cm_user_password1 in os.environ.keys():
          i['cm_user_password1']=os.environ[env_cm_user_password1].strip()

    # Check vars
    if 'cm_user_uoa' not in i or i['cm_user_uoa']=='':
       return {'cm_return':127, 'cm_error': '"cm_user_uoa" is not set in auth_user'}

    # Check if passwords are in set
    pwd1=''
    pwd2=''
    if 'cm_user_password' in i and i['cm_user_password']!='':
       r=convert_str_to_sha1({'cm_string':i['cm_user_password']})
       if r['cm_return']>0: return r
       pwd1=r['cm_string_sha1_base64']
    elif 'cm_user_password1' in i and i['cm_user_password1']!='':
       pwd1=i['cm_user_password1']
    elif 'cm_user_password2' in i and i['cm_user_password2']!='':
       pwd2=i['cm_user_password2']

    if pwd2=='':
       if pwd1=='':
          return {'cm_return':127, 'cm_error':'authentication is required for this operation, but password is not set'}

       # Convert to SHA1 again!
       r=convert_str_to_sha1({'cm_string':pwd1})
       if r['cm_return']>0: return r
       pwd2=r['cm_string_sha1_base64']

    # Load user
    r=cm_kernel.access({'cm_run_module_uoa':ini['dcfg']['cm_user_module_uoa'],
                        'cm_action':'load',
                        'cm_data_uoa':i['cm_user_uoa'],
                        'cm_admin':'yes'})
    if r['cm_return']>0: 
       if r['cm_return']==16: r={'cm_return':16, 'cm_error':'can\'t find user "'+i['cm_user_uoa']+'"'}
       return r

    user_cfg=r['cm_data_obj']['cfg']
    user_path=r['cm_path']
    user_uid=r['cm_uid']
    user_uoa=r['cm_uoa']
    user_alias=r['cm_alias']

    if r['cm_uoa'] in ini['dcfg'].get('admin_users',[]) or \
       r['cm_uid'] in ini['dcfg'].get('admin_users',[]):
       ini['web_auth']['cm_admin']='yes'

    # Check if user entry has a password
    if 'cm_user_password2' not in user_cfg:
       return {'cm_return':127, 'cm_error':'user "'+i['cm_user_uoa']+'" doesn\'t have a password'}

    # Check password
    if pwd2!=user_cfg['cm_user_password2']:
       return {'cm_return':127, 'cm_error':'authentication failed (passwords do not match)'}

    # Authenticated fine, return config of a user (to know which groups belong to, etc)
    return {'cm_return':0, 'cm_user_cfg':user_cfg, 'cm_user_password2':pwd2, 'cm_user_uid':user_uid, 
                           'cm_user_uoa':user_uoa, 'cm_user_alias':user_alias, 'cm_user_path':user_path}

# ============================================================================
def auth_user_and_ini_cm(i):

    """
    Check if need to auth user, try to auth and set cM vars

    Input:  {
              cm_user_uoa       - user UOA
              cm_user_password  - user password in plain text 
                                  (not secure, usually used for web form authentication)
              cm_user_password1 - user password in SHA1
                                  (more secure, can be stored in environment, session or cookies)
                                  cM authenicates by taking SHA1(cm_user_password1)...
              cm_user_password2 - only should be set here by web environment!
            }   

    Output: {
              cm_return            - return code = 0 if authenticated
              (cm_error)           - error text, if cm_return > 0

               If needed to auth or default user:
              cm_user_cfg          - user config if authenticated (to check groups, etc)
              cm_user_password1    - user password in SHA1
              cm_user_uid          - user UID
              cm_user_uoa          - user UOA
              cm_user_alias        - user alias
              cm_user_path         - user path
              cm_user_status       - 'no_user', 'default', 'logged_in' or 'failed'
            }
    """

    rx={'cm_return':0}

    # First set default user
    ini['web_auth']['cm_user_status']='no_user'
    if (i.get('cm_web','')!='yes' or ini['dcfg'].get('force_web_login','')!='yes') and \
        ini['dcfg'].get('cm_default_user_uoa','')!='':

       # Load default user using admin privileges
       user=ini['dcfg']['cm_default_user_uoa']
       r=access({'cm_run_module_uoa':ini['dcfg']['cm_user_module_uoa'],
                 'cm_action':'load',
                 'cm_data_uoa':user,
                 'cm_admin':'yes'})
       if r['cm_return']>0: 
          if r['cm_return']==16: r={'cm_return':16, 'cm_error':'can\'t load default user "'+user+' - please, run configure.sh"'}
          return r

       rx={'cm_user_cfg':r['cm_data_obj']['cfg'],
           'cm_user_uid':r['cm_uid'],
           'cm_user_uoa':r['cm_uoa'],
           'cm_user_alias':r['cm_alias'],
           'cm_user_path':r['cm_path'],
           'cm_user_status':'default'}

       ini['web_auth']=copy.deepcopy(rx)
       rx['cm_return']=0

       if r['cm_uoa'] in ini['dcfg'].get('admin_users',[]) or \
          r['cm_uid'] in ini['dcfg'].get('admin_users',[]):
          ini['web_auth']['cm_admin']='yes'

    # Try to auth if needed (check that not default user if not strict login)
    if i.get('cm_web','')=='yes':
       if (ini['dcfg'].get('force_web_login','')=='yes' or \
           ini['dcfg'].get('allow_web_user_auth','')=='yes'):

          ini['web_auth']['cm_admin']='no'

          if ini['dcfg'].get('force_web_login','')!='yes' and \
              (i.get('cm_user_uoa','')!='' and ini['dcfg'].get('cm_default_user_uoa','')!='' and \
              (i['cm_user_uoa']==ini['web_auth']['cm_user_uid'] or i['cm_user_uoa']==ini['web_auth']['cm_user_alias'])):
             ini['web_auth']['cm_error']='user already logged in'

             if i['cm_user_uoa'] in ini['dcfg'].get('admin_users',[]):
                ini['web_auth']['cm_admin']='yes'

          else:
             rx=auth_user(i)
             if rx['cm_return']>0: 
#                if ini['dcfg'].get('force_web_login','')=='yes':
                ini['web_auth']['cm_user_status']='failed'
                ini['web_auth']['cm_error']=rx['cm_error']
             else:
                ini['web_auth'].update(rx)
                ini['web_auth']['cm_user_status']='logged_in'

                if rx['cm_user_uoa'] in ini['dcfg'].get('admin_users',[]) or \
                   rx['cm_user_uid'] in ini['dcfg'].get('admin_users',[]):
                   ini['web_auth']['cm_admin']='yes'

          rx['cm_return']=0 # web front-end will be dealing with failed logins ...

    # Check if failed
    if ini['web_auth'].get('cm_user_status','')=='failed' and \
       ini['dcfg'].get('cm_default_user_uoa','')!='':

       # Load default user using admin privileges
       user=ini['dcfg']['cm_default_user_uoa']
       r=access({'cm_run_module_uoa':ini['dcfg']['cm_user_module_uoa'],
                 'cm_action':'load',
                 'cm_data_uoa':user,
                 'cm_admin':'yes'})
       if r['cm_return']>0: 
          if r['cm_return']==16: r={'cm_return':16, 'cm_error':'can\'t load default user "'+user+' - please, run configure.sh"'}
          return r

       rx={'cm_user_cfg':r['cm_data_obj']['cfg'],
           'cm_user_uid':r['cm_uid'],
           'cm_user_uoa':r['cm_uoa'],
           'cm_user_alias':r['cm_alias'],
           'cm_user_path':r['cm_path'],
           'cm_user_status':'default'}

       ini['web_auth']=copy.deepcopy(rx)
       rx['cm_return']=0

       if r['cm_uoa'] in ini['dcfg'].get('admin_users',[]) or \
          r['cm_uid'] in ini['dcfg'].get('admin_users',[]):
          ini['web_auth']['cm_admin']='yes'

    # Set default web style (either from kernel or from user)
    cm_default_web_uoa=''
    if i.get('cm_web_uoa','')!='':
       cm_default_web_uoa=i['cm_web_uoa']
    else:
       cm_default_web_uoa=''
       if ini['dcfg'].get('cm_default_web_uoa','')!='': 
          cm_default_web_uoa=ini['dcfg']['cm_default_web_uoa']

# Should not do that - we should move only styles to web.cfg in the future
#       # Get web template from user or default
#       if 'cm_user_cfg' in ini['web_auth'] and \
#          'cm_default_web_uoa' in ini['web_auth']['cm_user_cfg'] and \
#          ini['web_auth']['cm_user_cfg']['cm_default_web_uoa']!='':
#          cm_default_web_uoa=ini['web_auth']['cm_user_cfg']['cm_default_web_uoa']

    # Set and load
    if cm_default_web_uoa=='': cm_default_web_uoa=var_cm_cfg_default

    ini['web_style']['uoa']=cm_default_web_uoa

    r=cm_kernel.access({'cm_run_module_uoa':ini['dcfg']['cm_web_module_uoa'],
                        'cm_action':'load',
                        'cm_data_uoa':cm_default_web_uoa,
                        'cm_admin':'yes'})
    if r['cm_return']>0: return r

    ini['web_style']['cfg']=r['cm_data_obj']['cfg']
    ini['web_style']['path']=r['cm_path']

    return rx

# ============================================================================
def access_fe_as_string(s):
    """
    Front-end to single entry 'access' function as string (if someone finds it easier).

    Input:  string that will be converted to array (See "access" function).
    Output: See "access" function.
    """

    r=str2cm(s)
    if r['cm_return']>0: return r

    return access(r['cm_array'])

# ============================================================================
def access_fe_through_cmd(i):
    """
    FGG TBD: Currently used only in cM web server.
             Should be cleaned and updated!
             Should check that tmp files are deleted!

    Main function to access cM through a command line (useful for web server, etc).

    Input:  {
               ...                 - see "access" function from this module
               (cm_detach_console) - if 'yes', detach console (for installations or experiments)
            }
    Output: {
              cm_return - return code = 0 if successful
              cm_stdout - stdout 
              cm_stderr - stderr
            }
    """

    # Check if cM module is in the input
    if 'cm_run_module_uoa' not in i:
       return {'cm_return':1, 'cm_error':'no module specified'}
    module=i['cm_run_module_uoa']

    # Check if cM action is in the input
    if 'cm_action' not in i:
       return {'cm_return':1, 'cm_error':'no action specified for module '+i['cm_run_module_uoa']}
    action=i['cm_action']

    cm_stdout=''
    cm_stderr=''

    i1={}; i1.update(i)
    del (i1['cm_run_module_uoa'])
    del (i1['cm_action'])

    if i.get('cm_detach_console','')=='yes': 
       # Force text mode
       i1['cm_console']='txt'
       if 'cm_web' in i1: del(i1['cm_web'])

    rr={}; 
    rr['cm_stdout']=''
    rr['cm_stderr']=''

    # Save json to temporay file
    rx=cm_kernel.gen_cm_tmp_file({})
    if rx['cm_return']>0: 
       rr['cm_return']=rx['cm_return']
       rr['cm_stderr']=rx['cm_error']
       return rr
    tf=rx['cm_path']

    r=save_array_to_file_as_json({'cm_filename':tf, 'cm_array':i1})
    if r['cm_return']>0:
       rr['cm_return']=r['cm_return']
       rr['cm_stderr']=r['cm_error']
       return rr

    cmd='cm '+module+' '+action+' @'+tf
    if i.get('cm_detach_console','')=='yes':
       # Check if detached console is forbidden (allow for admin)
       if ini['dcfg'].get('forbid_detached_console','')=='yes' or \
          (i.get('cm_web','')=='yes' and \
           ini['dcfg'].get('forbid_detached_console_in_web','')=='yes' and \
           i.get('cm_user_uoa','')!='' and i.get('cm_user_uoa','') not in ini['dcfg'].get('admin_users',[])):
          rr['cm_return']=0
          rr['cm_stdout']='<B>cM error:</B> Detaching console is explicitly forbidden in the kernel'
          return rr

       # Load OS to get cmd for detached console
       os_uoa=ini['dcfg'].get('cm_default_os_uoa','')

       if os_uoa=='':
          rr['cm_return']=1
          rr['cm_stderr']='detached console is requested but default os uoa is not in kernel'
          return rr

       ii={'cm_run_module_uoa':ini['dcfg']['cm_os_module_uoa'],
           'cm_data_uoa':os_uoa,
           'cm_action':'load'}
       rx=access(ii)
       if rx['cm_return']>0: 
          rr['cm_return']=rx['cm_return']
          rr['cm_stderr']=rx['cm_error']
          return rr
       dos=rx['cm_data_obj']['cfg']

       dcmd=dos.get('cmd_for_detached_console','')
       if dcmd=='':
          rr['cm_return']=1
          rr['cm_stderr']='detached console is requested but cmd is not defined in OS'
          return rr

       dcmd=dcmd.replace(convert_str_to_special('cmd'), cmd)
       if dos.get('use_create_new_console_flag','')=='yes':
          process=subprocess.Popen(dcmd, stdin=None, stdout=None, stderr=None, shell=True, close_fds=True, creationflags=subprocess.CREATE_NEW_CONSOLE)
       else:
          # Will need to do the forking
          try:
             pid=os.fork()
          except OSError, e:
             rr['cm_return']=1
             rr['cm_stderr']='forking detached console failed!'
             return rr

          if pid==0:
             os.setsid()

             pid=os.fork()
             if pid!=0: os._exit(0)

             try:
                 maxfd=os.sysconf("SC_OPEN_MAX")
             except (AttributeError, ValueError):
                 maxfd=1024

             for fd in range(maxfd):
                 try:
                    os.close(fd)
                 except OSError:
                    pass

             os.open('/dev/null', os.O_RDWR)
             os.dup2(0, 1)
             os.dup2(0, 2)

             # Normally child process
             process=os.system(dcmd)
             os._exit(0)

       if i.get('cm_console')=='web':
          cm_stdout=dos.get('html_for_detached_console','')
          cm_stderr=''
       else:
          cm_stdout='Console was detached!'
          cm_stderr=''
    else:
       process=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
       cm_stdout,cm_stderr=process.communicate()

    rr['cm_return']=0
    rr['cm_stdout']=cm_stdout
    rr['cm_stderr']=cm_stderr

    return rr

# ============================================================================
def access_fe_from_cmd(a):
    """
    Front-end to single entry 'access' function from command line (bash, windows shell, etc)
    If cm_console=='', it will be set to 'txt'
    If cm_user_uoa and cm_user_password1 are not in input but in environment, they will be also used

    Input:  cmd list that will be converted to array (See "access" function)
    Output: See "access" function
    """
    r=array2cm(a)
    if r['cm_return']>0: 
       con=r['cm_console']
       if con=='': con='txt'
       prepare_final_out(r, {'cm_console':con, 'cm_output':r.get('cm_output','')})
       return r

    i=r['cm_array']

    # Check console
    if 'cm_console' not in i or i['cm_console']=='': i.update({'cm_console':'txt'})

#    # If no user/pass, try from environment
#    if 'cm_user_uoa' not in i or i['cm_user_uoa']=='':
#       if env_cm_user_uoa in os.environ.keys():
#          i['cm_user_uoa']=os.environ[env_cm_user_uoa].strip()
#
#    if 'cm_user_password1' not in i or i['cm_user_password1']=='':
#       if env_cm_user_password1 in os.environ.keys():
#          i['cm_user_password1']=os.environ[env_cm_user_password1].strip()

    return access_fe(i)

# ============================================================================
def access_fe(i):

    """
    Front-end to single entry 'access' function
    cM will be initialized if needed

    Input:  See "access" function
    Output: See "access" function
    """

    global ini

    # If error encountered during previous cM intialization, set error and quit
    if ini['init']==1:
       r={'cm_return':1, 'cm_error':'cM initialization failed', 'cm_console':i['cm_console']}
       prepare_final_out(r, {'cm_console':i.get('cm_console',''), 'cm_output':i.get('cm_output','')})
       return r

    # If cM is not yet initialized, perform initialization
    if ini['init']==-1:
       ini['init']=1 # Expect error and set to 0 if everything is fine

       r=init(i)
       if r['cm_return']>0: 
          prepare_final_out(r, {'cm_console':i.get('cm_console',''), 'cm_output':i.get('cm_output','')})
          return r

       ini['init']=0

    # Check if cM module is in the input
    if 'cm_run_module_uoa' not in i:
       r={'cm_return':1, 'cm_error':'no module specified'}
       prepare_final_out(r, {'cm_console':i.get('cm_console',''), 'cm_output':i.get('cm_output','')})
       return r

    # Check if cM action is in the input
    if 'cm_action' not in i:
       r={'cm_return':1, 'cm_error':'no action specified for module '+i['cm_run_module_uoa']}
       prepare_final_out(r, {'cm_console':i.get('cm_console',''), 'cm_output':i.get('cm_output','')})
       return r

    # Auth user if needed (do not fail here, just set vars that will be processed later)
    # If need to authenticate, forward to login page

    # Set default user
    rx=auth_user_and_ini_cm(i)
    if rx['cm_return']>0: 
       prepare_final_out(rx, {'cm_console':i.get('cm_console',''), 'cm_output':i.get('cm_output','')})
       return rx

    # Force removal of cm_user_password2 for security
    if i.get('cm_user_password2','')!='': del(i['cm_user_password2'])

    # If web environment, set web
    if i.get('cm_web')=='yes': ini['web']='yes'

    r=access(i)

    prepare_final_out(r, {'cm_console':i.get('cm_console',''), 'cm_output':i.get('cm_output','')})
    return r

# ============================================================================
def remote_access(i):

    """
    Remote access to cM

    Input:  {
              cm_remote_url             - remote URL
              (cm_user_uoa)             -
              (cm_user_password)        -
              (cm_user_password1)       -
              (remote_repo_uoa)

              (remote_pre_auth)         - if 'yes', need standard http login/password pre-authentication
              (cm_remote_user_name)     - remote username
              (cm_remote_user_password) - remote password

              (cm_web_module_uoa)       - module to run
              (cm_web_action)           - action to perform
                                          if =='download', prepare entry/file download through Internet

              (cm_save_to_file)         - if cm_web_action==download,
                                          save output to this file

              ...                       - all other request parameters
            }

    Output: {
              cm_return  - return code = 0 if successful
                                      > 0 if error
                                      < 0 if warning (rarely used at this moment)
              (cm_error) - error text, if cm_return > 0
            }
    """

    # Prepare request
    post=''

    url=i.get('cm_remote_url','')
    if url=='':
       return {'cm_return':1, 'cm_error':'cm_remote_url is not defined'}
    del(i['cm_remote_url'])

    if i.get('cm_user_uoa','')!='': 
       if post!='': post+='&'
       post+='cm_user_uoa='+i['cm_user_uoa']
       del(i['cm_user_uoa'])

    if i.get('cm_user_password','')!='': 
       if post!='': post+='&'
       post+='cm_user_password='+i['cm_user_password']
       del(i['cm_user_password'])

    if i.get('cm_user_password1','')!='': 
       if post!='': post+='&'
       post+='cm_user_password1='+i['cm_user_password1']
       del(i['cm_user_password1'])

    if i.get('remote_repo_uoa','')!='': 
       if post!='': post+='&'
       post+='cm_repo_uoa='+i['remote_repo_uoa']
       del(i['remote_repo_uoa'])

    # Check if needed remote pre-authentication
    base64string=''
    if i.get('remote_pre_auth', '')=='yes':
       del(i['remote_pre_auth'])

       username=''
       if i.get('cm_remote_user_name','')!='': 
          username=i['cm_remote_user_name']
          del(i['cm_remote_user_name'])

       password=''
       if i.get('cm_remote_user_password','')!='': 
          password=i['cm_remote_user_password']
          del(i['cm_remote_user_password'])

       base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')

    # Check if data download, not json and convert it to download request
    download=False
    if i.get('cm_web_action','')=='download' or i.get('cm_web_action','')=='show':
       download=True
       if post!='': post+='&'
       post+='cm_web_module_uoa=web&cm_web_action='+i['cm_web_action']
       del(i['cm_web_action'])
       if i.get('cm_web_module_uoa','')!='': del(i['cm_web_module_uoa'])
       if i.get('cm_console','')!='': del(i['cm_console'])
    else:
       if post!='': post+='&'
       post+='cm_console=json'

    # Prepare array to transfer through Internet
    r=convert_cm_array_to_uri({'cm_array':i})
    if post!='': post+='&'
    post+='cm_json='+unicode(r['cm_string1'])

    # Prepare URL request
    request = urllib2.Request(url, post)

    if base64string!='':
       request.add_header("Authorization", "Basic %s" % base64string)   

    # Check if proxy is needed
    aproxy={}
    if env_cm_remote_repo_proxy_https in os.environ.keys():
       aproxy['https']=os.environ[env_cm_remote_repo_proxy_https].strip()
    if env_cm_remote_repo_proxy_http in os.environ.keys():
       aproxy['http']=os.environ[env_cm_remote_repo_proxy_http].strip()

    if len(aproxy)>0:
       proxy = urllib2.ProxyHandler(aproxy)
       opener = urllib2.build_opener(proxy)
       urllib2.install_opener(opener)

    # Connect
    try:
        f=urllib2.urlopen(request)
    except Exception as e:
        return {'cm_return':1, 'cm_error':'Remote access failed ('+format(e)+')'}

    # Read from Internet
    try:
        s=str(f.read())
        f.close()
    except Exception as e:
        return {'cm_return':1, 'cm_error':'Failed reading stream from remote server ('+format(e)+')'}

    request=None

    r={'cm_return':0}

    # Check if download, not json!
    if download:
       name='default_download_name.dat'

       if i.get('cm_web_filename','')!='':
          name=os.path.basename(i['cm_web_filename'])
       if i.get('cm_save_to_file','')!='': 
          # FGG: unsecure - should add checks where to write and what
          # to avoid overwriting some important data or put code somewhere
          name=i['cm_save_to_file']

       try:
          f=open(name,'wb')
          f.write(s)
          f.close()
       except Exception as e:
          return {'cm_return':1, 'cm_error':'problem saving downloaded file ('+format(e)+')'}

    else:
       try:
          r=json.loads(s)
       except:
          return {'cm_return':32, 'cm_error':'Can\'t decode JSON ('+s+')'}

    return r

# ============================================================================
def access(i):

    """
    Single entry point to access all cM functions.

    Input:  {
              Set of global cM variables:

                cm_run_module_uoa - UOA of the module to run
                cm_action         - module action (function)

                (cm_console)       - by default, do not output anything except errors
                                     if 'txt' then output as plain text
                                     if 'web' then output for web (html, image, etc)
                                     if 'json' then output as json

                cm_web            - if 'yes', function was called from web environment
                cm_web_preprocess - if 'yes', we in the preprocessing stage in the web environment,
                                    where we can still change the http headers, etc ...

                If cM Web access or fully remote access:
                cm_user_uoa       - user UOA

                                    If requires user authentication in web environment!
                                    After authentication, passwords will be removed from the array

                cm_user_password  - user password in plain text 
                                    (not secure, usually used for web form authentication)
                cm_user_password1 - user password in SHA1
                                    (more secure, can be stored in environment, session or cookies)
                                     cM authenicates by taking SHA1(cm_user_password1)...

                If access to remote repository or fully remote access:

                cm_remote_user_uoa      - remote user UOA
                cm_remote_user_password - remote user password

              Set of parameters depending on the called module and action (function)
            }

    Output: {
              cm_return  - return code = 0 if successful
                                      > 0 if error
                                      < 0 if warning (rarely used at this moment)
              (cm_error) - error text, if cm_return > 0

              Set of output parameters depending on the called module and action (function)
            }
    """

    # Check if local access (possibly with remote repository)
    if 'cm_run_module_uoa' not in i: return {'cm_return':1, 'cm_error':'"cm_run_module_uoa" is not set'}
    if 'cm_action' not in i: return {'cm_return':1, 'cm_error':'"cm_action" is not set'}

    # Check again detached console (if disabled and we use php)
    if i.get('cm_detach_console','')=='yes' and ini['dcfg'].get('forbid_detached_console','')=='yes':
       r={'cm_return':1, 'cm_error':'Detaching console is explicitly forbidden in the kernel'}
       prepare_final_out(r, {'cm_console':i.get('cm_console',''), 'cm_output':i.get('cm_output','')})
       return r

    # Expand CID to cm_repo_uoa, cm_module_uoa, cm_data_uoa
    if 'cid' in i and i['cid']!='':
       rx=cm_kernel.cid2array(i)
       if rx['cm_return']>0: return rx

       i.update(rx['cm_array'])
       del(i['cid'])

    # Check cm_module_uoa vs cm_run_module_uoa
    if ('cm_module_uoa' not in i or i['cm_module_uoa']=='') and \
       'cm_run_module_uoa' in i and i['cm_run_module_uoa']!='':
       i['cm_module_uoa']=i['cm_run_module_uoa']

    # Check remote repository
    if (i.get('cm_repo_uoa','')!=''):
       cm_repo_uoa=i['cm_repo_uoa']

       r=cm_kernel.access({'cm_run_module_uoa':ini['dcfg']['cm_repo_module_uoa'],
                           'cm_action':'find_path_to_repository',
                           'find_repo_uoa':cm_repo_uoa})
       if r['cm_return']>0: return r

       d=r['cm_data_obj']['cfg']
       repo_uoa=r['cm_uoa']
       repo_uid=r['cm_uid']

       if 'path_type' in d and d['path_type']=='remote_url':
          cur_console=''
          if 'cm_console' in i and i['cm_console']!='': cur_console=i['cm_console']

          # Prepare remote request
          if i.get('cm_remote_url','')=='': i['cm_remote_url']=d['url']

          if 'cm_run_module_uoa' in i and i['cm_run_module_uoa']!='':
             i['cm_web_module_uoa']=i['cm_run_module_uoa']
             del(i['cm_run_module_uoa'])
          elif 'cm_module_uoa' in i and i['cm_module_uoa']!='':
             i['cm_web_module_uoa']=i['cm_module_uoa']
             del(i['cm_module_uoa'])
          if 'cm_action' in i: 
             i['cm_web_action']=i['cm_action']
             del(i['cm_action'])
          if 'cm_repo_uoa' in i: del(i['cm_repo_uoa'])

          i['cm_console']='json'

          username=''
          password=''
          password1=''

          # Check if needed remote authentication
          if d.get('remote_auth', '')=='yes':
             x1='\nRemote repository "'+repo_uoa+'" requires authentication!'
             x2=True

             if d.get('remote_auth_as_user', '')=='yes':
                username=ini['web_auth'].get('cm_user_cfg',{}).get('cm_username','')

             if username=='' and d.get('remote_auth_user', '')!='':
                username=d['remote_auth_user']

             if username=='' and ini['dcfg'].get('cm_default_user_uoa', '')!='':
                username=ini['dcfg']['cm_default_user_uoa']

             if env_cm_user_uoa in os.environ.keys() and os.environ[env_cm_user_uoa]!='':
                username=os.environ[env_cm_user_uoa].strip()

             if i.get('cm_remote_user_uoa','')!='':
                username=i['cm_remote_user_uoa']

             if username=='':
                if i.get('cm_web','')=='yes': 
                   return {'cm_return':1, 'cm_error':'remote authentication is required but "cm_remote_user_uoa" is not set!'}
                else:
                   if cm_repo_uoa in ini['remote_repo'] and ini['remote_repo'][cm_repo_uoa].get('username','')!='':
                      username=ini['remote_repo'][cm_repo_uoa]['username']
                   else:
                      if x2:
                         print_for_con(x1)
                         x2=False
                      username=raw_input('Enter username (or Enter to terminate operation): ')
                      if username=='': return {'cm_return':1, 'cm_error':'user terminated operation'}
                      if cm_repo_uoa not in ini['remote_repo']: ini['remote_repo'][cm_repo_uoa]={}
                      ini['remote_repo'][cm_repo_uoa]['username']=username

             if i.get('cm_remote_user_password','')!='':
                password=i['cm_remote_user_password']
             elif env_cm_user_password in os.environ.keys() and os.environ[env_cm_user_password]!='':
                password=os.environ[env_cm_user_password].strip()
             elif env_cm_user_password1 in os.environ.keys() and os.environ[env_cm_user_password1]!='':
                password1=os.environ[env_cm_user_password1].strip()
             else:
                if i.get('cm_web','')=='yes': return {'cm_return':1, 'cm_error':'remote authentication is required but "cm_remote_user_password" is not set!'}
                else:
                   if cm_repo_uoa in ini['remote_repo'] and ini['remote_repo'][cm_repo_uoa].get('password','')!='':
                      password=ini['remote_repo'][cm_repo_uoa]['password']
                   else:
                      if x2:
                         print_for_con(x1)
                         x2=False
                      password=getpass.getpass('Enter password for '+str(username)+' (or Enter to terminate operation): ')
                      password=password.strip()
                      if password=='': return {'cm_return':1, 'cm_error':'user terminated operation'}
                      if cm_repo_uoa not in ini['remote_repo']: ini['remote_repo'][cm_repo_uoa]={}
                      ini['remote_repo'][cm_repo_uoa]['password']=password

          rep=''
          if i.get('remote_repo_uoa','')!='':
             rep=i['remote_repo_uoa']
          elif d.get('remote_repo_uoa','')!='':
             rep=d['remote_repo_uoa']
          if rep!='': i['remote_repo_uoa']=rep

          i['cm_user_uoa']=username

          if password!='': 
             i['cm_user_password']=password
          if password1!='': 
             i['cm_user_password1']=password1

          # Check if needed remote pre-authentication
          if d.get('remote_pre_auth', '')=='yes':
             i['remote_pre_auth']='yes'

             x1='Remote repository requires http pre-authentication!'
             x2=True

             if username=='':
                if d.get('remote_auth_as_user', '')=='yes':
                   username=ini['web_auth'].get('cm_user_cfg',{}).get('cm_username','')
                elif d.get('remote_auth_user', '')!='':
                   username=d['remote_auth_user']
                elif i.get('cm_remote_user_uoa','')!='':
                   username=i['cm_remote_user_uoa']
                else:
                   if i.get('cm_web','')=='yes': 
                      return {'cm_return':1, 'cm_error':'remote authentication is required but "cm_remote_user_uoa" is not set!'}
                   else:
                      if x2:
                         print_for_con(x1)
                         x2=False
                      username=raw_input('Enter username (or Enter to terminate operation): ')
                      if username=='': return {'cm_return':1, 'cm_error':'user terminated operation'}

             if password=='':
                if i.get('cm_remote_user_password','')!='':
                   password=i['cm_remote_user_password']
                else:
                   if i.get('cm_web','')=='yes': return {'cm_return':1, 'cm_error':'remote authentication is required but "cm_remote_password" is not set!'}
                   else:
                      if x2:
                         print_for_con(x1)
                         x2=False
                      password=getpass.getpass('Enter password for '+str(username)+' (or Enter to terminate operation): ')
                      if password=='': return {'cm_return':1, 'cm_error':'user terminated operation'}

             i['cm_remote_user_name']=username
             i['cm_remote_user_password']=password

          # Send remote request
          r=remote_access(i)

          # restore console
          if cur_console!='': i['cm_console']=cur_console

          return r

    # Find function in original module and if not found, try common functions in "core" module
    for q in range(0,2):

        if q==0:
           module=i['cm_run_module_uoa']
           actions='cm_actions'
        else:
           module='core' # hardwired here
           actions='cm_common_actions'

        # Load module
        ii={}
        ii['cm_module_uoa']=module
        if 'cm_web' in i: ii['cm_web']=i['cm_web']
        r=load_module(ii)
        if r['cm_return']>0: 
           return r
        c=r['cm_code']
        e=r.get('external_module','')

        # Check if external or action exists in this module
        if e=='yes':
           a=getattr(c, i['cm_action'])
           return a(i)

        elif actions in c.ini['cfg']:
           r=find_index_in_array({
                                  'cm_array':c.ini['cfg'][actions],
                                  'cm_index':i['cm_action']
                                 })

           if r['cm_return']==0:
              #Check if skip web pre-processing
              if i.get('cm_web_preprocess','')=='yes' \
                 and r['cm_array'].get('cm_web_preprocess','')!='yes':
                 return {'cm_return':0, 'cm_web_postprocess':'yes'}

              f=r['cm_array']['func']; a=getattr(c, f)
              ii={}
              if q!=0: ii['cm_common_func']='yes'
              if 'params_default' in r['cm_array']:
                 merge_arrays({'cm_array':ii, 'cm_array1':r['cm_array']['params_default']})
              merge_arrays({'cm_array':ii, 'cm_array1':i})

              if 'cm_web' in i: ii['cm_web']=i['cm_web']

              return a(ii)

    return {'cm_return':1, 'cm_error':'action "'+i['cm_action']+'" not found in module '+i['cm_run_module_uoa']}

# ============================================================================
if __name__ == "__main__":

   '''
   cM command line wrapper
   cm <cm object> <action> parameters 

   '''

   r=access_fe_from_cmd(sys.argv[1:])
   exit(r['cm_return'])