module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

# Should always be here
ini={}
cm_kernel=None

# Local settings
import os
import time
import urllib
import urllib2
import json
import threading
import time
import re
import copy

# ============================================================================
def init(i):
    return {'cm_return':0}

# ============================================================================
def server(i):

    """
    Index Server

    Input:  {
              (cm_index_uoa)  - uoa with description of indexer
              (os_uoa)        - uoa with OS description (needed for script extension, for example)
              (cm_index_port) - web port
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    cm_kernel.print_for_con("For now we can only start server indefinitely")
    cm_kernel.print_for_con("but we should add a nice start/stop/resume support one day")

    # Check (default) index configuration
    index_uoa='default'
    if 'cm_index_uoa' in i and i['cm_index_uoa']!='': index_uoa=i['cm_index_uoa']
    elif 'cm_default_index_uoa' in cm_kernel.ini['dcfg'] and cm_kernel.ini['dcfg']['cm_default_index_uoa']!='':
       index_uoa=cm_kernel.ini['dcfg']['cm_default_index_uoa']

    if index_uoa=='':
       return {'cm_return':1, 'cm_error':'"cm_index_uoa" is not defined'}

    # Load index
    r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                        'cm_action':'load',
                        'cm_data_uoa':index_uoa})
    if r['cm_return']>0: return r
    index_cfg=r['cm_data_obj']['cfg']

    port=index_cfg.get('port','9200')
    support_port=index_cfg.get('support_port','9300')
    host=index_cfg['host']+':'+port

    # Check various variables
    if 'code_uoa' in index_cfg and index_cfg['code_uoa']!='':
       # We need then OS to load properly environment script

       # Check (default) OS configuration
       os_uoa=''
       if 'os_uoa' in i and i['os_uoa']!='': os_uoa=i['os_uoa']
       elif 'cm_default_os_uoa' in cm_kernel.ini['dcfg'] and cm_kernel.ini['dcfg']['cm_default_os_uoa']!='':
          os_uoa=cm_kernel.ini['dcfg']['cm_default_os_uoa']

       if os_uoa=='':
          return {'cm_return':1, 'cm_error':'"os_uoa" is not defined'}

       # Get environment
       r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['code'],
                           'cm_action':'get_env',
                           'cm_data_uoa':index_cfg['code_uoa'],
                           'os_uoa':os_uoa})
       if r['cm_return']>0: return r
       code_env=r['cm_string']

       # Load OS
       ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['os'],
           'cm_action':'load',
           'cm_data_uoa':os_uoa}
       r=cm_kernel.access(ii)
       if r['cm_return']>0: return r

       os_cfg=r['cm_data_obj']['cfg']

       if 'name' not in os_cfg:
          return {'cm_return':1, 'cm_error':'"name" is not defined in os configuration'}

       os_name=os_cfg['name']

       if os_name not in index_cfg['run_cmd']:
          return {'cm_return':1, 'cm_error':'"name" is not defined in os configuration'}

       run_cmd_t=index_cfg['run_cmd'][os_name]

       run_cmd=run_cmd_t.replace(cm_kernel.convert_str_to_special('code_env'), code_env)
       run_cmd=run_cmd.replace(cm_kernel.convert_str_to_special('index_port'), port)
       run_cmd=run_cmd.replace(cm_kernel.convert_str_to_special('index_support_port'), support_port)

       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Starting cM index server: '+host+' ...')
       cm_kernel.print_for_con('')
       cm_kernel.print_for_con(run_cmd)

       r=os.system(run_cmd)

       if r>0:
          return {'cm_return':1, 'cm_error':'problem starting cM index server (return code='+str(r)+')'}

#       time.sleep(6) # Force sleep to wait while server starts, then try to add 1 entry (in case of flush)
#
#       # Update 1 index
#       ii = {'cm_module_uoa':ini['cm_module_uid'], \
#             'cm_data_uoa':'default', \
#             'mode': 'update'}
#       r=index(ii)
#       if r['cm_return']>0:
#          cm_kernel.print_for_con('cM error: '+r['cm_error'])
#       elif r['cm_return']<0:
#          cm_kernel.print_for_con('cM warning: '+r['cm_warning'])



    return {'cm_return':0}

# ============================================================================
def flush(i):

    """
    Flush index

    Input:  {
              (cm_index_uoa)  - uoa with description of indexer
              (os_uoa)        - uoa with OS description (needed for script extension, for example)
              (cm_index_port) - web port
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    if cm_kernel.ini['dcfg'].get('use_indexing','')!='yes':
       return {'cm_return':1, 'cm_error':'indexing is OFF in the default kernel configuration'}

    cm_kernel.print_for_con("Flushing index ...")
    cm_kernel.print_for_con("")

    # Check (default) index configuration
    index_uoa=''
    if 'cm_index_uoa' in i and i['cm_index_uoa']!='': index_uoa=i['cm_index_uoa']
    elif 'cm_default_index_uoa' in cm_kernel.ini['dcfg'] and cm_kernel.ini['dcfg']['cm_default_index_uoa']!='':
       index_uoa=cm_kernel.ini['dcfg']['cm_default_index_uoa']

    if index_uoa=='':
       return {'cm_return':1, 'cm_error':'"cm_index_uoa" is not defined'}

    # Load index
    r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                        'cm_action':'load',
                        'cm_data_uoa':index_uoa})
    if r['cm_return']>0: return r
    index_cfg=r['cm_data_obj']['cfg']

    host=index_cfg['host']+':'+index_cfg.get('port','9200')

    if cm_kernel.ini['dcfg'].get('use_curl_for_indexing','')=='yes':
       # First delete index
       cmd='curl -XDELETE '+host
       os.system(cmd)
    else:
       url=host+'/_flush'

       request = urllib2.Request(url)

       s=''
       try:
          f=urllib2.urlopen(request, None)
       except urllib2.URLError, e:
          return {'cm_return':1, 'cm_error':'problem accessing index server ('+format(e)+')'}
       else:
          s=str(f.read())
          try:
             j=json.loads(s)
          except:
             cm_kernel.print_for_con('cM error: can\'t parse output from index server ('+s+')')
             return {'cm_return':1, 'cm_error':'can\'t parse output from index server ('+s+')'}
          if not j["ok"]:
             cm_kernel.print_for_con('cM error: flushing was not successful ('+s+')')
             return {'cm_return':1, 'cm_error':'flushing was not successful ('+s+')'}

          f.close()

       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Output from the index server:')
       cm_kernel.print_for_con(s)

    # Index at least yourself (module)

    ii={'cm_module_uoa':ini['cfg']['cm_modules']['cm-module'],
        'cm_data_uoa':ini['cm_module_uid'],
        'mode': 'add',
        'cm_console':''}
    return index(ii)

# ============================================================================
def test(i):

    """
    Test index server

    Input:  {
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    if cm_kernel.ini['dcfg'].get('use_indexing','')!='yes':
       return {'cm_return':1, 'cm_error':'indexing is OFF in the default kernel configuration'}

    cm_kernel.print_for_con("Testing index server ...")
    cm_kernel.print_for_con("")

    # Check (default) index configuration
    index_uoa=''
    if 'cm_index_uoa' in i and i['cm_index_uoa']!='': index_uoa=i['cm_index_uoa']
    elif 'cm_default_index_uoa' in cm_kernel.ini['dcfg'] and cm_kernel.ini['dcfg']['cm_default_index_uoa']!='':
       index_uoa=cm_kernel.ini['dcfg']['cm_default_index_uoa']

    if index_uoa=='':
       return {'cm_return':1, 'cm_error':'"cm_index_uoa" is not defined'}

    # Load index
    r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                        'cm_action':'load',
                        'cm_data_uoa':index_uoa})
    if r['cm_return']>0: return r
    index_cfg=r['cm_data_obj']['cfg']

    host=index_cfg['host']+':'+index_cfg.get('port','9200')

    if cm_kernel.ini['dcfg'].get('use_curl_for_indexing','')=='yes':
       # First delete index
       cmd='curl '+host
       os.system(cmd)
    else:
       request = urllib2.Request(host)

       s=''
       try:
          f=urllib2.urlopen(request, None)
       except urllib2.URLError, e:
          return {'cm_return':1, 'cm_error':'problem accessing index server ('+format(e)+')'}
       else:
          s=str(f.read())
          try:
             j=json.loads(s)
          except:
             cm_kernel.print_for_con('cM error: can\'t parse output from index server ('+s+')')
             return {'cm_return':1, 'cm_error':'can\'t parse output from index server ('+s+')'}
          if not j["ok"]:
             cm_kernel.print_for_con('cM error: flushing was not successful ('+s+')')
             return {'cm_return':1, 'cm_error':'flushing was not successful ('+s+')'}

          f.close()

       cm_kernel.print_for_con('')
       cm_kernel.print_for_con('Output from the index server:')
       cm_kernel.print_for_con(s)

    return {'cm_return':0}

# ============================================================================
def all(i):

    """
    ReIndex all repos

    Input:  {
              (index_repo_uoa)   - repo uoa to prune reindexing
              (index_module_uoa) - module uoa to prune reindexing
              (index_data_uoa)   - data uoa to prune reindexing (can include pattern * and ?)
              (delay)            - delay after indexing individual entry
              (sequential)       - if 'yes' perform sequential index (very slow, but reports errors)
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    # Check for pattern
    idu=i.get('index_data_uoa','')

    pattern=""
    if '*' in idu or '?' in idu: 
       pattern1=idu.replace('*','.*')
       pattern1=pattern1.replace('?','.?')
       pattern='.*'+pattern1+'$'
       idu=''

    if cm_kernel.ini['dcfg'].get('use_indexing','')!='yes':
       return {'cm_return':1, 'cm_error':'indexing is OFF in the default kernel configuration'}

    indexes=0

    # Check delay
    d=0.05
    if 'delay' in i and i['delay']!='': d=float(i['delay'])

    # Start timer
    t0=time.time()

    cm_kernel.print_for_con('Reindexing all entries in all repositories ...')

    cm_kernel.print_for_con('')

    # Search through repositories
    if 'index_repo_uoa' in i:
       # Load repo
       r2=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-repo'],
                            'cm_action':'load',
                            'cm_data_uoa':i['index_repo_uoa']})
       if r2['cm_return']>0: return r2
       rlist=[{'cm_uoa':r2['cm_uoa'], 'cm_uid':r2['cm_uid']}]
    else:
       r2=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-repo'],
                            'cm_action':'list'})
       if r2['cm_return']>0: return r2
       rlist=r2['cm_mixed']

    for repo1 in rlist:
        # Check print
        repo_uoa=repo1['cm_uoa']
        repo_uid=repo1['cm_uid']

        # Load repo to exclude remote
        r2=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-repo'],
                             'cm_action':'load',
                             'cm_data_uoa':repo_uid})
        if r2['cm_return']>0: return r2
        repo_cfg=r2['cm_data_obj']['cfg']
        if repo_cfg.get('path_type','')=='remote_url':
#               cm_kernel.print_for_con('')
#               cm_kernel.print_for_con('Skipping remote repo '+repo_uoa+' ...')
           pass
        else:
           # Search through modules
           mlist=[]
           if 'index_module_uoa' in i:
              # Load module
              r1=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-module'],
                                   'cm_action':'load',
                                   'cm_data_uoa':i['index_module_uoa']})
              if r1['cm_return']>0: return r1
              mlist=[{'cm_uoa':r1['cm_uoa'], 'cm_uid':r1['cm_uid']}]
           else:
              r1=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'],
                                   'cm_action':'list',
                                   'cm_repo_uoa':repo_uoa,
                                   'cm_only_available_modules':'yes'})
              if r1['cm_return']>0: return r1
              mlist=r1['cm_mixed']

           for module1 in mlist:
               module_uoa=module1['cm_uoa']
               module_uid=module1['cm_uid']

               # List data
               if idu in i:
                  # Load data
                  r3=cm_kernel.access({'cm_run_module_uoa':module_uoa,
                                       'cm_action':'load',
                                       'cm_repo_uoa':repo_uoa,
                                       'cm_data_uoa':idu})
                  if r3['cm_return']>0: return r3
                  dlist=[{'cm_uoa':r3['cm_uoa'], 'cm_uid':r3['cm_uid']}]
               else:
                  r3=cm_kernel.access({'cm_run_module_uoa':module_uoa,
                                       'cm_repo_uoa':repo_uoa,
                                       'cm_action':'list'})
                  if r3['cm_return']>0: return r3
                  dlist=r3['cm_mixed']

               for data1 in dlist:
                   data_uoa=data1['cm_uoa']
                   data_uid=data1['cm_uid']
                   data_daa=data1.get('cm_display_as_alias','')

                   process=True

                   # Check for pattern
                   if not re.match(pattern,data_uoa) and not re.match(pattern,data_daa):
                      process=False 

                   if process:
                      cm_kernel.print_for_con('Indexing '+repo_uoa+':'+module_uoa+':'+data_uoa+' entry (total entries='+str(indexes)+', elapsed time='+str("%.1f" % (time.time()-t0))+' sec.)')
                      indexes+=1

                      # Update entry
#                      cm_kernel.print_for_con('  Started updating entry, elapsed time='+str("%.1f" % (time.time()-t0))+' sec.)')
#                      ii={'cm_run_module_uoa':module_uoa,
#                          'cm_action':'update',
#                          'cm_repo_uoa':repo_uoa,
#                          'cm_data_uoa':data_uoa,
#                          'cm_skip_update_info':'yes'}
#                      r=cm_kernel.access(ii)
#                      if r['cm_return']>0: return r
#                      cm_kernel.print_for_con('  Stopped updating entry, elapsed time='+str("%.1f" % (time.time()-t0))+' sec.)')

#                      cm_kernel.print_for_con('  Started indexing entry, elapsed time='+str("%.1f" % (time.time()-t0))+' sec.)')
                      # Add index
                      ii = {'cm_repo_uoa':repo_uoa, \
                            'cm_repo_uid':repo_uid, \
                            'cm_module_uoa':module_uoa, \
                            'cm_module_uid':module_uid, \
                            'cm_data_uoa':data_uoa, \
                            'cm_data_uid':data_uid, \
                            'mode': 'add'}
                      if 'sequential' in i: ii['sequential']=i['sequential']
                      r=index(ii)
                      if r['cm_return']>0:
                         cm_kernel.print_for_con('cM error: '+r['cm_error'])
                      elif r['cm_return']<0:
                         cm_kernel.print_for_con('cM warning: '+r['cm_warning'])
#                      cm_kernel.print_for_con('  Stopped indexing entry, elapsed time='+str("%.1f" % (time.time()-t0))+' sec.)')

                      time.sleep(d)

    return {'cm_return':0}

# ============================================================================
def index(i):

    """
    Add or delete index

    Input:  {
              cm_module_uoa       - module UOA
              (cm_module_uid)
              cm_data_uoa         - data UOA
              (cm_data_uid)        
              (cm_repo_uoa)       - repo UOA
              (cm_repo_uid)
              mode                - 'delete' | 'add'
              (cm_index_uoa)      - index configuration
              (sequential)        - if 'yes' perform sequential index (very slow, but reports errors)
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    if cm_kernel.ini['dcfg'].get('use_indexing','')!='yes':
       return {'cm_return':1, 'cm_error':'indexing is OFF in the default kernel configuration'}

    # Check (default) index configuration
    index_uoa=''
    if 'cm_index_uoa' in i and i['cm_index_uoa']!='': index_uoa=i['cm_index_uoa']
    elif 'cm_default_index_uoa' in cm_kernel.ini['dcfg'] and cm_kernel.ini['dcfg']['cm_default_index_uoa']!='':
       index_uoa=cm_kernel.ini['dcfg']['cm_default_index_uoa']

    if index_uoa=='':
       return {'cm_return':1, 'cm_error':'"cm_index_uoa" is not defined'}

    # Load index
    r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                        'cm_action':'load',
                        'cm_data_uoa':index_uoa})
    if r['cm_return']>0: return r
    index_cfg=r['cm_data_obj']['cfg']

    host=index_cfg['host']+':'+index_cfg.get('port','9200')

    # Load data if incomplete UOA/UID
    data_uoa=i.get('cm_data_uoa','')
    data_uid=i.get('cm_data_uid','')

    repo_uoa=i.get('cm_repo_uoa','')
    repo_uid=i.get('cm_repo_uid','')

    module_uoa=i.get('cm_module_uoa','')
    module_uid=i.get('cm_module_uid','')

    data={}
    if i['mode']=='add' or data_uid=='' or repo_uid=='' or module_uid=='':
       ii={'cm_run_module_uoa':i['cm_module_uoa'],
           'cm_data_uoa':i['cm_data_uoa'],
           'cm_action':'load'}
       if 'cm_repo_uoa' in i: ii.update({'cm_repo_uoa':i['cm_repo_uoa']})
       rx=cm_kernel.access(ii)
       if rx['cm_return']>0: return rx

       data=rx['cm_data_obj']['cfg']

       data_uoa=rx['cm_uoa']
       data_uid=rx['cm_uid']

       repo_uoa=rx['cm_repo_uoa']
       repo_uid=rx['cm_repo_uid']

       module_uoa=rx['cm_module_uoa']
       module_uid=rx['cm_module_uid']

    index={}
    jindex=''
    tdata=copy.deepcopy(data) # so if we remove keys from index, other code is not affected
    if i['mode']=='add':
       # We force removal of not-needed keys to avoid saturations of repository

       kd=ini['cfg'].get('remove_keys_from_data_if_exceed_size',[])
       for q in kd:
           if q in tdata: 
              del(tdata[q])

       # Flatten array
       ri=cm_kernel.flatten_array({'cm_array': tdata})
       if ri['cm_return']>0: return ri
       flat_data=ri['cm_array']

       # Prepare index
       index = {'cm_module_uoa':module_uoa, \
                'cm_module_uid':module_uid, \
                'cm_data_uoa':data_uoa, \
                'cm_data_uid':data_uid, \
                'cfg': tdata,
                'flat_cfg':flat_data}

       if repo_uoa!='': index['cm_repo_uoa']=repo_uoa
       if repo_uid!='': index['cm_repo_uid']=repo_uid

       jindex = json.dumps(index)
       if 'json_size_limit' in index_cfg and len(jindex)>long(index_cfg['json_size_limit']):
          return {'cm_return':-1, 'cm_warning': 'json size exceeded limit ('+str(len(jindex))+')'}

#       ready=False
#
#       kd=ini['cfg'].get('remove_keys_from_data_if_exceed_size',[])
#       ikd=len(kd)
#       ik=0
#       while not ready:
#          # Flatten array
#          ri=cm_kernel.flatten_array({'cm_array': tdata})
#          if ri['cm_return']>0: return ri
#          flat_data=ri['cm_array']
#
#          # Prepare index
#          index = {'cm_module_uoa':module_uoa, \
#                   'cm_module_uid':module_uid, \
#                   'cm_data_uoa':data_uoa, \
#                   'cm_data_uid':data_uid, \
#                   'cfg': tdata,
#                   'flat_cfg':flat_data}
#
#          if repo_uoa!='': index['cm_repo_uoa']=repo_uoa
#          if repo_uid!='': index['cm_repo_uid']=repo_uid
#
#          jindex = json.dumps(index)
#          if 'json_size_limit' in index_cfg and len(jindex)>long(index_cfg['json_size_limit']):
#             if ik>=ikd:
#                return {'cm_return':-1, 'cm_warning': 'json size exceeded limit ('+str(len(jindex))+')'}
#
#             q=kd[ik]
#             if q in tdata: del(tdata[q])
#             ik+=1
#          else:
#             ready=True

    # Prepare URL
    url=host+'/'+module_uid+'/'+data_uid+'/1'

    r={'cm_return':0}

    if cm_kernel.ini['dcfg'].get('use_curl_for_indexing','')=='yes':
       # Generate output file
       rx=cm_kernel.gen_cm_tmp_file({})
       if rx['cm_return']>0: return rx
       tfo1=rx['cm_path']

       # Generate output file
       rx=cm_kernel.gen_cm_tmp_file({})
       if rx['cm_return']>0: return rx
       tfo2=rx['cm_path']

       # First delete index
       cmd='curl -XDELETE '+url+' -s -o '+tfo1
       os.system(cmd)
       # Ignore output
       if os.path.isfile(tfo1): os.remove(tfo1)

       # Second, if mode=='add', add index
       if i.get('mode','')=='add':
          # Save json to temporay file
          rx=cm_kernel.gen_cm_tmp_file({})
          if rx['cm_return']>0: return rx
          tf=rx['cm_path']
          r=cm_kernel.save_array_to_file_as_json({'cm_filename':tf, 'cm_array':index})
          if r['cm_return']>0: return r

          cmd='curl -XPUT '+url+' -d @'+tf+' -s -o '+tfo2
          os.system(cmd)

          # Read output
          r1=cm_kernel.load_json_file({'cm_filename':tfo2})
          if r1['cm_return']>0: return r1
          xcfg=r1['cm_array']

          if not xcfg.get('ok',True):
             return {'cm_return':1, 'cm_error': 'indexing failed'}

          os.remove(tf)
          os.remove(tfo2)
      
    else:
       if i.get('sequential','')=='yes':
          r=index_seq(url, jindex, i['mode'], module_uid, data_uid)
       else:
          t=threading.Thread(target=index_seq, args=(url, jindex, i['mode'], module_uid, data_uid))
          t.start()
          # TBD: need to add check if was success or not!

    return r

# ============================================================================
def index_seq(url, jindex, mode, module_uid, data_uid):

    """
    Add or delete index

    Input:  {
              url    - Elastic search URL
              jindex - index in json to add
              mode   - 'delete' | 'add'
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    if cm_kernel.ini['dcfg'].get('use_indexing','')!='yes':
       return {'cm_return':1, 'cm_error':'indexing is OFF in the default kernel configuration'}

    request = urllib2.Request(url)
    if mode=='add' or mode=='delete':
       request.get_method = lambda: 'DELETE'

    try:
       f=urllib2.urlopen(request, jindex)
    except urllib2.URLError as e:
       if mode=='':
          cm_kernel.print_for_con('cM error: can\'t index ('+format(e)+')')
          return {'cm_return':1, 'cm_error':'can\'t index ('+format(e)+')'}

       pass # If not found, should ignore; but otherwise TBD - should print error (such as connection refused)
    else:
       s=str(f.read())
       try:
          j=json.loads(s)
       except:
          cm_kernel.print_for_con('cM error: can\'t parse output during indexing '+module_uoa+':'+data_uoa)
          return {'cm_return':1, 'cm_error':'can\'t parse output during indexing'}
       if not j["ok"]:
          cm_kernel.print_for_con('cM error: indexing was not successful during indexing '+module_uoa+':'+data_uoa)
          return {'cm_return':1, 'cm_error':'indexing was not successful indexing '+module_uoa+':'+data_uoa}

       f.close()

    r={'cm_return':0}
    if mode=='add':
       r=index_seq(url, jindex, '', module_uid, data_uid) # repeat add after delete

    return r

# ============================================================================
def search(i):

    """
    Search

    Input:  {
              cm_es_string   - simple search
              cm_es_query    - elastic search query
              (cm_index_uoa) - index configuration
              (size)         - limit return array
            }

    Output: {
              cm_return  - return code >0 if error
              cm_array   - array of UIDs
            }
    """

    if cm_kernel.ini['dcfg'].get('use_indexing','')!='yes':
       return {'cm_return':1, 'cm_error':'indexing is OFF in the default kernel configuration'}

    size=i.get('size','10000')

    if 'cm_es_string' not in i and 'cm_es_query' not in i:
       return {'cm_return':1, 'cm_error':'neither "cm_es_string" nor "cm_es_query" is defined'}

    all=[]

    # Check (default) index configuration
    index_uoa=''
    if 'cm_index_uoa' in i and i['cm_index_uoa']!='': index_uoa=i['cm_index_uoa']
    elif 'cm_default_index_uoa' in cm_kernel.ini['dcfg'] and cm_kernel.ini['dcfg']['cm_default_index_uoa']!='':
       index_uoa=cm_kernel.ini['dcfg']['cm_default_index_uoa']

    if index_uoa=='':
       return {'cm_return':1, 'cm_error':'"cm_index_uoa" is not defined'}

    # Load index
    r=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                        'cm_action':'load',
                        'cm_data_uoa':index_uoa})
    if r['cm_return']>0: return r
    index_cfg=r['cm_data_obj']['cfg']

    host=index_cfg['host']+':'+index_cfg.get('port','9200')

    # Check query
    jindex=None
    if 'cm_es_string' in i:
       url=host+'/_search?q='+urllib.quote_plus(i['cm_es_string'].encode('utf-8'))+'&size='+size
    else:
       url=host+'/_search?size='+size
       jindex = json.dumps({"query":i['cm_es_query']})

    try:
       if jindex==None:
          f=urllib2.urlopen(url)
       else:
          f=urllib2.urlopen(url, jindex)

    except urllib2.URLError, e:
       x='error while accessing index server ('+format(e)
       if jindex!=None:
         x+='; '+json.dumps(jindex, indent=2)
       x+=')'
       return {'cm_return':1, 'cm_error':x}
    else:
       s=str(f.read())
       try:
          j=json.loads(s)
       except:
          return {'cm_return':1, 'cm_error':'can\'t parse output during search'}

       f.close()

       # Cache repo and module
       repo={}
       module={}

       if 'hits' in j and 'total' in j['hits'] and long(j['hits']['total'])>0:
          for x in j['hits']['hits']:
              y=x['_source']

              d=y['cfg']

              # Adding to the list
              ii={'cm_module_uoa':y['cm_module_uoa'],
                  'cm_module_uid':y['cm_module_uid'],
                  'cm_data_uoa':y['cm_data_uoa'],
                  'cm_data_uid':y['cm_data_uid']}
              if 'cm_repo_uoa' in y: ii.update({'cm_repo_uoa':y['cm_repo_uoa']})
              if 'cm_repo_uid' in y: ii.update({'cm_repo_uid':y['cm_repo_uid']})
              if 'cm_display_as_alias' in d: ii.update({'cm_display_as_alias':d['cm_display_as_alias']})

              repo_uoa=y['cm_repo_uoa']
              if repo_uoa not in repo:
                 # Load repo to check if should ignore external repos for search
                 r9=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-repo'],
                                      'cm_data_uoa':repo_uoa,
                                      'cm_action':'load'})
                 if r9['cm_return']>0: 
                    repo[repo_uoa]={'access':'no'}
                 else:
                    r9d=r9['cm_data_obj']['cfg']
                    if r9d.get('exclude_from_search','')=='yes':
                       repo[repo_uoa]={'access':'no'}
                    else:
                       repo[repo_uoa]={'access':'yes', 'cm_display_as_alias':r9d.get('cm_display_as_alias','')}
                       ii['cm_repo_display_as_alias']=r9d.get('cm_display_as_alias','')
              else:
                 ii['cm_repo_display_as_alias']=repo[repo_uoa].get('cm_display_as_alias','')

              if repo[repo_uoa]['access']=='yes':

                 module_uoa=y['cm_module_uoa']
                 if module_uoa not in module:
                    # Load repo to check if should ignore external repos for search
                    r8=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-module'],
                                         'cm_data_uoa':module_uoa,
                                         'cm_action':'load'})
                    if r8['cm_return']>0: 
                       module[module_uoa]={'access':'no'}
                    else:
                       r8d=r8['cm_data_obj']['cfg']
                       module[module_uoa]={'access':'yes', 'cm_display_as_alias':r8d.get('cm_display_as_alias','')}
                       ii['cm_module_display_as_alias']=r8d.get('cm_display_as_alias','')
                 else:
                    ii['cm_module_display_as_alias']=module[module_uoa].get('cm_display_as_alias','')

                 # Check fine-grain access
                 ac=cm_kernel.ini['dcfg'].get('access_control',{})
                 if i.get('cm_admin','')!='yes' and cm_kernel.ini['web']=='yes' and ac.get('use','')=='yes':
                    rx=cm_kernel.access_control({'dcfg':d, 'module_uoa':y['cm_module_uoa'], \
                                                 'module_uid':y['cm_module_uid'], 'data_uoa':y['cm_data_uoa'], 'key':'read'})
                    if rx['cm_return']==0:
                       all.append(ii)
                 else:
                    all.append(ii)

    if i.get('cm_console','')=='txt':
       for x in all:
           cm_kernel.print_for_con(x['cm_repo_uoa']+':'+x['cm_module_uoa']+':'+x['cm_data_uoa'])
#       cm_kernel.print_for_con(json.dumps(all, indent=4))

    return {'cm_return':0, 'cm_array':all}

# ============================================================================
def clean(i):

    """
    Search

    Input:  {
              (index_repo_uoa) -   Repository UOA to clean indexes
              (index_module_uoa) - Module UOA to clean indexes
              (index_data_Uoa) -   Data UOA to clean indexes
            }

    Output: {
              cm_return  - return code >0 if error
            }
    """

    # Prepare search string
    es_string=''

    x=i.get('index_repo_uoa','')
    if x!='':
       if es_string!='': es_string+=' AND '
       es_string+=' ('
       es_string+='(cm_repo_uid:"'+x+'") OR (cm_repo_uoa:"'+x+'")'
       es_string+=')'

    x=i.get('index_module_uoa','')
    if x!='':
       if es_string!='': es_string+=' AND '
       es_string+=' ('
       es_string+='(cm_module_uid:"'+x+'") OR (cm_module_uoa:"'+x+'")'
       es_string+=')'

    x=i.get('index_data_uoa','')
    if x!='':
       if es_string!='': es_string+=' AND '
       es_string+=' ('
       es_string+='(cm_data_uid:"'+x+'") OR (cm_data_uoa:"'+x+'")'
       es_string+=')'

    if es_string=='':
       return {'cm_return':1,'cm_error':'empty query - if you would like to clean the whole index, use "flush"'}

    r=search({'cm_es_string':es_string})
    if r['cm_return']>0: return r

    all=r['cm_array']
    j=0
    for q in all:
        j+=1
        cm_kernel.print_for_con('Cleaning '+q['cm_repo_uoa']+':'+q['cm_module_uoa']+':'+q['cm_data_uoa']+' entry ('+str(j)+' of '+str(len(all))+')')

        ii={'cm_repo_uoa':q['cm_repo_uoa'],
            'cm_repo_uid':q['cm_repo_uid'],
            'cm_module_uoa':q['cm_module_uoa'],
            'cm_module_uid':q['cm_module_uid'],
            'cm_data_uoa':q['cm_data_uoa'],
            'cm_data_uid':q['cm_data_uid'],
            'mode':'delete'}
        r=index(ii)
        if r['cm_return']>0: return r

    return {'cm_return':0}