module.py

Go to the documentation of this file.

#
# Collective Mind
#
# See cM LICENSE.txt for licensing details.
# See cM Copyright.txt for copyright details.
#
# Developer(s): (C) Grigori Fursin, started on 2011.09
#

import os
import json
import zipfile
import time
import copy as xcopy # since we have "copy" function in this module

ini={}
cm_kernel=None

# ============================================================================
def init(i):

    return {'cm_return':0}

# ============================================================================
def add_data(i):

    """
    Add repository

    Input:  {
              (path)        - path to create repository (otherwise current) or URL
              path_type     - absolute, relative_to_cm, remote_url
              (cm_repo_uoa) - repo UOA where to create entry
              (cm_array)    - additional description
              (repo_alias)  - alias on the disk
              (repo_name)   - user friendly name (when listing repositories)
              (repo_uid)    - set UID for the repository - useful 
                              when creating personal user repository in shared environments ...
            }

    Output: {
              cm_return  - return code >0 if error
              cm_path    - path to created data entry
              ...        - other paramaters from core add_data function
            }
    """

    # If web access, move to 'core' module
    if i.get('cm_console','')=='web':
       i['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
       i['cm_action']='add'
       i['cm_subaction_add']='yes'
       return cm_kernel.access(i)

    # Check if cm_array or CMD
    if 'cm_array' in i: ir=i['cm_array']
    else: 
       # FGG: in original cM I was adding everything from CMD to the entry
       # but at the end it was too messy, so we only add to entry from cm_array now
       #ir=i
       ir={}
    if i.get('path_type','')!='': ir['path_type']=i['path_type']
    if i.get('path','')!='': ir['path']=i['path']

    # Second, process request
    # Prepar path
    p=os.getcwd()
    if 'path' in ir: p=ir['path']
    else: ir['path']=p

    if ir.get('path_type','')=='relative_to_cm':
       p=cm_kernel.ini[cm_kernel.env_cm_root]
       if ir.get('path','')!='':
          p=os.path.join(p, ir['path'])

    path_type=''
    if 'path_type' in ir: path_type=ir['path_type']
    else: ir['path_type']='absolute'

    if path_type=='':
       return {'cm_return':1, 'cm_error':'path type is not defined'}
    elif path_type=='remote_url':
       p=''

    # Check if directory already exists
    if path_type!='remote_url':
       p1=os.path.join(p, cm_kernel.ini['dcfg']['cmr_ext'])

       found=False
       if os.path.isdir(p):
          # Check if exist in the repository
          r=get_all_repo_paths({})
          if r['cm_return']>0: return r

          all=r['cm_array']

          for x in all:
              if 'cm_path_without_cmr' in x and os.path.isdir(x['cm_path_without_cmr']) and os.stat(x['cm_path_without_cmr'])==os.stat(p):
                 found=True
                 break 

          if found:
             if not os.path.isdir(p1): os.makedirs(p1)

             return {'cm_return':1, 'cm_error':'repository already exist at this path '+p1}

    if ir.get('repo_name','')!='': ir['cm_display_as_alias']=ir['repo_name']

    # Go to common core function to create entry
    ii={};ii.update(i)
    if ir.get('repo_alias','')!='': ii['cm_data_uoa']=ir['repo_alias']
    if ir.get('repo_uid','')!='': ii['cm_data_uid']=ir['repo_uid']
    ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
    ii['cm_module_uoa']=i['cm_run_module_uoa']
    ii['cm_array']=ir
    ii['use_default_repo']='yes' # Always create repository in default repo!
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    # Create directory
    print p1
    x=''
    if path_type!='remote_url' and not found and not os.path.isdir(p1):
       os.makedirs(p1)
       x=' at '+p1

    if i.get('cm_console','')=='txt':
       cm_kernel.print_for_con('Repository successfully created'+x)

    return r

# ============================================================================
def update_data(i):

    """
    Update repository

    Input:  {
            }

    Output: {
            }
    """

    # If web access, move to 'core' module
    if i.get('cm_console','')=='web':
       i['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
       i['cm_action']='update'
       return cm_kernel.access(i)

    # Check if cm_array or CMD
    if 'cm_array' in i: ir=i['cm_array'] 
    else: 
       # FGG: in original cM I was adding everything from CMD to the entry
       # but at the end it was too messy, so we only add to entry from cm_array now
       #ir=i
       ir={}
       if i.get('path_type','')!='': ir['path_type']=i['path_type']
       if i.get('path','')!='': ir['path']=i['path']

    # Load data to get path
    ii={}
    ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
    ii['cm_action']='load'
    if 'cm_module_uoa' in i: ii['cm_module_uoa']=i['cm_module_uoa']
    if 'cm_data_uoa' in i: ii['cm_data_uoa']=i['cm_data_uoa']
    if 'cm_repo_uoa' in i: ii['cm_repo_uoa']=i['cm_repo_uoa']
    rx=cm_kernel.access(ii)
    if rx['cm_return']>0: return rx

    ii={};ii.update(i)
    ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
    ii['cm_console']=''
    if cm_kernel.ini.get('cm_console','')!='web': ii['cm_console']=i.get('cm_console','')
    if cm_kernel.ini.get('web','')=='yes': ii['cm_report_in_html']='yes'

    ii['cm_array']=ir

    ru=cm_kernel.access(ii)
    if ru['cm_return']>0: return ru

    # We need it here since this function overloads original 'core' update_data function
    ru['cm_skip_report']='yes'

    return ru

# ============================================================================
def find_path_to_repository(i):

    """
    Find path to a given repository

    Input:  {
              (find_repo_uoa)     - repo UOA
              (get_default_repo)  - if 'yes' and find_repo_uoa=='', get path to default repo
              (cm_console)        - if 'txt', output as text to console
              (cm_admin)          - if 'yes', override access control - only internal use,
                                    can not be used during web access (explicitly removed)
            }

    Output: {
              cm_return        - return code >0 if error
              cm_path          - path to repository
              cm_data_obj      - cM data object
              cm_uid           - uid (from UOA)
              cm_alias         - alias (from UOA)
              cm_uoa           - cm_data_uoa (alias or uid if alias=='')
              (cur_module_uoa) - current module UOA if in current path
              (cur_data_uoa)   - current data UOA if in current path
            }
    """

    p=''

    find_repo_uoa=''

    # If repository is not given, check current, local and then default repositories
    cur_info={}

    if 'find_repo_uoa' in i and i['find_repo_uoa']!='':
       find_repo_uoa=i['find_repo_uoa']
    else:
       er=False

       if i.get('get_default_repo','')!='yes':
          r=cm_kernel.find_path_to_current_repository({})
          if r['cm_return']>0: return r
          elif r['cm_return']==0:
             p=r['cm_path']
             if r.get('cm_module_uoa','')!='': cur_info['cur_module_uoa']=r['cm_module_uoa']
             if r.get('cm_data_uoa','')!='': cur_info['cur_data_uoa']=r['cm_data_uoa']
          else:
             er=True

       if er or i.get('get_default_repo','')=='yes':
          if 'cm_local_repo' in cm_kernel.ini and os.path.isdir(cm_kernel.ini['cm_local_repo']):
             p=cm_kernel.ini['cm_local_repo']
          elif 'cm_default_repo' in cm_kernel.ini and os.path.isdir(cm_kernel.ini['cm_default_repo']):
             p=cm_kernel.ini['cm_default_repo']
          else:
             return {'cm_return':0, 'cm_error':'can\'t find repository'}

       # Check uid of this repository
       r1=get_all_repo_paths({})
       if r1['cm_return']>0: return r1

       found=False
       for x in r1['cm_array']:
           if os.path.isdir(p) and os.path.isdir(x['cm_path']) \
              and os.stat(p)==os.stat(x['cm_path']):
              found=True
              break

       if not found:
          return {'cm_return':1, 'cm_error': 'access is likely forbidden to repo '+p}

       find_repo_uoa=x['cm_uid']

    # Try to load repository to check various parameters ...
    ii={'cm_action':'load',
        'cm_run_module_uoa':ini['cm_module_uid'],
        'cm_data_uoa':find_repo_uoa,
        'cm_admin':i.get('cm_admin','')}
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    d=r['cm_data_obj']
    uid=r['cm_uid']
    uoa=r['cm_uoa']
    alias=r['cm_alias']

    # If  remote 
    p=''
    if d['cfg'].get('path_type','')=='remote_url':
       p='#remote_url#'
    else:
       # Prepare path 
       r2=prepare_path({'cm_cfg':r['cm_data_obj']['cfg']})
       if r2['cm_return']>0: return r2
       p=r2['cm_path']

       if not os.path.exists(p):
          try:
             os.makedirs(p)
          except Exception as e:
             pass

    r['cm_path']=p

    if i.get('cm_console','')=='txt':
       cm_kernel.print_for_con(p)

    r.update(cur_info)

    return r

# ============================================================================
def test_repository(i):

    """
    Test repository (mainly remote access)
    It just try to load tmp:tmp and returns ok, if it exists or not found,
    thus meaning that the structure of repository/access is fine

    Input:  {
              test_repo_uoa - repo UOA

              ...           - various additional parameters 
                              (for example, for authentication)
            }

    Output: {
              cm_return     - return code >0 if error
            }
    """

    repo=i.get('test_repo_uoa','')
    if repo=='':
       return {'cm_return':1, 'cm_error':'"test_repo_uoa" is not specified'}

    con=i.get('cm_console','')

    # First try to load repository 
    ii={'cm_action':'load',
        'cm_run_module_uoa':ini['cm_module_uid'],
        'cm_data_uoa':repo}
    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    # Try to load some data from the repository
    i['cm_action']='load'
    i['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-module']
    i['cm_console']='json'
    if 'cm_module_uoa' in i: del(i['cm_module_uoa'])
    i['cm_data_uoa']='tmp'

    if repo!='': 
       i['cm_repo_uoa']=repo
       del(i['test_repo_uoa'])
    r=cm_kernel.access(i)
    if r['cm_return']>0 and r['cm_return']!=16: return r

    if con=='txt':
       cm_kernel.print_for_con('Ok')

    return {'cm_return':0}

# ============================================================================
def get_all_repo_paths(i):

    """
    Get paths for all repositories (has to be low level operation!)

    Input:  {
              (cm_admin)          - if 'yes', override access control - only internal use,
                                    can not be used during web access (explicitly removed)
            }

    Output: {
              cm_return  - return code >0 if error
              cm_array   - array with paths
            }
    """

    # Get repositories where to search for 'repo' - has to be here!!!
    r=cm_kernel.find_path_to_work_repositories({})
    if r['cm_return']>0: return r
    a=r['cm_array']
    b=[]

    # Look through all repo UIDs
    r1={'cm_module_uoa':ini['cm_module_uid']}
    # 'cm_only_uid':'yes'}
    for x in a:
        r1['cm_path']=x
        r1['cm_admin']=i.get('cm_admin','')
        # Obtain list of data from the repository
        r=cm_kernel.get_all_uoa(r1)
        if r['cm_return']==0:
           for y in r['cm_array']:
               r1['cm_data_uoa']=y
               r1['cm_admin']=i.get('cm_admin','')
               r=cm_kernel.load_data(r1)
               if r['cm_return']==0:
                  r2=prepare_path({'cm_cfg':r['cm_data_obj']['cfg']})
                  if r2['cm_return']>0: return r2

                  p=r2['cm_path']
                  p1=r2['cm_path_without_cmr']
                  if p!='' and p!=cm_kernel.ini['dcfg']['cmr_ext']:
                     found=False
                     for q in b:
                         if os.path.isdir(q['cm_path']) and os.path.isdir(p) and os.stat(q['cm_path'])==os.stat(p):
                            found=True
                            break
                     if not found: 
                        b.append({'cm_path':p,
                                  'cm_path_without_cmr':p1,
                                  'cm_uoa':r1['cm_data_uoa'],
                                  'cm_uid':r['cm_uid'],
                                  'cm_alias':r['cm_alias']})

    if i.get('cm_console','')=='txt':
       for x in b:
           cm_kernel.print_for_con(x['cm_path'])

    return {'cm_return':0, 'cm_array':b}

# ============================================================================
def prepare_path(i):

    """
    Prepare path from the repository entry as absolute path 
    (convert from relative or absolute)

    Input:  {
              cm_cfg              - cfg file of the repository
            }

    Output: {
              cm_return           - return code >0 if error
              cm_path             - path
              cm_path_without_cmr - path without .cmr
            }
    """

    z=i['cm_cfg']
    p=''
    p1=''

    if 'path_type' in z:
       if z['path_type']=='relative_to_cm':
          p1=os.path.normpath(os.path.join(cm_kernel.ini[cm_kernel.env_cm_root], z['path']))
       elif z['path_type']=='absolute':
          p1=os.path.normpath(z['path'])
       p=os.path.normpath(os.path.join(p1, cm_kernel.ini['dcfg']['cmr_ext']))

    return {'cm_return':0, 'cm_path':p, 'cm_path_without_cmr':p1}

# ============================================================================
def share(i):

    """
    Check out repository if it is shared

    Input:  {
              cm_command    - explicit command for sharing tool 'co' or 'commit'
              (cm_repo_uoa) - repository UOA; if empty, update all repositories that are shared
            }                
    Output: {
              cm_return  - return code >0 if error
            }
    """

    # Check some vars
    cx=i.get('cm_command','')
    if cx=='': return {'cm_return':1, 'cm_error':'command is not specified in "repo/share"'}

    # Build list of repos if needed
    repos=[]

    if i.get('cm_repo_uoa','')=='':
       jj={}
       jj['cm_run_module_uoa']=ini['cm_module_uid']
       jj['cm_action']='list'
       rr=cm_kernel.access(jj)
       if rr['cm_return']>0: return rr

       for xx in rr['cm_mixed']:
           x=xx['cm_uoa'];  xu=xx['cm_uid'];  xa=xx['cm_alias'];  xd=xx['cm_display_html']

           r=find_path_to_repository({'find_repo_uoa':xu})
           if r['cm_return']==0:
              repos.append({'uoa':r['cm_uoa'], 'display':r['cm_display_as_alias'], 'cfg':r['cm_data_obj']['cfg'], 'path':r['cm_path']})
    else:
       r=find_path_to_repository({'find_repo_uoa':i['cm_repo_uoa']})
       if r['cm_return']>0: return r

       repos.append({'uoa':r['cm_uoa'], 'display':r['cm_display_as_alias'], 'cfg':r['cm_data_obj']['cfg'], 'path':r['cm_path']})

    # go through repositories
    for q in repos:
       d=q['cfg']
       p=q['path']

       if d.get('share','')!='yes':
          if i.get('cm_repo_uoa','')!='': return {'cm_return':1, 'cm_error':'this repository is not shared'}
          else: continue

       if i.get('cm_repo_uoa','')=='': cm_kernel.print_for_con('************************************************************')
       else:                           cm_kernel.print_for_con('')
       x='Updating '
       if q['display']!='': x+=q['display']
       else: x+=q['uoa']

       cm_kernel.print_for_con(x)
       cm_kernel.print_for_con('')

       if 'sharing_tool' not in d:
          if i.get('cm_repo_uoa','')!='': return {'cm_return':1, 'cm_error':'"sharing_tool" is not specified for this repository'}
          else: continue
          
       if d['sharing_tool'] not in ini['cfg']['sharing_tools']:
          if i.get('cm_repo_uoa','')!='': return {'cm_return':1, 'cm_error':'"sharing_tool" is unknown for this repository'}
          else: continue

       command=ini['cfg']['sharing_tools'][d['sharing_tool']][cx]
       command1=command.replace(cm_kernel.convert_str_to_special('sharing_url'), d['sharing_url'])
       command2=command1.replace(cm_kernel.convert_str_to_special('cm_path'), p)

       cm_kernel.print_for_con('Executing '+command2+' ...')
       cm_kernel.print_for_con('')

       r=os.system(command2)
       if r>0:
          if i.get('cm_repo_uoa','')!='': return {'cm_return':1, 'cm_error':'exit code '+str(r)}
          else: 
             cm_kernel.print_for_con('')
             cm_kernel.print_for_con('Repository update likely failed - exit code: '+str(r))
             cm_kernel.print_for_con('')
             var=raw_input('Press enter to continue: ')
             cm_kernel.print_for_con('')

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Updating finished!')

    return {'cm_return':0}

# ============================================================================
def download(i):

    """
    Download repository using cM web interface (possibly substitute old entries)

    Input:  {
              (web_repo_uoa)        - repository UOA; if empty, update all repositories that are shared
              (overwrite)           - if 'yes', overwrite entries
              (download_all)        - if 'yes', download all
              (skip_large_download) - if 'yes', skip entries with large downloads
              (interactive)         - if 'yes', interactive
              (ask)                 - if 'yes', ask what to do with modules with many entries
              (cm_remote_user_uoa)  - change default user to access repo ...
            }                
    Output: {
              cm_return  - return code >0 if error
            }
    """

    ow=i.get('overwrite','')
    cru=i.get('cm_remote_user_uoa','')

    cm_kernel.print_for_con('Warning: currently it can be a very long operation')
    cm_kernel.print_for_con('         and we hope to speed it up in the future - please, be patient!')
    cm_kernel.print_for_con('')

    # Start timer
    t0=time.time()

    # Search through repositories
    rlist=[]
    if i.get('web_repo_uoa','')!='':
       # Load repo
       r2=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                            'cm_action':'load',
                            'cm_data_uoa':i['web_repo_uoa']})
       if r2['cm_return']>0: return r2
       rlist=[{'cm_uoa':r2['cm_uoa'], 'cm_uid':r2['cm_uid']}]
    else:
       r2=cm_kernel.access({'cm_run_module_uoa':ini['cm_module_uid'],
                            'cm_action':'list'})
       if r2['cm_return']>0: return r2
       rlist=r2['cm_mixed']

    for repo1 in rlist:
        # Check print
        repo_uoa=repo1['cm_uoa']
        repo_uid=repo1['cm_uid']

        ww=repo_uoa

        # Load repo
        ii={}
        ii['cm_run_module_uoa']=ini['cm_module_uid']
        ii['cm_action']='load'
        ii['cm_data_uoa']=ww
        rr=cm_kernel.access(ii)
        if rr['cm_return']>0 and rr['cm_return']!=16: return rr
        if rr['cm_return']==0:
           d=rr['cm_data_obj']['cfg']

           wru=d.get('web_repo_uoa','')
           rru=d.get('remote_repo_uoa','')

           # Check repository that supports downloading
           if wru!='' and rru!='':
              cm_kernel.print_for_con('============================================================')
              cm_kernel.print_for_con('Processing repository '+ww+' ...')

              ii={}
              ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
              ii['cm_action']='list'
              ii['cm_repo_uoa']=wru
              ii['remote_repo_uoa']=rru
              ii['cm_only_available_modules']='yes'
              if cru!='': ii['cm_remote_user_uoa']=cru
              r1=cm_kernel.access(ii)
              if r1['cm_return']==0:

                 # Iterating over available modules in the repo
                 for q in sorted(r1['cm_mixed'], key=lambda k: k['cm_uoa']):
                     q1=q['cm_uid']
                     q2=q['cm_display_as_alias']
                     q3=q['cm_uoa']
                     cm_kernel.print_for_con('************************************************************')
                     cm_kernel.print_for_con('Retrieving entries for module "'+q3+'":')

                     # Load module
                     ii={}
                     ii['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-module']
                     ii['cm_action']='load'
                     ii['cm_data_uoa']=q1
                     rr=cm_kernel.access(ii)
                     if rr['cm_return']>0: return rr
                     dm=rr['cm_data_obj']['cfg']

                     # List data entries
                     jj={}
                     jj['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
                     jj['cm_action']='list'
                     jj['cm_module_uoa']=q['cm_uid']
                     jj['cm_repo_uoa']=wru
                     jj['remote_repo_uoa']=rru
                     jj['skip_data_cfg']='yes'

                     s='M' # minimal download by default

                     if i.get('download_all','')=='yes':          s='A'
                     elif i.get('skip_large_download','')=='yes': s='S'
                     elif i.get('interactive','')=='yes':         s='I'
                     elif i.get('ask','')=='yes':                 s=''

                     if cru!='': jj['cm_remote_user_uoa']=cru

                     all=[]

                     # Check if module have "minimal setup" to prune downloads
                     if dm.get('minimal_setup_for_download','')=='yes':
                        if s=='':
                           cm_kernel.print_for_con('')
                           cm_kernel.print_for_con('NOTE: there may be many entries associated with this module!')
                           cm_kernel.print_for_con('      Press <Enter>  to download only minimal set (default)')
                           cm_kernel.print_for_con('            A<Enter> to download all entries')
                           cm_kernel.print_for_con('            I<Enter> to download interactively entries')
                           cm_kernel.print_for_con('            S<Enter> to skip this module')
                           s=raw_input('')

                        if s=='s' or s=='S':
                           continue
                        elif s=='' or s=='m' or s=='M':
                           # adding setup.minimal class for listing
                           jj['cm_classes_uoa']=ini['cfg']['class_setup_minimal_uoa']

                     jj_copy=xcopy.deepcopy(jj)
                     r2=cm_kernel.access(jj)
                     if r2['cm_return']>0: return r2
                     for q in r2['cm_mixed']:
                         all.append(q)

                     if s=='' or s=='m' or s=='M':
                        # adding setup.minimal class for listing
                        jj=jj_copy
                        jj['cm_classes_uoa']=ini['cfg']['class_setup_minimal1_uoa']

                        r2=cm_kernel.access(jj)
                        if r2['cm_return']>0: return r2
                        for q in r2['cm_mixed']:
                            all.append(q)
                      
                     cm_kernel.print_for_con('')
                     for k in sorted(all, key=lambda k: k['cm_uoa']):
                         k1=k['cm_uid']
                         k2=k['cm_display_as_alias']
                         k3=k['cm_uoa']

                         download=True
                         if s=='I' or s=='i':
                            x=raw_input('Would you like to download cM entry "'+q3+':'+k3+'" (Y/N(default)):')
                            if x=='N' or x=='n' or x=='': download=False
                            
                         if download:
                            cm_kernel.print_for_con('')
                            cm_kernel.print_for_con('Downloading cM entry "'+q3+':'+k3+'" (elapsed time='+str("%.1f" % (time.time()-t0))+' sec.) ...')

                            # Check if already exists
                            exists=True
                            jj={}
                            jj['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
                            jj['cm_action']='load'
                            jj['cm_module_uoa']=q3
                            jj['cm_data_uoa']=k3
                            jj['cm_repo_uoa']=ww
                            if cru!='': ii['cm_remote_user_uoa']=cru
                            r2=cm_kernel.access(jj)
                            if r2['cm_return']>0: 
                               if r2['cm_return']!=16: return r2
                               exists=False

                            path=''

                            # Individually check a few modules (should move to cfg)
                            if exists: 
                               path=r2['cm_path']
                            else:
                               # Create local entry
                               jj={}
                               jj['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
                               jj['cm_action']='add'
                               jj['cm_module_uoa']=q3
                               jj['cm_data_uoa']=k3
                               jj['cm_data_uid']=k1
                               jj['cm_repo_uoa']=ww
                               r2=cm_kernel.access(jj)
                               if r2['cm_return']>0: return r2
                               path=r2['cm_path']

                            if not exists or ow=='yes':
                               if exists and ow!='yes':
                                  cm_kernel.print_for_con('   Entry exists and overwrite mode is off - skipping ...')

                               # Get data description
                               jj={"cm_action":"load",
                                   "cm_run_module_uoa":ini['cfg']['cm_modules']['cm-core'],
                                   "cm_module_uoa":q3,
                                   "cm_data_uoa":k3,
                                   "cm_repo_uoa":wru,
                                   "remote_repo_uoa":rru}
                               if cru!='': jj['cm_remote_user_uoa']=cru
                               r2=cm_kernel.access(jj)
                               if r2['cm_return']>0: return r2
                               dd=r2['cm_data_obj']['cfg']

                               # Get list of files
                               jj={"cm_action":"get_list_of_all_files",
                                   "cm_run_module_uoa":ini['cfg']['cm_modules']['cm-core'],
                                   "cm_module_uoa":q3,
                                   "cm_data_uoa":k3,
                                   "cm_repo_uoa":wru,
                                   "remote_repo_uoa":rru}
                               if cru!='': jj['cm_remote_user_uoa']=cru
                               r2=cm_kernel.access(jj)
                               if r2['cm_return']>0: return r2

                               lf=r2['cm_array']
                               if len(lf)>6:
                                  cm_kernel.print_for_con('   Downloading archive ...')
                                  # If many files, pack entry before downloading
                                  fn=os.path.join(path, 'downloaded.zip')

                                  # Download
                                  jj={"cm_action":"download",
                                      "cm_run_module_uoa":ini['cfg']['cm_modules']['cm-core'],
                                      "cm_module_uoa":q3,
                                      "cm_data_uoa":k3,
                                      "cm_repo_uoa":wru,
                                      "remote_repo_uoa":rru,
                                      "cm_save_to_file":fn,
                                      "cm_all_files":"yes"}
                                  if cru!='': jj['cm_remote_user_uoa']=cru
                                  r2=cm_kernel.access(jj)
                                  if r2['cm_return']>0: return r2

                                  cm_kernel.print_for_con('   Unpacking (archive size='+str(os.path.getsize(fn))+' bytes) ...')

                                  # Unzip
                                  f=open(fn,'rb')
                                  z=zipfile.ZipFile(f)
                                  for d in z.namelist():
                                      if os.sep!='/': 
                                         d=d.replace('/', os.sep)

                                      if not d.startswith('..') and not d.startswith('/') and not d.startswith('\\'):
                                         dr=os.path.dirname(d)
                                         pp2=os.path.normpath(os.path.join(path,d))
                                         if d.endswith('/') or d.endswith('\\'): 
                                            # create directory 
                                            if not os.path.exists(pp2): os.makedirs(pp2)
                                         else:
                                            # check subdirectory
                                            if dr!='':
                                               dr1=os.path.join(path,dr)
                                               if not os.path.exists(dr1): os.makedirs(dr1)
                                            # extract file
                                            fine=True
                                            if os.path.exists(pp2):
                                               if ow=='yes':
                                                  # check if overwrite
                                                  os.remove(pp2)
                                               else:
                                                  fine=False

                                            if fine:
                                               fo=open(pp2, 'wb')
                                               fo.write(z.read(d))
                                               fo.close()
                                  f.close()
                                  os.remove(fn)   
                               else:
                                  # Overwrite data.json directly
                                  r4=cm_kernel.save_array_to_file_as_json({'cm_filename':os.path.join(path,cm_kernel.var_cm_info,cm_kernel.var_cm_data), \
                                                                           'cm_array':dd})
                                  if r4['cm_return']>0: return r4
                                  
                                  # If just a few files, download directly
                                  for q in lf:
                                      ql=q
                                      if os.sep!='/': 
                                         ql=q.replace('/', os.sep)

                                      cm_kernel.print_for_con('   Downloading file '+ql+' ...')

                                      dr=os.path.join(path,ql)
                                      # Check if sub-directory
                                      path, filename = os.path.split(dr)
                                      if not os.path.exists(path): 
                                         os.makedirs(path)

                                      # Download
                                      jj={"cm_action":"show",
                                          "cm_run_module_uoa":ini['cfg']['cm_modules']['cm-core'],
                                          "cm_module_uoa":q3,
                                          "cm_data_uoa":k3,
                                          "cm_repo_uoa":wru,
                                          "remote_repo_uoa":rru,
                                          "cm_web_filename":q,
                                          "cm_save_to_file":dr,
                                          "cm_web_content":"unknown"}
                                      if cru!='': jj['cm_remote_user_uoa']=cru
                                      r2=cm_kernel.access(jj)
                                      if r2['cm_return']>0: return r2

                               # Update local entry (for indexing)
                               jj={}
                               jj['cm_run_module_uoa']=ini['cfg']['cm_modules']['cm-core']
                               jj['cm_action']='update'
                               jj['cm_module_uoa']=q3
                               jj['cm_data_uoa']=k1
                               jj['cm_repo_uoa']=ww
                               jj['cm_array']={}
                               jj['cm_skip_update_info']='yes'
                               r2=cm_kernel.access(jj)
                               if r2['cm_return']>0: return r2
                            else:
                               cm_kernel.print_for_con('   Entry exists and overwrite mode is off - skipping ...')

    return {'cm_return':0}

# ============================================================================
def web_copy(i):

    """
    Copy repository to another one (and clean target if needed)

    Input:  {
            }

    Output: {
              cm_return  - return code >0 if error
              cm_path    - path to file
              cm_uid     - generated uid
            }
    """

    # Get web style
    if 'cfg' in cm_kernel.ini['web_style']: web=cm_kernel.ini['web_style']['cfg']
    else:
       return {'cm_return':1, 'cm_error':'web style is not defined'}

    cm_kernel.print_for_con('<span class="cm-title">Copy repository</span><br>')

    # Detecting/restoring data from forms
    a1={}
    r=cm_kernel.access({'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
                        'cm_action':'detect_form_params',
                        'cm_array':i, 
                        'cm_prefix':'#form1'})
    if r['cm_return']>0: return r
    cm_form_array1=r['cm_array']
    cm_form_commands1=r['cm_commands']

    # Get data description for this action
    r=cm_kernel.get_data_description({'cm_module_uoa':ini['cm_module_uoa'], 
                                      'cm_which_action':i['cm_action']})
    if r['cm_return']>0: return r
    cm_data_desc1=r['cm_data_desc']
    cm_params_default1=r['cm_params_default']

    # Check default
    forms_exists='yes'
    if len(cm_form_array1)==0:
       a1=cm_params_default1

       forms_exists='no'
    else:
       r=cm_kernel.restore_flattened_array({'cm_array':cm_form_array1, 
                                            'cm_replace_in_keys':{'^35^':'#', '^64^':'@'}})
       if r['cm_return']>0: return r
       a1=r['cm_array']

    # Check if clean
    if 'cm_submit_copy' in i:
       ii={}
       ii['cm_run_module_uoa']=ini['cm_module_uid']
       ii['cm_action']='copy'

       ii['cm_detach_console']='yes'

       ii.update(a1)

       r=cm_kernel.access_fe_through_cmd(ii)
       if r['cm_return']>0: return r

       x='Check detached console for progress!'
       if r.get('cm_stdout','')!='': x=r['cm_stdout']
       cm_kernel.print_for_con('<BR>' )
       cm_kernel.print_for_con(x)
       cm_kernel.print_for_con('<BR><BR>' )

    else:
       cm_kernel.print_for_con('<FORM ACTION="" name="add_edit" METHOD="POST" enctype="multipart/form-data" accept-charset="utf-8">' )

       ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-web'],
           'cm_action':'visualize_data',
           'cm_array':a1,
           'cm_data_desc':cm_data_desc1,
           'cm_form_commands':cm_form_commands1,
           'cm_separator':'#',
           'cm_separator_form':'#form1#',
           'cm_forms_exists':forms_exists,
           'cm_support_raw_edit':'no',
           'cm_mode':'add'}
       if 'cm_raw_edit' in i: ii['cm_raw_edit']=i['cm_raw_edit']
       if 'cm_back_json' in i: ii['cm_back_json']=i['cm_back_json']
       r=cm_kernel.access(ii)
       if r['cm_return']>0: return r
       cm_kernel.print_for_web(r['cm_string'])

       cm_kernel.print_for_con('<input type="submit" class="cm-button" name="cm_submit_copy" value="(Clean and) Copy">')

       cm_kernel.print_for_con('</FORM><br>')

    return {'cm_return':0}

# ============================================================================
def copy(i):

    """
    Copy repository to another one (and clean target if needed)

    Input:  {
              orig_repo_uoa       - original repository
              (orig_module_uoa)   - copy data for this module
              (cm_classes_uoa)    - prune data by classes

              target_repo_uoa     - target repository
              (clean_target_repo) - force cleaning of target repo
            }

    Output: {
              cm_return  - return code >0 if error
              cm_path    - path to file
              cm_uid     - generated uid
            }
    """

    # Check some vars
    trepo=i.get('target_repo_uoa','')
    if trepo=='': return {'cm_return':1,'cm_error':'parameter "target_repo_uoa" is not set in repo/copy'}

    orepo=i.get('orig_repo_uoa','')
    omodule=i.get('orig_module_uoa','')

    # Check if need cleaning of target repository
    if i.get('clean_target_repo','')=='yes':
       r=clean({'target_repo_uoa':i['target_repo_uoa'], 
                'target_module_uoa':i.get('orig_module_uoa',''),
                'target_data_uoa':i.get('orig_data_uoa',''),
                'cm_classes_uoa':i.get('cm_classes_uoa',[])})
       if r['cm_return']>0: return r

    cm_kernel.print_for_con('************************************************************')
    cm_kernel.print_for_con('Copying repository ...')
    cm_kernel.print_for_con('')

    # Copying data
    ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'], 
        'cm_action':'list'}
    if orepo!='': ii['cm_repo_uoa']=orepo
    if omodule!='': ii['cm_module_uoa']=omodule
    if i.get('orig_data_uoa','')!='': ii['prune_data_uoa']=i['orig_data_uoa']
    if len(i.get('cm_classes_uoa',[]))>0: ii['cm_classes_uoa']=i['cm_classes_uoa']

    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    for xx in r['cm_mixed']:
        x=xx['cm_uoa'];  xu=xx['cm_uid'];  xa=xx['cm_alias'];  xd=xx['cm_display_html']

        cm_kernel.print_for_con('  Copying '+xx['cm_module_uoa']+':'+x+' ...')
        
        ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'],
            'cm_action':'copy',
            'orig_module_uoa':xx['cm_module_uid'],
            'orig_data_uoa':xu,
            'target_repo_uoa':trepo}
        if orepo!='': ii['orig_repo_uoa']=orepo

        r=cm_kernel.access(ii)
        if r['cm_return']>0: 
           cm_kernel.print_for_con('    Error copying data: '+r['cm_error'])

    cm_kernel.print_for_con('')
    cm_kernel.print_for_con('Copying finished!')

    return {'cm_return':0}

# ============================================================================
def clean(i):

    """
    Clean repository 

    Input:  {
              target_repo_uoa     - target repo UOA
              (target_module_uoa) - clean data for this module
              (cm_classes_uoa)    - prune data by classes
              (target_data_uoa)   - data to clean
          
            }

    Output: {
              cm_return  - return code >0 if error
              cm_path    - path to file
              cm_uid     - generated uid
            }
    """

    cm_kernel.print_for_con('************************************************************')
    cm_kernel.print_for_con('Cleaning repository ...')
    cm_kernel.print_for_con('')

    # Check some vars
    trepo=i.get('target_repo_uoa','')
    if trepo=='': return {'cm_return':1,'cm_error':'parameter "target_repo_uoa" is not set in repo/clean'}
    tmodule=i.get('target_module_uoa','')

    # Get list of data:
    ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'], 
        'cm_action':'list',
        'cm_repo_uoa':trepo}
    if tmodule!='': ii['cm_module_uoa']=tmodule
    if i.get('target_data_uoa','')!='': ii['prune_data_uoa']=i['target_data_uoa']
    if len(i.get('cm_classes_uoa',[]))>0: ii['cm_classes_uoa']=i['cm_classes_uoa']

    r=cm_kernel.access(ii)
    if r['cm_return']>0: return r

    for xx in r['cm_mixed']:
        x=xx['cm_uoa'];  xu=xx['cm_uid'];  xa=xx['cm_alias'];  xd=xx['cm_display_html']

        cm_kernel.print_for_con('  Deleting '+xx['cm_module_uoa']+':'+x+' ...')
        
        ii={'cm_run_module_uoa':ini['cfg']['cm_modules']['cm-core'],
            'cm_action':'delete',
            'cm_repo_uoa':trepo,
            'cm_module_uoa':xx['cm_module_uid'],
            'cm_data_uoa':xu}
        r=cm_kernel.access(ii)
        if r['cm_return']>0: 
           cm_kernel.print_for_con('    Error deleting data: '+r['cm_error'])

    return {'cm_return':0}