cc2svn.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#===============================================================================
# The MIT License
# 
# Copyright (c) 2009 Vadim Goryunov
# 
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# 
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#===============================================================================
# 25.09.2013 Lukasz Tasz - enhanced handling of different location of the same element version
#			   between manu labels - this is the best cc2svn tool I ever seen ;)
"""
NAME
    cc2svn.py - converts ClearCase view files to SVN dump
    The dump can be loaded by SVN using 'cat svndump.txt | svnadmin load' command.    

SYNOPSIS
    cc2svn.py -run | -help

DESCRIPTION
    The tool uses the current ClearCase view to list the CC history (ct lshi -rec)
    then it goes through the history and processes each record.        
    That means that the tool does not transfer those files that are not visible from the current CC view.
    However the tool transfers all CC labels to SVN tags correctly. For that in the second phase 
    it sets config_spec of the current view to match the label (element * LABEL) for each given label 
    and checks that no files are lost during the first phase.
    WARNING: Side effect - the tool changes the config_spec of the current working ClearCase view.
             Do not use the view during the tool work.  
     
    All branches except the /main are created using 'svn cp' command basing on the CC parent branch.
    There is a difference in creating the branches in ClearCase and SVN.
    SVN copies all files from parent branch to the target like: svn cp branches/main branches/dev_branch
    ClearCase creates the actual branch for file upon checkout operation only.
    In other words the tool can't guarantee the content of /branches will be exactly like in ClearCase.    
    But the tool guarantees the labels are transferred correctly.
    
    The tool uses cache directory to place ClearCase version files there. The cache speeds up the transfer process
    in many times in subsequent attempts (up to 10 times). It may be recommended to start the tool 2 days before the
    actual transfer loading all files to the cache. So only new versions appeared during these days will be retrieved from
    ClearCase in the day of the transfer.        
    Actually the tool caches any data retrieved from ClearCase including the history file.
    
    The tool provides the possibility to retry/ignore any ClearCase command if error occurs.
    The tool will put empty file to the cache if you ignore ClearCase retrieving operation error.
    Make sure you know what you are doing when ignoring the error.
    
    See config.py for options description. 
    
    Timing: CC repository of 5 GB (~120.000 revisions) is converted in ~1 hour using the pre-cached files.

COMMAND LINE OPTIONS
    -run    starts the tool
    -help   prints this help
   
FILES          
    ./config.py            main configuration file written like a python module.
    ./config.autoprops     extension -> svn properties mapping. See config.py for details
    
AUTHOR
    Vadim Goryunov (vadim.goryunov@gmail.com)
    
LICENSING
    cc2svn.py is distributed under the MIT license.
"""

from __future__ import with_statement
import os, subprocess, time, sys, hashlib, codecs, fnmatch
import string
import traceback

USAGE = "Usage: %(cmd)s -run | -help" % { "cmd" : sys.argv[0] } 

if len(sys.argv) <= 1:
    print USAGE
    sys.exit(1)

if sys.argv[1] == "-help":
    print __doc__
    sys.exit(0)

if sys.argv[1] != "-run":
    print USAGE
    sys.exit(1)    

############# constants ######################
    
HISTORY_FIELD_SEPARATOR = "@@@"    

HISTORY_FORMAT = "%Nd;%En;%Vn;%o;%l;%a;%m;%u;%Nc;%Dn;\n".replace(";", HISTORY_FIELD_SEPARATOR)    

CC_DATE_FORMAT = "%Y%m%d.%H%M%S"
SVN_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.000000Z"

FILEREAD_CHUNKSIZE = 512

############# parameters ######################
CC_LABELS_FILE = None
CC_BRANCHES_FILE = None
DUMP_SINCE_DATE = None

from config import *

CLEARTOOL = os.path.realpath(CLEARTOOL)
CC_VOB_DIR = os.path.realpath(CC_VOB_DIR)
CACHE_DIR = os.path.realpath(CACHE_DIR)
SVN_AUTOPROPS_FILE = os.path.realpath(SVN_AUTOPROPS_FILE)
SVN_DUMP_FILE = os.path.realpath(SVN_DUMP_FILE)
HISTORY_FILE = os.path.realpath(HISTORY_FILE)

if CC_LABELS_FILE:
    CC_LABELS_FILE = os.path.realpath(CC_LABELS_FILE)    
    
if CC_BRANCHES_FILE:
    CC_BRANCHES_FILE = os.path.realpath(CC_BRANCHES_FILE)

if DUMP_SINCE_DATE:            
    DUMP_SINCE_DATE = time.strptime(DUMP_SINCE_DATE, CC_DATE_FORMAT)

CCVIEW_TMPFILE = CACHE_DIR + "/label_config_spec_tmp_cc2svnpy"
CCVIEW_CONFIGSPEC = CACHE_DIR + "/user_config_spec_tmp_cc2svnpy"

CC_LABELS_STRUCTURE = {}

############# utilities ######################

def logMessage(text):
    print time.strftime("%Y/%m/%d %H:%M:%S:"), text

def info(text):
    logMessage("INFO: " + text)

def warn(text):
    logMessage("WARNING: " + text)

def error(text):
    logMessage("ERROR: " + text)
    traceback.print_exc(file=sys.stdout)


def shellCmd(cmd, cwd=None, outfile=None):
    outfd = subprocess.PIPE
    outStr = ""
    status = ""
    while True:
        try:
            if outfile: 
                outfd = open(outfile, 'wb')
            if cwd and not os.path.exists(cwd):
                raise RuntimeError("No such file or directory: '" + cwd + "'")
            p = subprocess.Popen(cmd, cwd=cwd, stdout=outfd, stderr=subprocess.PIPE, shell=True, close_fds=True)
            (outStr, errStr) = p.communicate()
            if outfile:
                outfd.close()
            if p.returncode != 0:
                raise RuntimeError("Exit code: " + str(p.returncode) + "\n" + errStr)
            if len(errStr) > 0:
                raise RuntimeError("Command has non-empty error stream: \n" + errStr)
        except:
            error("Command failed: " + cmd + "\n" + str(sys.exc_info()[1]))
            status = askRetryContinueExit()
            if status == "retry": continue
        break
    return (status, outStr)

gIgnoreAll = False
def askRetryContinueExit():
    global gIgnoreAll
    if gIgnoreAll:
        return "ignore"    
    while True:
        print "\nRetry/Ignore/IgnoreAll/Exit? [r/i/a/x] (r:Enter): ",
        answer = sys.stdin.readline().strip()
        if answer == "" or answer == "r": return "retry"
        if answer == "i": return "ignore"
        if answer == "a": 
            gIgnoreAll = True
            return "ignore"
        if answer == "x": sys.exit(1)
        
def askYesNo(question):    
    while True:
        print "\n"+question+" [y/n] (y:Enter): ",
        answer = sys.stdin.readline().strip()
        if answer == "" or answer == "y": return True
        if answer == "n": return False

def toUTF8(text):
    return codecs.utf_8_encode(text)[0]

def rblocks(f, blocksize=4096):
    """Read file as series of blocks from end of file to start.

    The data itself is in normal order, only the order of the blocks is reversed.
    ie. "hello world" -> ["ld","wor", "lo ", "hel"]
    Note that the file must be opened in binary mode.
    """
    if 'b' not in f.mode.lower():
        raise Exception("File must be opened using binary mode.")
    size = os.stat(f.name).st_size
    fullblocks, lastblock = divmod(size, blocksize)

    # The first(end of file) block will be short, since this leaves 
    # the rest aligned on a blocksize boundary.  This may be more 
    # efficient than having the last (first in file) block be short
    f.seek(-lastblock,2)
    yield f.read(lastblock)

    for i in xrange(fullblocks-1,-1, -1):
        f.seek(i * blocksize)
        yield f.read(blocksize)

def rlines(f, keepends=False):
    """Iterate through the lines of a file in reverse order.

    If keepends is true, line endings are kept as part of the line.
    """
    buf = ''
    for block in rblocks(f):
        buf = block + buf
        lines = buf.splitlines(keepends)
        # Return all lines except the first (since may be partial)
        if lines:
            lines.reverse()
            buf = lines.pop() # Last line becomes end of new first line.
            for line in lines:
                yield line
    yield buf  # First line.

def try_to_decode(s, encodings=('ascii', 'utf8', 'latin1')):
    for encoding in encodings:
        try:
            return s.decode(encoding)
        except UnicodeDecodeError:
            pass
    print "Failed to decode '%s', ignoring" % s
    return s.decode('ascii', 'ignore')
    
############# heart of the script ######################

class SvnProperties:
    def __init__(self):
        self.keyset = {}
        self.totalLen = 10 # len('PROPS-END\n')
        
    def reset(self):
        self.keyset.clear()
        self.totalLen = 10
        
    def set(self, key, value):
        if self.keyset.has_key(key): # this will probably not happen
            self.totalLen -= self.calcPropLength(key, self.keyset.get(key))
        self.keyset[key] = value
        self.totalLen += self.calcPropLength(key, value)
        
    def calcPropLength(self, key, value):
        klen = len("K " + str(len(key)) + "\n" + key + "\n")
        vlen = len("V " + str(len(value)) + "\n" + value + "\n")
        return klen + vlen
    
    def writeLength(self, out):
        out.write("Prop-content-length: " + str(self.totalLen) + "\n");
    
    def writeContent(self, out):
        for key,value in self.keyset.iteritems():
            out.write("K " + str(len(key)) + "\n");
            out.write(key + "\n");
            out.write("V " + str(len(value)) + "\n");
            if value: out.write(value)
            out.write("\n")        
        out.write("PROPS-END\n");
    
    def dump(self, out):
        self.writeLength(out);        
        out.write("Content-length: " + str(self.totalLen) + "\n");
        out.write("\n");
        self.writeContent(out);
        out.write("\n\n");
        
EmptyProps = SvnProperties()

class SvnAutoProps:
    def __init__(self, filename):
        self.autoProps = {}
        self.load(filename)
        
    def load(self, filename):
        info("Loading svn auto properties from " + filename)
        
        file = open(filename,'r')
        for line in file:
            try: pattern, str = line.strip().split(" = ")
            except: continue
            props = SvnProperties()
            self.autoProps[pattern] = props
            for avp in str.split(";"):
                fields = avp.split("=")
                key = fields[0]
                if len(fields) > 1: 
                    value = fields[1]                    
                else:
                    value = ""
                props.set(key, value)
        file.close()
                
    def getProps(self, filepath):
        filename = os.path.basename(filepath)
        for pattern, props in self.autoProps.iteritems():
            if fnmatch.fnmatch(filename, pattern):
                return props
        return EmptyProps

class CCRecord:
    pass

class CCHistoryParser:
    def __init__(self):
        self.prevline = ""

    def parseLabels(self, s):
        # format: (label1, label2, label3)
        if len(s) > 0 and s.startswith("(") and s.endswith(")"):        
            return s[1:-1].split(", ")
        else:
            return [] 
            
    def processLine(self, line):
        if len(self.prevline) > 0:
            line = line + "\n" + self.prevline
        
        fields = line.split(HISTORY_FIELD_SEPARATOR)
        
        if len(fields) < 11:
            self.prevline = line;
            return None;
        elif len(fields) > 11:
            error("Wrong history line: " + line)
            self.prevline = ""
            return None
        self.prevline = ""
        
        # 20090729.162424;path/to/dir;/main/branch/another/1;checkin;(LABEL_1, LABEL2);;directory version;user1;Added file element file.cpp;
        
        ccRecord = CCRecord()
        ccRecord.date = time.strptime(fields[0], CC_DATE_FORMAT)
        ccRecord.path = os.path.normpath(fields[1]);
        ccRecord.revision = fields[2];
        ccRecord.operation = fields[3];
        ccRecord.labels = self.parseLabels(fields[4]);
        ccRecord.type = fields[6];
        ccRecord.author = fields[7];
        ccRecord.comment = try_to_decode(fields[8])
	ccRecord.dbid = fields[9];
        
        revisionParts = ccRecord.revision.split('/')
        if len(revisionParts) > 0:
            ccRecord.branchNames = revisionParts[1:-1]
            ccRecord.revNumber = revisionParts[-1]
        else:
            ccRecord.branchNames = []
            ccRecord.revNumber = "-1"
        return ccRecord

def writeContentLength(out, len):
    out.write("Content-length: " + str(len) + "\n");

def writeTextContentLength(out, len):
    out.write("Text-content-length: " + str(len) + "\n");

def writeNodePath(out, nodePath):
    out.write("Node-path: " + try_to_decode(nodePath).encode('utf8') + "\n");

def writeNodeKind(out, nodeKind):
    out.write("Node-kind: " + nodeKind + "\n");

def writeNodeAction(out, nodeAction):
    out.write("Node-action: " + nodeAction + "\n");

def calculateLengthAndChecksum(filename):
    textContentLength = 0;
    md = hashlib.md5()        
    file = open(filename, 'rb')
    while 1:
        s = file.read(FILEREAD_CHUNKSIZE)
        if s: 
            md.update(s)
            textContentLength += len(s)
        else: break
    file.close()
    checksum = md.hexdigest()
    return (textContentLength, checksum)

def writeContent(out, filename):
    file = open(filename, 'rb')                             
    while 1:
        s = file.read(FILEREAD_CHUNKSIZE)
        if s: out.write(s);
        else: break
    file.close()

def dumpSvnFile(out, action, path, props, contentFilename):
    writeNodePath(out, path);
    writeNodeKind(out, "file");
    writeNodeAction(out, action);
    
    props.writeLength(out);
    
    textContentLength, checksum = calculateLengthAndChecksum(contentFilename);
    writeTextContentLength(out, textContentLength);
    out.write("Text-content-md5: " + checksum + "\n");
    writeContentLength(out, textContentLength + props.totalLen );
    out.write("\n");
    
    props.writeContent(out);
    
    writeContent(out, contentFilename);
    out.write("\n\n");

def dumpSvnCopy(out, kind, copyfromPath, copyfromRev, target):
    writeNodePath(out, target);
    writeNodeKind(out, kind);
    writeNodeAction(out, "add");
    out.write("Node-copyfrom-rev: " + str(copyfromRev) + "\n");    
    out.write("Node-copyfrom-path: " + toUTF8(copyfromPath) + "\n");
    out.write("\n");        
        
def dumpSvnDir(out, path):
    writeNodePath(out, path);
    writeNodeKind(out, "dir");
    writeNodeAction(out, "add");
    out.write("\n");    

def dumpSvnDelete(out, path):
    writeNodePath(out, path);    
    writeNodeAction(out, "delete");
    out.write("\n");                

       
def getSvnBranchPath(branch):
    return "branches/" + branch

def getSvnTagPath(tag):
    return "tags/" + tag

class SvnRevisionProps:
    def __init__(self):
        self.properties = SvnProperties()
    
    def reset(self):
        self.properties.reset()
            
    def dump(self, out):
        self.properties.dump(out);
        
    def setAuthor(self, author):
        self.properties.set("svn:author", toUTF8(author));
    
    def setDate(self, date):
        self.properties.set("svn:date", time.strftime(SVN_DATE_FORMAT, date))
    
    def setMessage(self, message):
        self.properties.set("svn:log", toUTF8(message));
        
    def setCCRevision(self, ccrevision):
        self.properties.set("ClearcaseRevision", ccrevision);
        
    def setCCLabels(self, cclabels):
        labelStr = ", ".join(cclabels)
        self.properties.set("ClearcaseLabels", labelStr);


class FileSet(set):
    def __init__(self, root):
        self.root = root
        
    def getAbsolutePath(self, path):
        return self.root + "/" + path

class WriteStream:
    def __init__(self, file):
        self.enabled = True
        self.file = file
        pass
    
    def enable(self):
        self.enabled = True
    
    def disable(self):
        self.enabled = False
        
    def disabled(self):
        return self.enabled == False
        
    def write(self, data):
        if self.enabled:
            self.file.write(data)
    
               
class Converter:
    def __init__(self, dumpfile, labels, branches, autoProps):
        self.autoProps = autoProps        
        self.labels = labels                
        if self.labels is not None:
            self.checklabels = self.labels
        else:
            self.checklabels = set()    
        self.branches = branches
        self.out = WriteStream(dumpfile)
        
        self.svnTree = {} # branch/label -> FileSet
        self.ccTree = set() # (ccpath, ccrev)
        self.svnRevNum = 1 
        self.cachedir = CACHE_DIR
        self.revProps = SvnRevisionProps()       
        
        self.out.write("SVN-fs-dump-format-version: 2\n\n")
        
        if DUMP_SINCE_DATE is not None:
            self.out.disable()
        
        if SVN_CREATE_BRANCHES_TAGS_DIRS:
            self.dumpRevisionHeader()
            dumpSvnDir(self.out, getSvnBranchPath(""))
            dumpSvnDir(self.out, getSvnTagPath(""))
        
    def dumpRevisionHeader(self):
        self.out.write("Revision-number: " + str(self.svnRevNum) + "\n");
        self.svnRevNum += 1
        self.revProps.dump(self.out)
    
    def setRevisionProps(self, ccRecord):
        # self.revProps.reset() - not required since we are overwriting the same keys each time        
        self.revProps.setMessage(ccRecord.comment)
        self.revProps.setAuthor(ccRecord.author)
        self.revProps.setDate(ccRecord.date)    
        self.revProps.setCCRevision(ccRecord.revision)
        #self.revProps.setCCLabels(ccRecord.labels)
        
    def dumpFile(self, ccRecord, action, symlink=False):
        contentFilename = self.getFile(ccRecord.path, ccRecord.revision, symlink)
        props = self.autoProps.getProps(ccRecord.svnpath)
        if symlink and action is "add":
            props.set("svn:special", "*")
        dumpSvnFile(self.out, action, ccRecord.svnpath, props, contentFilename)
        
    def createParentDirs(self, fileSet, path):
        dir = os.path.dirname(path)
        if dir and dir not in fileSet:
            self.createParentDirs(fileSet, dir)
            dirpath = fileSet.getAbsolutePath(dir)
            dumpSvnDir(self.out, dirpath)
            fileSet.add(dir)
    
    def getTagFileset(self, label):
        fileSet = self.svnTree.get(label)
        if fileSet is None:
            fileSet = FileSet(getSvnTagPath(label))
            self.svnTree[label] = fileSet
            dumpSvnDir(self.out, fileSet.root)
        return fileSet
    
    def processLabels(self, ccRecord, updateLabels=True):
        self.ccTree.add( (ccRecord.path, ccRecord.revision) )
        first = True
        copyfromRev = self.svnRevNum-1
        for cclabel in ccRecord.labels:
            if self.labels is None or cclabel in self.labels:
                if first:
                    self.dumpRevisionHeader()
                    first = False
                    
                fileSet = self.getTagFileset(cclabel)
                self.createParentDirs(fileSet, ccRecord.path)
                copyfromPath = ccRecord.svnpath
		
		if (not CC_LABELS_STRUCTURE.has_key(cclabel)):
                    self.saveConfigSpec(CCVIEW_CONFIGSPEC)
		    labelFilename = self.getLabelContent(cclabel)
		    self.setLabelSpec(cclabel)
		    labelList = {}
		    with open(labelFilename, 'r') as file:
		        for l in file:
		            inode, fileLocationUnderLabel = l.strip().split(";")
			    if labelList.has_key(inode):
				    labelList[inode].append(fileLocationUnderLabel)
			    else:
                               first=[fileLocationUnderLabel]
			       labelList[inode]=first
		    CC_LABELS_STRUCTURE[cclabel]=labelList
		    self.setConfigSpec(CCVIEW_CONFIGSPEC)
		
		for versionElement in  CC_LABELS_STRUCTURE[cclabel][ccRecord.dbid]:
		    copytoPath = fileSet.getAbsolutePath(versionElement)
                    dumpSvnCopy(self.out, "file", copyfromPath, copyfromRev, copytoPath)
		
                
                
                if self.labels is None and updateLabels:
                    self.checklabels.add(cclabel) # will be used in completeLabels phase
        pass
        
    def process(self, ccRecord):
        #    OPERATION;TYPE
        #    checkin;directory version
        #    checkin;version
        #    mkbranch;directory version
        #    mkbranch;version
        #    mkelem;directory version - means version 0
        #    mkelem;version    - means version 0
        #    mkslink;symbolic link
        # not of interest:
        #    **null operation kind**;file element
        #    checkout;directory version
        #    checkout;version
        #    lock;branch
        #    mkbranch;branch
        #    mkelem;branch
        #    mkelem;directory element
        #    mkelem;file element
        
        if ccRecord.path == ".": return
        
        type = ccRecord.type
        operation = ccRecord.operation
        
        ccRecord.svnbranch = len(ccRecord.branchNames) > 0 and ccRecord.branchNames[-1] or "unknown" 
        ccRecord.svnpath = getSvnBranchPath(ccRecord.svnbranch) + "/" + ccRecord.path
        
        if self.branches is not None and ccRecord.svnbranch not in self.branches:
            return            
                
        if DUMP_SINCE_DATE is not None and self.out.disabled() and ccRecord.date > DUMP_SINCE_DATE:
            self.out.enable()
            
        self.setRevisionProps(ccRecord)
        
        if type == "version": # file
            if operation == "checkin" or operation == "mkbranch" or operation == "mkelem":
                # create or modify file                 
                branchFileSet = self.svnTree.get(ccRecord.svnbranch)
                if branchFileSet is not None:
                    # branch is already known
                    self.dumpRevisionHeader()
                    if ccRecord.path in branchFileSet:
                        # file is already in the set - svn modify
                        self.dumpFile(ccRecord, "change")
                        pass
                    else:
                        # new file in branch - svn add
                        self.createParentDirs(branchFileSet, ccRecord.path)                      
                        self.dumpFile(ccRecord, "add")
                        branchFileSet.add(ccRecord.path)
                        pass
                    
                    self.processLabels(ccRecord)
                    pass
                else:
                    # new branch
                    copyfromRev = self.svnRevNum - 1                    
                    self.dumpRevisionHeader()                    
                    if len(ccRecord.branchNames) < 2:
                        # new top level branch
                        newBranchFileSet = FileSet(getSvnBranchPath(ccRecord.svnbranch))
                        self.svnTree[ccRecord.svnbranch] = newBranchFileSet     
                        
                        dumpSvnDir(self.out, newBranchFileSet.root)                     
                        pass
                    else:
                        parentSvnBranch = ccRecord.branchNames[-2]
                        
                        parentBranchFileSet = self.svnTree.get(parentSvnBranch)
                        if parentBranchFileSet:
                            # operation - svn cp
                            copyfromPath = getSvnBranchPath(parentSvnBranch)
                            copytoPath = getSvnBranchPath(ccRecord.svnbranch)
                            
                            newBranchFileSet = parentBranchFileSet.copy()
                            newBranchFileSet.root = copytoPath 
                            self.svnTree[ccRecord.svnbranch] = newBranchFileSet
                                                        
                            dumpSvnCopy(self.out, "dir", copyfromPath, copyfromRev, copytoPath)
                            
                        else:
                            error("ClearCase history is corrupted: child branch appeared before the parent one for file " +
                                  ccRecord.path + "@@" + ccRecord.revision)                            
                            if askYesNo("Create branch anyway and ignore the error? (or exit)"):
                                newBranchFileSet = FileSet(getSvnBranchPath(ccRecord.svnbranch))
                                self.svnTree[ccRecord.svnbranch] = newBranchFileSet
                                dumpSvnDir(self.out, newBranchFileSet.root)
                            else:
                                sys.exit(1)
                        pass
                                    
                    if ccRecord.path in newBranchFileSet:
                        # file is already in the set - svn modify if cc version is not 0
                        if ccRecord.revNumber != "0":
                            self.dumpFile(ccRecord, "change")
                        pass
                    else:
                        # new file in branch - svn add
                        self.createParentDirs(newBranchFileSet, ccRecord.path)                      
                        self.dumpFile(ccRecord, "add")
                        newBranchFileSet.add(ccRecord.path)
                        pass
                    
                    self.processLabels(ccRecord)
                    pass                                
                pass
            
        elif type == "directory version":
            if operation == "checkin" or operation == "mkbranch" or operation == "mkelem":
                # new or modify dir
                branchFileSet = self.svnTree.get(ccRecord.svnbranch)
                if branchFileSet:
                    # branch is already known
                    if ccRecord.path in branchFileSet:
                        # dir is already in the set - it must be adding or removing some files
                        # if some file is removed from the dir - we will not get any history for it
                        # unless it resurrected after - we will ignore this case
                        pass
                    else:
                        # new dir in the branch
                        self.dumpRevisionHeader()
                        self.createParentDirs(branchFileSet, ccRecord.path)
                        dumpSvnDir(self.out, ccRecord.svnpath)                        
                        branchFileSet.add(ccRecord.path)
                        pass
                    # save the dir version in cc tree for label processing stage
                    self.ccTree.add( (ccRecord.path, ccRecord.revision) )
                    pass
                else:
                    # new branch for the dir - wait until there are files in the branch
                    # do nothing                                     
                    pass
                pass
        elif type == "symbolic link" and operation == "mkslink":
            # just get the latest version of the file - we can not track the history of the link
            ccRecord.svnbranch = PUT_CCLINKS_TO_BRANCH
            ccRecord.svnpath = getSvnBranchPath(ccRecord.svnbranch) + "/" + ccRecord.path
            branchFileSet = self.svnTree.get(ccRecord.svnbranch)
            if branchFileSet is not None:
                if ccRecord.path in branchFileSet:
                    self.dumpRevisionHeader()
                    self.dumpFile(ccRecord, "change", symlink=True)                        
                    pass
                else:
                    self.dumpRevisionHeader()
                    self.createParentDirs(branchFileSet, ccRecord.path)
                    self.dumpFile(ccRecord, "add", symlink=True)                
                    branchFileSet.add(ccRecord.path)
                    pass
                pass
            else:
                warn("The branch " + ccRecord.svnbranch + " does not exists. Skip the link " + ccRecord.path)
                pass
            pass
        pass
    
    def getFile(self, path, revision, symlink=False):
        ccfile = path
        localfile = os.path.normpath(self.cachedir + "/" + path)
        if revision: 
            ccfile = ccfile + "@@" + revision
            localfile = os.path.normpath(localfile + "/" + revision)
        localfileDir = os.path.dirname(localfile)
        if not os.path.exists(localfileDir):
            os.makedirs(localfileDir, mode=0777)
        cacheExists = os.path.exists(localfile)
        if cacheExists and CHECK_ZEROSIZE_CACHEFILE:
            cacheExists = os.path.getsize(localfile) > 0
         
        if not cacheExists:
            if symlink:
                symlinkfile = os.path.normpath(CC_VOB_DIR + "/" + ccfile)
                if os.path.islink(symlinkfile):
                    content = os.readlink(symlinkfile)
                    outfile = open(localfile, 'wb')
                    outfile.write("link " + content)
                    outfile.close()
                    pass
                else:
                    raise RuntimeError("File " + symlinkfile + " is not a symbolic link")
            else: 
                cmd = CLEARTOOL + " get -to '" + string.replace(localfile, "'", "'\\''") + "' '" + string.replace(ccfile, "'", "'\\''") + "'"
                (status, out) = shellCmd(cmd, cwd=CC_VOB_DIR)
                if status == "ignore":
                    if not os.path.exists(localfile): open(localfile, 'w').close()
        return localfile
    
    def getFileDetails(self, ccrevfile):
        
        localfile = os.path.normpath(self.cachedir + "/" + ccrevfile.replace('@@', '/') + "_descr")
        localfileDir = os.path.dirname(localfile)
        if not os.path.exists(localfileDir):
            os.makedirs(localfileDir, mode=0777)
        
        outStr = ""
        cacheExists = os.path.exists(localfile) and os.path.getsize(localfile) > 0
        if cacheExists:
            with open(localfile, 'r') as file:
                for line in file:
                    outStr += line
        else:
            cmd = CLEARTOOL + " descr -fmt '" + HISTORY_FORMAT + "'  '" + ccrevfile + "'"
            (status, outStr) = shellCmd(cmd, cwd=CC_VOB_DIR)
            with open(localfile, 'w') as file:
                file.write(outStr)                    
        return outStr
        
    def getLabelContent(self, label):       
        labelFilename = os.path.join(CACHE_DIR, label)
        if not os.path.exists(labelFilename):
	    #cmd = CLEARTOOL + " find . -ver 'version(" + label + ")' -exec '/opt/ibm/RationalSDLC/clearcase/linux_x86/bin/cleartool describe -fmt \"%n;%Dn\n\" $CLEARCASE_PN'"
            cmd = CLEARTOOL + " find . -ver 'version(" + label + ")' -print"
            shellCmd(cmd, cwd=CC_VOB_DIR, outfile=labelFilename)
	    os.rename(labelFilename, labelFilename+'.tmp')
	    finalFileFd=os.open(labelFilename, os.O_RDWR|os.O_CREAT)
            with open(labelFilename+'.tmp', 'r') as file:
		for line in file:
		    fd=os.open(CC_VOB_DIR + os.sep + line.strip(), os.O_RDONLY)
		    info=os.fstat(fd)
                    os.write(finalFileFd, str(info.st_ino)+';'+line)
		    os.close(fd)
	    os.close(finalFileFd)
        return labelFilename
    
    
    def saveConfigSpec(self, file):                
        cmd = CLEARTOOL + " catcs"        
        shellCmd(cmd, outfile=file)
            
    def setConfigSpec(self, file):
        cmd = CLEARTOOL + " setcs " + file        
        shellCmd(cmd)
    
    def setLabelSpec(self, label):
        with open(CCVIEW_TMPFILE, 'w') as file:
                file.write("element * " + label + "\n")        
        self.setConfigSpec(CCVIEW_TMPFILE)        
        
    def completeLabels(self):
        # we need to add to labels those files that are not visible from ClearCase view
        # these are the files that were removed or renamed
	return
    
        info("Checking labels")
        
        parser = CCHistoryParser()
        ccRecord = CCRecord()

        if self.checklabels:
            self.saveConfigSpec(CCVIEW_CONFIGSPEC)
            
        for label in self.checklabels:   
            info("Checking " + label)
            self.setLabelSpec(label)
            try:            
                labelFilename = self.getLabelContent(label)            
                with open(labelFilename, 'r') as file:            
                    for line in file:
                        ccrevfile = line.strip()                        
                        try:
                            (path, revision) = ccrevfile.split('@@')
                        except:
                            warn("label content file " + ccrevfile + " has no revision after @@")
                            continue                                        
                        if path == ".": continue
                        path = os.path.normpath(path)
                        
                        if (path, revision) not in self.ccTree:   
                            details = self.getFileDetails(ccrevfile)
                            ccRecord = parser.processLine(details)
                            if ccRecord and ccRecord.type == "version": # file   

				if DUMP_SINCE_DATE is not None:                                
	                                if ccRecord.date > DUMP_SINCE_DATE:
        	                            self.out.enable()
                	                else:
                        	            self.out.disable()
                                                       
                                info("Found file " + path + "@@" + revision)                            
                                self.setRevisionProps(ccRecord)
                                self.dumpRevisionHeader()
                                fileSet = self.getTagFileset(label)
                                self.createParentDirs(fileSet, ccRecord.path)
                                
                                ccRecord.svnpath = fileSet.getAbsolutePath(ccRecord.path)
                                self.dumpFile(ccRecord, "add")
                                fileSet.add(ccRecord.path)
                                
                                if label in ccRecord.labels:
                                    ccRecord.labels.remove(label)                            
                                self.processLabels(ccRecord, updateLabels=False)
                            else:
                                self.ccTree.add( (path, revision) )
            except KeyboardInterrupt, e:
                raise e
            except:
                error(str(sys.exc_info()[1]))
        
        if self.checklabels:
            self.setConfigSpec(CCVIEW_CONFIGSPEC)
        pass

############# main functions ######################
    
def getCCHistory(filename):
    info("Loading CC history to " + filename)
    
    if os.path.exists(filename):
        info("File " + filename + " already exists")
        if askYesNo("Use this file?"):
            return filename        
    
    cmd = CLEARTOOL + " lshistory -recurse -fmt '" + HISTORY_FORMAT + "'"
    shellCmd(cmd, cwd=CC_VOB_DIR, outfile=filename)
    pass

def readList(filename):
    resList = None
    if filename:
        info("Reading " + filename)
        resList = set()
        with open(filename, 'r') as file:
            for line in file:
                resList.add(line.strip())
    return resList

def main():
    
    try:
    
        labels = readList(CC_LABELS_FILE)
        
        branches = readList(CC_BRANCHES_FILE)
    
        getCCHistory(HISTORY_FILE)
        
        autoProps = SvnAutoProps(SVN_AUTOPROPS_FILE)
        
        info("Processing ClearCase history, creating svn dump " + SVN_DUMP_FILE)
    
        with open(SVN_DUMP_FILE, 'wb') as dumpfile:
        
            converter = Converter(dumpfile, labels, branches, autoProps)
            
            parser = CCHistoryParser()
            
            with open(HISTORY_FILE, 'rb') as historyFile: 
                for line in rlines(historyFile): # reading lines in reverse order
                    ccRecord = parser.processLine(line)
                    if ccRecord:
                        converter.process(ccRecord)
            
            converter.completeLabels()
        
        info("Completed")
            
    except SystemExit:
        info("Exiting")
    except KeyboardInterrupt, e:
        error("Interrupted by user")    
    except:
        error(str(sys.exc_info()[1]))
    
if __name__ == "__main__": 
    main()