From b98faf20fa00969410d7a73df45e6213d9dbc6f9 Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 28 Oct 2011 14:40:33 -0200 Subject: [PATCH 01/99] Boot Manager - Gerenciador de instacao de bootloaders --- carbono/boot_manager/__init__.py | 11 ++ carbono/boot_manager/boot_manager.py | 101 ++++++++++++ carbono/boot_manager/disk_utils.py | 237 +++++++++++++++++++++++++++ carbono/boot_manager/grub2.py | 39 +++++ carbono/boot_manager/grub_legacy.py | 169 +++++++++++++++++++ carbono/boot_manager/syslinux.py | 210 ++++++++++++++++++++++++ carbono/boot_manager/utils_misc.py | 83 ++++++++++ 7 files changed, 850 insertions(+) create mode 100644 carbono/boot_manager/__init__.py create mode 100644 carbono/boot_manager/boot_manager.py create mode 100644 carbono/boot_manager/disk_utils.py create mode 100644 carbono/boot_manager/grub2.py create mode 100644 carbono/boot_manager/grub_legacy.py create mode 100644 carbono/boot_manager/syslinux.py create mode 100644 carbono/boot_manager/utils_misc.py diff --git a/carbono/boot_manager/__init__.py b/carbono/boot_manager/__init__.py new file mode 100644 index 0000000..5b6789c --- /dev/null +++ b/carbono/boot_manager/__init__.py @@ -0,0 +1,11 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella +from carbono.boot_manager.boot_manager import BootManager +#from carbono.boot_manager.grub_legacy import GrubLegacy +#from carbono.boot_manager.utils import Utils +#from carbono.boot_manager.disk_utils import DiskUtils +#from carbono.boot_manager.grub2 import Grub2 +#from carbono.boot_manager.syslinux import Syslinux diff --git a/carbono/boot_manager/boot_manager.py b/carbono/boot_manager/boot_manager.py new file mode 100644 index 0000000..352c949 --- /dev/null +++ b/carbono/boot_manager/boot_manager.py @@ -0,0 +1,101 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella + +from carbono.boot_manager.grub_legacy import GrubLegacy +from carbono.boot_manager.grub2 import Grub2 +from carbono.boot_manager.syslinux import Syslinux +from carbono.boot_manager.disk_utils import * +from carbono.boot_manager.utils_misc import * +from carbono.log import log +#from shutil import move as moveFile +#from shutil import copy2 as copyFile + +class BootManager: + MOUNT_OPTIONS = "" + GRUB2_OPTIONS = "" + + def __init__(self, source_device, boot_man="grub", make_config_file=False, chroot_option=False): + self.device = source_device.strip() + self.boot_manager = boot_man.lower() + self.boot_manager = self.boot_manager.strip() + self.chroot_option = chroot_option + self.mounted_devices = [] + self.real_root = None + self.dev_boot_tuples = None + self.make_config_file = make_config_file + + def install_boot_manager(self, boot_only_dev=None, dev_boot_tuples=None): + #dev_boot_tuples --> lista de duplas do tipo (particao_boot_sistema1, particao_sistema1) + + self.dev_boot_tuples = dev_boot_tuples + directory = mount_partition(self.device) + boot_folder = directory+"boot" + + if (boot_only_dev is not None) and (boot_only_dev): + mount_partition(boot_only_dev,directory+"boot") + + if "grub2" in self.boot_manager: + boot_opt = Grub2() + if self.chroot_option: + directory,self.real_root = chroot(directory) + if self.make_config_file: + #on live cd, this only works under chroot + if not self.chroot_option: + directory,self.real_root = chroot(directory) + if boot_opt.build_grub2_menu(directory): + log.warning("Grub2 config file generated") + if boot_opt.install_grub2(self.device, directory): + log.warning("Grub2 successfully installed") + else: + log.error("Installing Grub2 failed") + + elif "grub" in self.boot_manager: + boot_opt=GrubLegacy() + + if self.chroot_option: + directory,self.real_root = chroot(directory) + boot_folder="/boot" + version=grub_version() + if ("grub-legacy" in version) and version: + if self.make_config_file: + boot_opt.build_grub_legacy_menu(boot_folder,dev_boot_tuples) + if boot_opt.install_grub_legacy(self.device, directory): + log.warning("Grub-Legacy successfully installed") + else: + log.error("Installing Grub-Legacy failed") + else: + boot_opt = Grub2() + log.error("Grub-legacy is not installed in {0}".format(self.device)) + log.warning("INSTALING GRUB2 INSTEAD") + if self.make_config_file: + if boot_opt.build_grub2_menu(directory): + log.warning("Grub2 config file generated") + if boot_opt.install_grub2(self.device, directory): + log.warning("Grub2 successfully installed") + else: + log.error("Installing Grub2 failed") + + elif "syslinux" in self.boot_manager: + boot_opt = Syslinux() + if self.chroot_option: + directory,self.real_root=chroot(directory) + boot_folder="/boot" + if self.make_config_file: + + boot_opt.build_sislinux_menu(boot_folder,dev_boot_tuples) + if boot_opt.install_syslinux(self.device,directory): + log.warning("Syslinux successfully installed") + else: + log.error("Installing Syslinux failed") + + if self.chroot_option: + try: + undo_chroot(self.real_root) + except: + log.error("chroot exit failed") + + umount_all_devices() + umount_all_devices_system() diff --git a/carbono/boot_manager/disk_utils.py b/carbono/boot_manager/disk_utils.py new file mode 100644 index 0000000..c8ec345 --- /dev/null +++ b/carbono/boot_manager/disk_utils.py @@ -0,0 +1,237 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella + +import os +import parted +from carbono.device import Device +from carbono.disk import Disk +from carbono.utils import * +from carbono.boot_manager.utils_misc import * +from carbono.log import log + +mounted_devices = [] +MOUNT_OPTIONS = "" + +def list_devices(device): + devices_linux = [] + devices_windows = [] + #device = device[:8] + device="/dev/{0}".format(device) + try: + p = parted.Device(device) + disk = parted.Disk(p) + for i in disk.partitions: + if "ext" in "{0}".format(i.fileSystem): + devices_linux.append("{0}".format(i.path)) + elif ("ntfs" in "{0}".format(i.fileSystem)) or ("fat" in "{0}".format(i.fileSystem)): + devices_windows.append("{0}".format(i.path)) + + devices_linux.sort() + bubble_sort(devices_linux) + + return devices_linux, devices_windows + except: + return None,None + +def get_filesystem_type(device): + + dev = device[:8] + p = parted.Device(dev) + t = parted.Disk(p) + pt = parted.Disk.getPrimaryPartitions(t) + ptl = parted.Disk.getLogicalPartitions(t) + for i in pt: + #scan the primary partitions + if device in i.path: + if "ext" in "{0}".format(i.fileSystem): + fileS = "ext" + elif ("ntfs" in "{0}".format(i.fileSystem)): + fileS = "win" + elif ("fat" in "{0}".format(i.fileSystem)): + fileS = "fat" + elif "swap" in "{0}".format(i.fileSystem): + fileS = "swap" + else: + fileS = "other" + return fileS + for i in ptl: + #scan the logical partitions + if device in i.path: + if "ext" in "{0}".format(i.fileSystem): + fileS = "ext" + elif ("ntfs" in "{0}".format(i.fileSystem)): + fileS = "win" + elif ("fat" in "{0}".format(i.fileSystem)): + fileS = "fat" + elif "swap" in "{0}".format(i.fileSystem): + fileS = "swap" + else: + fileS = "other" + return fileS + #print "\nsda{0} --> {1} --> {2}\n".format(i.path,i.type,fileS) + + +def mount_device(device,folder): + ''' + Mount devices (as /dev, /sys) + Usefull to mount chroot needed devices + ''' + if folder is None: + log.error("No folder as received as mounting point") + if device is None: + log.error("No device as received to mount") + output,erro,ret = run_simple_command_echo("mount -o bind /{0} {1}{0}".format(device, folder),True) + if ret is not 0: + log.error("error output={0} command output={1}".format(erro,output)) + raise ErrorMountingFilesystem + mounted_devices.insert(0,"{0}{1}".format(folder,device)) + +def mount_partition(device, folder = None): + ''' + Monta a particao recebida.(Ex.: /dev/sda1) + Caso 'folder' nao tenha um destino valido, + a particao eh montada em uma pasta temporaria + ''' + mount_device = None + if folder is None: + tmpd = make_temp_dir() + else: + tmpd = folder + + if device is None: + log.error("No device given") + return False + + output,erro,ret = run_simple_command_echo("mount {0} {1} {2}".format(MOUNT_OPTIONS, + device, tmpd),True) + + if "already" in erro: + log.error("{0} is arealdy exclusively mounted.".format(mount_device)) + log.info("Trying to umount device above, and re-run") + umount_real(mount_device) + output,erro,ret = run_simple_command_echo("mount {0} {1} {2}".format(MOUNT_OPTIONS, + mount_device, tmpd),True) + + if ret is not 0: + log.error("{0}".format(erro)) + + #raise ErrorMountingFilesystem + if folder is not None: + mounted_devices.insert(0, folder) + else: + mounted_devices.append(tmpd) + return tmpd + +def umount_partition(directory): + ''' + Desmonta a ultima particao montada + ou a pasta/particao recebida + ''' + if directory is None: + log.error("directory to umount not received") + return None + run_simple_command_echo("sync") + output,erro,ret = run_simple_command_echo("umount {0}".format(directory),True) + + if ret is 0: + #pops the mounted folder from the list + for i in mounted_devices: + if directory in i: + mounted_devices.pop(mounted_devices.index(i)) + else: + log.error("error output={0} command output={1}".format(erro,output)) + + return ret + + +def get_boot_flag(device): + ''' return if the device given has the boot flag or not''' + dev = device[:8] + p = parted.Device(dev) + t = parted.Disk(p) + cont=0 + for i in t.partitions: + if i.getFlag(parted.PARTITION_BOOT): + cont+=1 + if device in i.path: + aux = i.getFlag(parted.PARTITION_BOOT) + if cont >1: + return False + else: + return aux + +def unset_boot_flag(device): + ''' unset the boot flag in the given device''' + dev = device[:8] + p = parted.Device(dev) + t = parted.Disk(p) + + for i in t.partitions: + if device in i.path: + aux = i.unsetFlag(parted.PARTITION_BOOT) + break + t.commit() + return aux + +def unset_all_boot_flags(device): + ''' unset all the boot flags''' + dev = device[:8] + p = parted.Device(dev) + t = parted.Disk(p) + for i in t.partitions: + i.unsetFlag(parted.PARTITION_BOOT) + t.commit() + + +def set_boot_flag(device): + ''' set the boot flag in the given device''' + dev = device[:8] + p = parted.Device(dev) + t = parted.Disk(p) + for i in t.partitions: + if device in i.path: + aux = i.setFlag(parted.PARTITION_BOOT) + break + t.commit() + return aux + + +def umount_all_devices(): + for i in mounted_devices: + ret = umount_partition(i) + if ret is not 0: + log.warning("umount device {0} failed".format(i)) + #raise ErrorUmountingFilesystem + +def umount_real(device): + while True: + output,err,ret = run_simple_command_echo("umount {0}".format(device)) + if ("not" in err) or ("busy" in err): + break + +def umount_all_devices_system(): + mount_cmd,err,ret = run_simple_command_echo("mount") + for j in ["a","b","c","d"]: + for i in range(1,20): + if "/dev/sd{0}{1}".format(j,i) in mount_cmd: + umount_real("/dev/sd{0}{1}".format(j,i)) + +def chroot(root): + real_root = os.open("/", os.O_RDONLY) + mount_device("dev",root) + mount_device("sys",root) + mount_device("proc",root) + os.chdir(root) + log.info("chroot {0}".format(root)) + os.chroot(root) + return "/",real_root + +def undo_chroot(real_root): + log.info("Exiting chroot") + os.fchdir(real_root) + os.chroot(".") + os.close(real_root) + os.chdir("/") diff --git a/carbono/boot_manager/grub2.py b/carbono/boot_manager/grub2.py new file mode 100644 index 0000000..c465403 --- /dev/null +++ b/carbono/boot_manager/grub2.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella + +from carbono.utils import * +from carbono.boot_manager.utils_misc import * +from carbono.boot_manager.disk_utils import * +from carbono.log import log + +class Grub2: + + def build_grub2_menu(self, directory, arq=None): + + boot_folder = directory+"boot" + if arq is None: + arq = "grub.cfg" + output,erro,ret = run_simple_command_echo("grub-mkconfig -o \ + {0}/grub/{1}".format(boot_folder,arq),True) + if ret is not 0: + log.error("Generating Grub2 config file failed") + log.error("error output={0} command output={1}".format(erro,output)) + return False + else: + return True + + def install_grub2(self, device, directory): + + boot_folder = directory+"boot" + dev = device[:8] + + output,erro,ret = run_simple_command_echo("grub-install \ + --root-directory={0} {1}".format(directory, dev),True) + if ret is not 0: + log.error("error output={0} command output={1}".format(erro,output)) + return False + else: + return True diff --git a/carbono/boot_manager/grub_legacy.py b/carbono/boot_manager/grub_legacy.py new file mode 100644 index 0000000..5441b72 --- /dev/null +++ b/carbono/boot_manager/grub_legacy.py @@ -0,0 +1,169 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella + +from carbono.utils import * +from carbono.boot_manager.utils_misc import * +from carbono.boot_manager.disk_utils import * +from carbono.log import log + +class GrubLegacy: + + def build_grub_legacy_menu(self,boot_folder, dev_boot_tuples=None): + + ''' + search for linux kernels in all devices. search windows boot files. + built the menu.lst in the target device, also copy grub-legacy files there. + ''' + cont = 0 + menu_linux = [] + menu_windows = [] + FILE = "{0}/grub/menu.lst".format(boot_folder) + try: + f = open(FILE,"w") + except: + os.makedirs("{0}/grub/".format(boot_folder)) + ret,err,out = run_simple_command_echo("touch {0}/grub/menu.lst".format(boot_folder)) + f = open(FILE,"w") + menu_linux.append("#AUTOMAGIC GENERATED\n") + menu_linux.append("default saved\n") + menu_linux.append("timeout 10\n") + menu_linux.append("password --md5 $1$7QG0J0$l2j8MS763EKQ3u.sDdh8Z0\n") + + dev = os.listdir("/dev") + + for i in dev: + if ("sd" in i) and (len(i)==3): + devices_linux,devices_windows = list_devices(i) + if devices_linux is not None: + for lin in devices_linux: + directory = mount_partition(lin) + boot_directory = directory+"/boot" + try: + filenames = os.listdir("{0}".format(boot_directory)) + except: + filenames=None + try: + filenames2 = os.listdir("{0}".format(directory)) + except: + filenames2 = None + var_temp = None + vmlinuz = None + if filenames is not None: + filenames.sort() + boot_me = "/boot" + for aux in filenames: + if "init" in aux: + initrd = aux + if len(aux) < 12: + var_temp="" + else: + var_temp = aux[11:] # take just the kernel version + elif ((var_temp is not None) and ("vmlinuz" in aux) and (var_temp in aux)) or ("vmlinuz"==aux): + vmlinuz = aux + cont+=1 + if (filenames2 is not None) and (vmlinuz is None): + filenames2.sort() + boot_me = "" + for aux in filenames2: + if "init" in aux: + initrd = aux + if len(aux) < 12: + var_temp="" + else: + var_temp = aux[11:] # take just the kernel version + elif ((var_temp is not None) and ("vmlinuz" in aux) and (var_temp in aux)) or ("vmlinuz"==aux): + vmlinuz = aux#+"MAOEEE" + cont+=1 + + if (vmlinuz is not None): + line = distro_name(directory) + if line is None: + line = cont + menu_linux.append("\ntitle Linux {0} in {1}\n".format(line,lin)) + menu_linux.append("root (hd0,{0})\n".format(int(lin[8:])-1)) + temp = False + + if (dev_boot_tuples is not None): + for i in dev_boot_tuples: + if lin in i[0]: + temp = True + menu_linux.append("kernel {0}/{1} root={2} ro quiet splash\ + \n".format(boot_me,vmlinuz,i[1])) + dev_boot_tuples.pop(dev_boot_tuples.index(i)) + if not dev_boot_tuples: + dev_boot_tuples = None + if not temp: + #if temp_dev is None: + menu_linux.append("kernel {0}/{1} root={2} ro quiet splash\ + \n".format(boot_me,vmlinuz,lin)) + #else: + # menu_linux.append("kernel {0}/{1} root = {2} ro quiet splash\ + # \n".format(boot_me,vmlinuz,temp_dev)) + # temp_dev = None + menu_linux.append("initrd {0}/{1}\n".format(boot_me,initrd)) + menu_linux.append("quiet\n") + menu_linux.append("savedefault\n") + else: + temp_dev = lin + + umount_partition(directory) + cont = 0 + for win in devices_windows: + cont+= 1 + filenames = None + directory = mount_partition(win) + try: + filenames = os.listdir("{0}".format(directory)) + except: + log.info("Folder {0} didn't exist or it'a empty".format(directory)) + + if filenames is not None: + bootWin = False + for i in filenames: + if "boot" in i.lower(): + bootWin = True + if bootWin: + windev = int(win[8:]) + + menu_windows.append("\ntitle Windows {0}\n".format(cont)) + ''' + the two line below should exist if windows is on sdb + ''' + if "1" not in win: + #if windows is not in the firs filesystem + menu_windows.append("map (hd0) (hd{0})\n".format(windev)) + menu_windows.append("map (hd{0}) (hd0)\n".format(windev)) + menu_windows.append("rootnoverify (hd0,{0})\n".format(windev-1)) + menu_windows.append("makeactive\n") + menu_windows.append("chainloader +1\n") + + for i in menu_linux: + f.write(i) + for i in menu_windows: + f.write(i) + f.close() + + def install_grub_legacy(self, device, directory): + ''' + install grub-legacy in sdX, giving de mounted device as the root device + ''' + boot_folder = directory+"boot" + dev = device[:8] + try: + output,erro,ret = run_simple_command_echo("grub-install \ + --root-directory={0} {1}".format(directory, dev),True) + except: + run_simple_command_echo("grep -v rootfs /proc/mounts > /etc/mtab") + output,erro,ret = run_simple_command_echo("grub-install \ + --root-directory={0} {1}".format(directory, dev),True) + if ret is not 0: + log.error("Grub installation failed. [grub-install \ + --root-directory={0} {1}]".format(directory, dev)) + log.error("error output={0} command output={1}".format(erro,output)) + return False + else: + return True + diff --git a/carbono/boot_manager/syslinux.py b/carbono/boot_manager/syslinux.py new file mode 100644 index 0000000..8d049d7 --- /dev/null +++ b/carbono/boot_manager/syslinux.py @@ -0,0 +1,210 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella + +from shutil import copy2 as copyFile + +from carbono.utils import * +from carbono.boot_manager.disk_utils import * +from carbono.boot_manager.utils_misc import * +from carbono.log import log + +class Syslinux: + + def build_sislinux_menu(self,boot_folder,dev_boot_tuples = None): + + KERNEL_OPTIONS="ro quiet"#if trying to boot in a netbook, add boot=hdd + cont = 0 + contador_win=0 + try: + os.listdir("{0}/syslinux/".format(boot_folder)) + + except: + os.makedirs("{0}/syslinux/".format(boot_folder)) + FILE = "{0}/syslinux/syslinux.cfg".format(boot_folder) + try: + f = open(FILE,"w") + except: + ret,err,out = run_simple_command_echo("touch {0}/syslinux/syslinux.cfg".format(boot_folder)) + f = open(FILE,"w") + + menu_linux = [] + menu_windows = [] + menu_linux.append("UI menu.c32\n") + menu_linux.append("PROMPT 1\n") + menu_linux.append("TIMEOUT 100\n") + contador_label=0 + + dev = os.listdir("/dev") + for i in dev: + if ("sd" in i) and (len(i)==3): + + devices_linux,devices_windows = list_devices(i) + if devices_linux is not None: + for lin in devices_linux: + + directory = mount_partition(lin) + boot_directory = directory+"/boot" + try: + filenames = os.listdir("{0}".format(boot_directory)) + except: + filenames=None + try: + filenames2 = os.listdir("{0}".format(directory)) + except: + filenames2 = None + var_temp = None + vmlinuz = None + if filenames is not None: + filenames.sort() + boot_me = "/boot/" + for aux in filenames: + if "init" in aux: + initrd = aux + if len(aux) < 12: + var_temp="" + else: + var_temp = aux[11:] # take just the kernel version + elif (((var_temp is not None) and ("vmlinuz" in aux) and (var_temp in aux)) + or ("vmlinuz"==aux)): + vmlinuz = aux + cont+=1 + if (filenames2 is not None) and (vmlinuz is None): + filenames2.sort() + boot_me = "/" + for aux in filenames2: + if "init" in aux: + initrd = aux + if len(aux) < 12: + var_temp="" + else: + var_temp = aux[11:] # take just the kernel version + elif (((var_temp is not None) and ("vmlinuz" in aux) and (var_temp in aux)) + or ("vmlinuz"==aux)): + vmlinuz = aux#+"MAOEEE" + cont+=1 + + if (vmlinuz is not None): + line = distro_name(directory) + if line is None: + line = cont + temp = True + if (dev_boot_tuples is not None): + for i in dev_boot_tuples: + if ((lin in i[0]) or (lin in i[1])): + temp = False + if (lin not in i[0]): + contador_label+=1 + menu_linux.append("\nLABEL {0}\n".format(contador_label)) + menu_linux.append("MENU LABEL Linux {0} in {1}\n".format(line,lin)) + menu_linux.append("KERNEL {0}{1}\nAPPEND root={2} {3}\ + \n".format(boot_me,vmlinuz,i[1],KERNEL_OPTIONS)) + menu_linux.append("INITRD {0}{1}\n".format(boot_me,initrd)) + break + else: + dev_boot_tuples.pop(dev_boot_tuples.index(i)) + break + if not dev_boot_tuples: + dev_boot_tuples = None + if temp: + #if temp_dev is None: + contador_label+=1 + menu_linux.append("\nLABEL {0}\n".format(contador_label)) + menu_linux.append("MENU LABEL Linux {0} in {1}\n".format(line,lin)) + menu_linux.append("KERNEL {0}{1}\nAPPEND root={2} {3}\ + \n".format(boot_me,vmlinuz,lin,KERNEL_OPTIONS)) + menu_linux.append("INITRD {0}{1}\n".format(boot_me,initrd)) + #else: + # menu_linux.append("kernel {0}/{1} root = {2} ro quiet splash\ + # \n".format(boot_me,vmlinuz,temp_dev)) + # temp_dev = None + + + else: + temp_dev = lin + + umount_partition(directory) + cont = 0 + #fim for + if devices_windows is not None: + for win in devices_windows: + filenames = None + + directory = mount_partition(win) + try: + filenames = os.listdir("{0}".format(directory)) + except: + log.warning("Folder {0} didn't exist or it'a empty".format(directory)) + if filenames is not None: + bootWin = False + for i in filenames: + if "windows" in i.lower(): + bootWin = True + if bootWin: + windev = int(win[8:]) + contador_label+=1 + menu_windows.append("\nLABEL {0}\n".format(contador_label)) + menu_windows.append("MENU LABEL Windows\n") + menu_windows.append("COM32 chain.c32\n") + menu_windows.append("APPEND hd{0} {1}\n".format(contador_win,windev)) + contador_win+=1 + + menu_windows.append("\nLABEL Reboot\n") + menu_windows.append("MENU LABEL Reboot\n") + menu_windows.append("COM32 reboot.c32\n") + menu_windows.append("\nLABEL Shutdown\n") + menu_windows.append("MENU LABEL Shutdown\n") + menu_windows.append("COMBOOT poweroff.com\n") + for i in menu_linux: + f.write(i) + for i in menu_windows: + f.write(i) + f.close() + + + def install_syslinux(self, device, directory): + + boot_folder = directory+"boot" + dev = device[:8] + try: + log.info("cp /usr/lib/syslinux/menu.c32 {0}/syslinux".format(boot_folder)) + copyFile("/usr/lib/syslinux/menu.c32", boot_folder+"/syslinux") + log.info("cp /usr/lib/syslinux/chain.c32 {0}/syslinux".format(boot_folder)) + copyFile("/usr/lib/syslinux/chain.c32", boot_folder+"/syslinux") + log.info("cp /usr/lib/syslinux/reboot.c32 {0}/syslinux".format(boot_folder)) + copyFile("/usr/lib/syslinux/reboot.c32", boot_folder+"/syslinux") + log.info("cp /usr/lib/syslinux/poweroff.com {0}/syslinux".format(boot_folder)) + copyFile("/usr/lib/syslinux/poweroff.com", boot_folder+"/syslinux") + + except: + log.error("Error copying files") + return False + try: + aux = get_filesystem_type(device) + if "fat" in aux: + run_simple_command_echo("syslinux -d {0}/syslinux {1}".format(boot_folder,device),True) + elif "ext" in aux: + log.info("extlinux --install {0}/syslinux".format(boot_folder)) + run_simple_command_echo("extlinux --install {0}/syslinux".format(boot_folder),True) + + else: + log.error("Filesystem not accepted.") + return False + + except: + log.error("installing syslinux in {0} failed".format(device)) + return False + try: + run_simple_command_echo("dd if=/usr/lib/syslinux/mbr.bin of={0} \ + bs=440 conv=notrunc count=1".format(dev),True) + except: + log.error("dd failed") + return False + + if not get_boot_flag(device): + unset_all_boot_flags(device) + if set_boot_flag(device): + log.info("{0} has been set as bootable device".format(device)) + return True diff --git a/carbono/boot_manager/utils_misc.py b/carbono/boot_manager/utils_misc.py new file mode 100644 index 0000000..fd44c3e --- /dev/null +++ b/carbono/boot_manager/utils_misc.py @@ -0,0 +1,83 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella + +import os +import subprocess +import shlex +import re +from carbono.log import log + +def bubble_sort(list2): + #swap_test = False + for i in range(0, len(list2) - 1): + swap_test = False + for j in range(0, len(list2) - i - 1): + if len(list2[j]) > len(list2[j + 1]): + list2[j], list2[j + 1] = list2[j + 1], list2[j] # swap + swap_test = True + if swap_test == False: + break + +def distro_name(directory): + #Return the name of the linux distro, or "None" if it didn't find it + try: + expr = re.compile(r"\\+.+",re.DOTALL) + files = os.listdir("{0}etc/".format(directory)) + tmp_filei = None + tmp_filer = None + for i in files: + if "issue" in i: + tmp_filei = "{0}/etc/issue".format(directory) + elif "-release" in i: + tmp_filer = "{0}/etc/{1}".format(directory,i) + if tmp_filei is not None: + FILE = tmp_filei + fl = open(FILE,"r") + line = fl.readline() + line = expr.sub('',line) + elif tmp_filer is not None: + FILE = tmp_filer + fl = open(FILE,"r") + for l in fl: + if "DISTRIB_DESCRIPTION" in l: + line = l[20:] + line = expr.sub('',line) + line = re.sub(r'"+','',line) + line = re.sub(r'\n+','',line) + return line + except: + return None + + +def grub_version(): + try: + version = os.popen("grub-install --version").read() + except: + log.error("Grub is not installed") + return False + if "0.97" in version: + return "grub-legacy" + elif "1." in version: + return "grub2" + elif "GNU" in version: + return "grub-legacy" + + +def run_simple_command_echo(cmd, echo=False): + ''' + run a given command + returns the output, errors (if any) and returncode + ''' + if echo: + log.info("{0}".format(cmd)) + p = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE) + p.wait() + output, err = p.communicate() + ret = p.returncode + return output,err,ret From ea6a6ca9cd85d529dfa5259f5c3f68010602a185 Mon Sep 17 00:00:00 2001 From: Lucas Ces Date: Wed, 25 Jan 2012 17:56:59 -0200 Subject: [PATCH 02/99] Trocando distutils por setuptools para ter bdist_egg --- carbono/config.py | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/carbono/config.py b/carbono/config.py index 8cfb794..66842b5 100644 --- a/carbono/config.py +++ b/carbono/config.py @@ -15,6 +15,10 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, see . + +import os + + FILE_PATTERN = "{name}-{partition}-{volume}.data" BLOCK_SIZE = 1048576 # 1MB BASE_SYSTEM_SIZE = 0 #TODO (in bytes) @@ -23,6 +27,9 @@ VERSION = (0, 1, 0, "alpha", 0) def get_version(): + if ("BUILD_ID" in os.environ): + return os.environ["BUILD_ID"] + version = "%s.%s" % (VERSION[0], VERSION[1]) if VERSION[2]: version = "%s.%s" % (version, VERSION[2]) diff --git a/setup.py b/setup.py index 120fefa..7cfc538 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, see . -from distutils.core import setup +from setuptools import setup from carbono.config import get_version DEPENDS = "python2.7, partclone, ntfsprogs, btrfs-tools, e2fsprogs, genisoimage" From fb43d3041a3149afdf86f7927fc189481c6870be Mon Sep 17 00:00:00 2001 From: Lucas Alvares Gomes Date: Thu, 26 Jan 2012 08:58:16 -0200 Subject: [PATCH 03/99] Test if the processor is hyperthreading --- carbono/buffer_manager/__init__.py | 8 ++++++- carbono/buffer_manager/simple_manager.py | 28 ++++++++++++++++++++++++ carbono/buffer_manager/work_manager.py | 2 +- carbono/utils.py | 9 ++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 carbono/buffer_manager/simple_manager.py diff --git a/carbono/buffer_manager/__init__.py b/carbono/buffer_manager/__init__.py index 0e21071..8b27676 100644 --- a/carbono/buffer_manager/__init__.py +++ b/carbono/buffer_manager/__init__.py @@ -16,13 +16,19 @@ # along with this program; if not, see . from carbono.buffer_manager.work_manager import WorkManager +from carbono.buffer_manager.simple_manager import SimpleManager from carbono.buffer_manager.dummy_manager import DummyManager +from carbono.utils import * + class BufferManagerFactory: def __init__(self, read_callback, job_callback=None): if job_callback: - self._manager = WorkManager(read_callback, job_callback) + if available_processors() <= 2 and is_hyperthreading(): + self._manager = SimpleManager(read_callback, job_callback) + else: + self._manager = WorkManager(read_callback, job_callback) else: self._manager = DummyManager(read_callback) diff --git a/carbono/buffer_manager/simple_manager.py b/carbono/buffer_manager/simple_manager.py new file mode 100644 index 0000000..b126bbe --- /dev/null +++ b/carbono/buffer_manager/simple_manager.py @@ -0,0 +1,28 @@ +#!/usr/bin/python +# coding: utf-8 +# Copyright (C) 2011 Lucas Alvares Gomes +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +from carbono.buffer_manager.dummy_manager import DummyManager + +class SimpleManager(DummyManager): + + def __init__(self, read_callback, job_callback): + DummyManager.__init__(self, read_callback) + self.job = job_callback + + def put(self, data): + worked_data = self.job(data) + self.output_buffer.put(worked_data) diff --git a/carbono/buffer_manager/work_manager.py b/carbono/buffer_manager/work_manager.py index bbf4269..2242187 100644 --- a/carbono/buffer_manager/work_manager.py +++ b/carbono/buffer_manager/work_manager.py @@ -71,7 +71,7 @@ def stop(self): class WorkManager(Thread): - def __init__(self, read_callback, job_callback=None): + def __init__(self, read_callback, job_callback): Thread.__init__(self) self.read_block = read_callback self.job = job_callback diff --git a/carbono/utils.py b/carbono/utils.py index b537586..1e019e9 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -116,6 +116,15 @@ def getinstance(): def available_processors(): return multiprocessing.cpu_count() +def is_hyperthreading(): + with open("/proc/cpuinfo", "r") as f: + for line in f.readlines(): + if line.startswith("flags"): + if "ht" in line.split(): + return True + break + return False + def available_memory(percent=100): free = 0 with open("/proc/meminfo", 'r') as f: From e103586a595aa7063cffee52d4abb8a191311de8 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 4 Feb 2013 17:48:44 -0200 Subject: [PATCH 04/99] Setup.py atualizado --- setup.py | 45 ++++++++++++++++----------------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/setup.py b/setup.py index 7cfc538..d602070 100755 --- a/setup.py +++ b/setup.py @@ -1,36 +1,23 @@ #!/usr/bin/python -# coding: utf-8 -# Copyright (C) 2011 Lucas Alvares Gomes -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - +import os from setuptools import setup from carbono.config import get_version -DEPENDS = "python2.7, partclone, ntfsprogs, btrfs-tools, e2fsprogs, genisoimage" -DESC = "A hard disk imaging and recovery application" +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + -setup(name = "carbono", - version = get_version(), - author = "Lucas Alvares Gomes", - author_email = "lucasagomes@gmail.com", - url = "http://umago.info/carbono", - description = DESC, - license = "GNU GPLv2", - packages = ["carbono","carbono.buffer_manager", +setup( + name = "carbono", + version = get_version(), + author = "Bruno Casella", + author_email = "bruno.casella@gmail.com", + description = ("A hard disk imaging and recovery application"), + license = "GPL", + keywords = "network_manager dbus", + url = "http://umago.info/carbono", + packages = ["carbono","carbono.buffer_manager", "carbono.filesystem", "carbono.ui", "carbono.image_reader"], - scripts = ["scripts/carbono"], - ) - + scripts = ["scripts/carbono"], +) From 437151dedf4ca6ae4bdfe0fedac8f57b8272a574 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 5 Feb 2013 10:52:49 -0200 Subject: [PATCH 05/99] Removido verificacao de HT ao criar buffers --- carbono/buffer_manager/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carbono/buffer_manager/__init__.py b/carbono/buffer_manager/__init__.py index 8b27676..6c0e66f 100644 --- a/carbono/buffer_manager/__init__.py +++ b/carbono/buffer_manager/__init__.py @@ -25,7 +25,7 @@ class BufferManagerFactory: def __init__(self, read_callback, job_callback=None): if job_callback: - if available_processors() <= 2 and is_hyperthreading(): + if available_processors() <= 2:# and is_hyperthreading(): self._manager = SimpleManager(read_callback, job_callback) else: self._manager = WorkManager(read_callback, job_callback) From bdff96de9cd01402039ad574caaede08ef2d72b7 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 5 Feb 2013 10:53:22 -0200 Subject: [PATCH 06/99] Retirada tag -q ao criar processo do partclone, pois dava excecao --- carbono/filesystem/ext.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/carbono/filesystem/ext.py b/carbono/filesystem/ext.py index 053417b..dc3d79e 100644 --- a/carbono/filesystem/ext.py +++ b/carbono/filesystem/ext.py @@ -52,7 +52,8 @@ def get_used_size(self): def open_to_read(self): """ """ - cmd = "{0} -q -c -s {1} -o -".format(which("partclone.extfs"), self.path) + #cmd = "{0} -q -c -s {1} -o -".format(which("partclone.extfs"), self.path) + cmd = "{0} -c -s {1} -o -".format(which("partclone.extfs"), self.path) try: self.process = RunCmd(cmd) self.process.run() @@ -62,7 +63,8 @@ def open_to_read(self): def open_to_write(self, uuid=None): """ """ - cmd = "{0} -q -r -o {1} -s - ".format(which("partclone.extfs"), self.path) + #cmd = "{0} -q -r -o {1} -s - ".format(which("partclone.extfs"), self.path) + cmd = "{0} -r -o {1} -s - ".format(which("partclone.extfs"), self.path) try: self.process = RunCmd(cmd) self.process.run() From 0e9bb2f88328827994d6dd78f2ddf8112e2725df Mon Sep 17 00:00:00 2001 From: Bruno Date: Tue, 5 Mar 2013 09:36:58 -0300 Subject: [PATCH 07/99] Modulo que pega as informacoes da hd e cria particoes adicionado. Not finished, of course. --- carbono/caspart/__init__.py | 6 ++ carbono/caspart/caspart.py | 101 ++++++++++++++++++++++++++++++++++ carbono/caspart/utils_misc.py | 83 ++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+) create mode 100644 carbono/caspart/__init__.py create mode 100644 carbono/caspart/caspart.py create mode 100644 carbono/caspart/utils_misc.py diff --git a/carbono/caspart/__init__.py b/carbono/caspart/__init__.py new file mode 100644 index 0000000..247ecfe --- /dev/null +++ b/carbono/caspart/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella +from caspart import CasPart diff --git a/carbono/caspart/caspart.py b/carbono/caspart/caspart.py new file mode 100644 index 0000000..756b6ea --- /dev/null +++ b/carbono/caspart/caspart.py @@ -0,0 +1,101 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella +import re + +from carbono.log import log +from carbono.caspart.utils_misc import * + +class CasPart: + + def __init__(self, particoes = None): + ''' + recebe um dicionario de particao e tamanho em MB + eg.: {'/dev/sda1': 20480, '/dev/sda2': 1024} + ''' + if particoes is not None: + self.particoes = particoes + else: + log.error("Nao foi informada nenhuma particao a ser criada.") + + def verify_4k(self, hd = 'sda'): + ''' + Verifica o tipo de tamanho fisico dos blocos da HD + Se for 4096bytes, retorna True + Caso contrario (512 bytes) retorna False + ''' + f = open("/sys/block/{0}/queue/physical_block_size".format(hd)) + physical_block_size = f.readline() + if physical_block_size == "4096": + return True + return False + + + def fdisk_parser(self, hd='sda'): + ''' + Pega informacoes da hd a partir do comando fdisk -l + ''' + + info = {'sector_size_physical': 0, 'sector_size_logical': 0, + 'cylinders_number': 0, 'sectors_per_track': 0, + 'total_sectors': 0, 'disk_size':0} + + out,err,ret = run_simple_command_echo("fdisk -l /dev/{0}".format(hd)) + if ret is not 0: + print "Erro ao pegar informacoes do hd" + + for linha in out.splitlines(): + if "Disk /dev/{0}".format(hd) in linha: + + #split the numbers of the line into a list + #them get the last occurence, + #which is the size of the disk in bytes + info['disk_size'] = re.findall('\d+',linha)[-1] + elif "cylinders" in linha: + + #[heads (nao usado), sectors/track, cylinders, total sectors] + lst_aux = re.split('[,]',linha) + for str_aux in lst_aux: + if "sectors/track" in str_aux: + info['sectors_per_track'] = re.findall('\d+', str_aux)[-1] + elif "cylinders" in str_aux: + info['cylinders_number'] = re.findall('\d+', str_aux)[-1] + elif "total" in str_aux: + info['total_sectors'] = re.findall('\d+', str_aux)[-1] + elif "Sector size" in linha: + #[logical, physical] + info['sector_size_logical'] = re.findall('\d+',linha)[0] + #info['sector_size_physical'] = re.findall('\d+',linha)[-1] + + return info + + + def get_hd_info(self, hd = 'sda'): + + hd_info_dic = {'sector_size_physical': 0, 'sector_size_logical': 0, + 'cylinders_number': 0, 'sectors_per_track': 0, + 'total_sectors': 0, 'disk_size':0, 'ptable_type': None} + + parser_info = self.fdisk_parser(hd) + + for aux in parser_info: + hd_info_dic[aux] = parser_info[aux] + + hd_info_dic['sector_size_physical'] = 4096 if self.verify_4k(hd) else 512 + hd_info_dic['ptable_type'] = 'gpt' if self.verify_gpt(hd) else 'mbr' + + return hd_info_dic + + + def verify_gpt(self, hd = 'sda'): + ''' + Verifica se a tabela de particao é GPT + ''' + out,err,ret = run_simple_command_echo("fdisk -l /dev/{0}".format(hd)) + if ret is not 0: + print "Erro ao verificar tipo da tabela de particao. {0}".format(err) + if "gpt" in out.lower(): + return True + return False diff --git a/carbono/caspart/utils_misc.py b/carbono/caspart/utils_misc.py new file mode 100644 index 0000000..fd44c3e --- /dev/null +++ b/carbono/caspart/utils_misc.py @@ -0,0 +1,83 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Bruno Fernandes Casella + +import os +import subprocess +import shlex +import re +from carbono.log import log + +def bubble_sort(list2): + #swap_test = False + for i in range(0, len(list2) - 1): + swap_test = False + for j in range(0, len(list2) - i - 1): + if len(list2[j]) > len(list2[j + 1]): + list2[j], list2[j + 1] = list2[j + 1], list2[j] # swap + swap_test = True + if swap_test == False: + break + +def distro_name(directory): + #Return the name of the linux distro, or "None" if it didn't find it + try: + expr = re.compile(r"\\+.+",re.DOTALL) + files = os.listdir("{0}etc/".format(directory)) + tmp_filei = None + tmp_filer = None + for i in files: + if "issue" in i: + tmp_filei = "{0}/etc/issue".format(directory) + elif "-release" in i: + tmp_filer = "{0}/etc/{1}".format(directory,i) + if tmp_filei is not None: + FILE = tmp_filei + fl = open(FILE,"r") + line = fl.readline() + line = expr.sub('',line) + elif tmp_filer is not None: + FILE = tmp_filer + fl = open(FILE,"r") + for l in fl: + if "DISTRIB_DESCRIPTION" in l: + line = l[20:] + line = expr.sub('',line) + line = re.sub(r'"+','',line) + line = re.sub(r'\n+','',line) + return line + except: + return None + + +def grub_version(): + try: + version = os.popen("grub-install --version").read() + except: + log.error("Grub is not installed") + return False + if "0.97" in version: + return "grub-legacy" + elif "1." in version: + return "grub2" + elif "GNU" in version: + return "grub-legacy" + + +def run_simple_command_echo(cmd, echo=False): + ''' + run a given command + returns the output, errors (if any) and returncode + ''' + if echo: + log.info("{0}".format(cmd)) + p = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE) + p.wait() + output, err = p.communicate() + ret = p.returncode + return output,err,ret From 5a3531ebcb2d219d864a7027ced2cf3c53acb261 Mon Sep 17 00:00:00 2001 From: Bruno Date: Tue, 19 Mar 2013 14:35:22 -0300 Subject: [PATCH 08/99] Funcao verifica 4k --- carbono/utils.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/carbono/utils.py b/carbono/utils.py index 1e019e9..9eda56f 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -180,3 +180,19 @@ def check_if_root(): if os.getuid() == 0: return True return False + +def verify_4k(hd = "sda"): + ''' + Retorna o tamanho fisico do setor + ''' + try: + f = open("/sys/block/{0}/queue/physical_block_size".format(hd)) + block = f.readline() + if "4096" in block: + return(4096) + #se nao for 4K, considera-se 512 + return(512) + except Exception as e: + #nao tem HD (uma vm sem hd, por exemplo) + return(512) + From 3fa54a5a79f9fbdb5b3f10caf136945129f07d20 Mon Sep 17 00:00:00 2001 From: Bruno Date: Tue, 19 Mar 2013 17:18:18 -0300 Subject: [PATCH 09/99] Mudando hardcode do local do arquivo de log para /var/log/mstech/carbono.log --- carbono/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carbono/config.py b/carbono/config.py index 66842b5..134610c 100644 --- a/carbono/config.py +++ b/carbono/config.py @@ -22,7 +22,7 @@ FILE_PATTERN = "{name}-{partition}-{volume}.data" BLOCK_SIZE = 1048576 # 1MB BASE_SYSTEM_SIZE = 0 #TODO (in bytes) -LOG_FILE = "/var/tmp/carbono.log" +LOG_FILE = "/var/log/mstech/carbono.log" EOF = -1 VERSION = (0, 1, 0, "alpha", 0) From a0dfb534e9cb11f2d3dae31fd0bd31cb5162833b Mon Sep 17 00:00:00 2001 From: Bruno Date: Wed, 20 Mar 2013 16:02:11 -0300 Subject: [PATCH 10/99] Log de criacao com sucesso / cancelada --- carbono/image_creator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/carbono/image_creator.py b/carbono/image_creator.py index 0cb8a0c..2973bff 100644 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -225,11 +225,12 @@ def create_image(self): iso_creator.run() if self.canceled: + log.info("Creation canceled") self.notify_status("canceled", {"operation": "Create image"}) else: self.notify_status("finish") - log.info("Creation finished") + log.info("Creation finished") def stop(self): if self.active: From 345093838bee36ad7b8546a7bb7b7a4940a2d910 Mon Sep 17 00:00:00 2001 From: Bruno Date: Thu, 4 Apr 2013 12:35:23 -0300 Subject: [PATCH 11/99] Mudancas no algoritmo de calculo para 4k, e no modo que aplica as tabela. Desbilita o alinhamento da libparted. --- carbono/caspart/caspart.py | 159 ++++++++++++++++++++++++++++++++++--- 1 file changed, 147 insertions(+), 12 deletions(-) diff --git a/carbono/caspart/caspart.py b/carbono/caspart/caspart.py index 756b6ea..aebe1ac 100644 --- a/carbono/caspart/caspart.py +++ b/carbono/caspart/caspart.py @@ -4,10 +4,16 @@ # Author: MStech # Bruno Fernandes Casella import re +import parted +from _ped import partition_flag_get_by_name + +from carbono.device import Device +from carbono.disk import Disk from carbono.log import log from carbono.caspart.utils_misc import * + class CasPart: def __init__(self, particoes = None): @@ -20,6 +26,12 @@ def __init__(self, particoes = None): else: log.error("Nao foi informada nenhuma particao a ser criada.") + def _wait_devices(self, disk): + """Wait OS create all device nodes""" + for p in disk.partitions: + while not os.path.exists(p.path): + time.sleep(1) + def verify_4k(self, hd = 'sda'): ''' Verifica o tipo de tamanho fisico dos blocos da HD @@ -28,7 +40,7 @@ def verify_4k(self, hd = 'sda'): ''' f = open("/sys/block/{0}/queue/physical_block_size".format(hd)) physical_block_size = f.readline() - if physical_block_size == "4096": + if "4096" in physical_block_size: return True return False @@ -72,6 +84,17 @@ def fdisk_parser(self, hd='sda'): return info + def verify_gpt(self, hd = 'sda'): + ''' + Verifica se a tabela de particao é GPT + ''' + out,err,ret = run_simple_command_echo("fdisk -l /dev/{0}".format(hd)) + if ret is not 0: + print "Erro ao verificar tipo da tabela de particao. {0}".format(err) + if "gpt" in out.lower(): + return True + return False + def get_hd_info(self, hd = 'sda'): hd_info_dic = {'sector_size_physical': 0, 'sector_size_logical': 0, @@ -86,16 +109,128 @@ def get_hd_info(self, hd = 'sda'): hd_info_dic['sector_size_physical'] = 4096 if self.verify_4k(hd) else 512 hd_info_dic['ptable_type'] = 'gpt' if self.verify_gpt(hd) else 'mbr' - return hd_info_dic - - - def verify_gpt(self, hd = 'sda'): + return hd_info_dic + + def calculates_startofpart(self, end, size, disk='sda'): ''' - Verifica se a tabela de particao é GPT + Given the end of the last partition and the size in GB of the current + one, calculates where this one should start, returning a sector that + don't give an cilinder conflict - where the end of last partition and + the start of the new one is in the same cilinder - and also does not + get an wrong sector - where it could place a start/and of a part in the + middle of a 4096b sector. ''' - out,err,ret = run_simple_command_echo("fdisk -l /dev/{0}".format(hd)) - if ret is not 0: - print "Erro ao verificar tipo da tabela de particao. {0}".format(err) - if "gpt" in out.lower(): - return True - return False + print "to aqui" + hd_info_dict = self.get_hd_info(disk) + cylinder_end = end/int(hd_info_dict['sectors_per_track']) + cylinder_start = cylinder_end + 1 + sector_start = cylinder_start * int(hd_info_dict['sectors_per_track']) + if self.verify_4k(disk): + print sector_start + sector_start *= 512 + + print "why the luck{0}".format(sector_start % 4096) + print (sector_start % 4096) / 4096 + if (sector_start % 4096): + #sector_start += (sector_start % 4096)*4096 - (sector_start % 4096) + sector_start += ((sector_start % 4096)/4096 +1) * 4096 - (sector_start % 4096) + sector_start /= 512 + return sector_start + + + + def calculates_endofpart(self, start, size_in_gb, disk='sda'): + ''' + Given the size of the partition, gives where it should end + It doesn't get a conflictant sector, just as the function above + ''' + #hd_info_dict = get_hd_info(disk) + print disk + size = int(size_in_gb) * 1024 * 1024 *1024 #bytes + if not self.verify_4k(disk): + print "not 4k" + #print start + print size + end = int(start)*512 + size + if (end % 512): #if end is not divisable by 512b, its wrong (?) + #then, goes to the end of the next sector + end += (end % 512)*512 - (end % 512) + print end + return end / 512 + else: #4096b + print "4k" + end = int(start)*512 + size + if (end % 4096): #if end is not divisable by 4k, its wrong + #then, goes to the end of that sector + end += (end % 4096)*4096 - (end % 4096) + return end / 512 + + def partionate_disk(self, part_list = [['sda1','ntfs','30',''], + ['sda2','ext3','20','boot']]): + ''' + particiona o disco + ''' + for part in part_list: + #get the hd using the 3 first letters of the 1st elem. of the dict. + disk = "{0}".format(part[0][:3]) + break + disk_path = "/dev/{0}".format(disk) + device = Device(disk_path) + disk_obj = Disk(device) + start = end = 0 + layout = list() #objs of parted.partition will be added in the list + disk_obj.unsetFlag(parted.DISK_CYLINDER_ALIGNMENT) + for part in part_list: + #got through all elements of the received dict + + if part[0][3:] == '1': + #if is the 1st partition, then it should start at sector 2048 + start = 2048 + end = self.calculates_endofpart(start, part[2], disk) + else: + #starting from 2nd partition, we will make the calculus to where + # it will start/end using the last partition ending + start = self.calculates_startofpart(end, part[2], disk) + + end = self.calculates_endofpart(start, part[2], disk) + #length = int(part[2]) * 1024 * 1024 *1024 / device.sectorSize + length = end - start + #end = start + length - 1 + print "AQUI {0} {1} {2}".format(start,end,length) + + geometry = parted.geometry.Geometry(device = disk_obj.device, + start = start, + #length = length) + end = end) + print "pqp" + part_fs = None + fs = part[1] + print geometry + if fs is not None: + part_fs = parted.filesystem.FileSystem(type = fs, + geometry = geometry) + print part_fs + partition = parted.partition.Partition(disk=disk_obj, + type=0, + geometry=geometry, + fs=part_fs) + #TODO swap e etc ? + #partition.setFlag(partition_flag_get_by_name(p.flags)) + print partition + print disk_obj.partitionAlignment + layout.append(partition) + + disk_obj.deleteAllPartitions() + + #last_partition = len(layout) + print layout + constraint = parted.Constraint(device=disk_obj.device) + for cont, partition in enumerate(layout, 1): + disk_obj.addPartition(partition, constraint) + #could use 'cont' to know if is the last part, + #and expand it to end of the hard drive + + #commit to device, OS and wait until OS create all devices nodes + disk_obj.commitToDevice() + disk_obj.commitToOS() + self._wait_devices(disk_obj) From 4483e60f800503e4fc438f936504f0ebbcb6f678 Mon Sep 17 00:00:00 2001 From: Bruno Date: Thu, 4 Apr 2013 16:35:11 -0300 Subject: [PATCH 12/99] Tirando prints desnecessarios --- carbono/caspart/caspart.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/carbono/caspart/caspart.py b/carbono/caspart/caspart.py index aebe1ac..1079732 100644 --- a/carbono/caspart/caspart.py +++ b/carbono/caspart/caspart.py @@ -120,17 +120,12 @@ def calculates_startofpart(self, end, size, disk='sda'): get an wrong sector - where it could place a start/and of a part in the middle of a 4096b sector. ''' - print "to aqui" hd_info_dict = self.get_hd_info(disk) cylinder_end = end/int(hd_info_dict['sectors_per_track']) cylinder_start = cylinder_end + 1 sector_start = cylinder_start * int(hd_info_dict['sectors_per_track']) if self.verify_4k(disk): - print sector_start sector_start *= 512 - - print "why the luck{0}".format(sector_start % 4096) - print (sector_start % 4096) / 4096 if (sector_start % 4096): #sector_start += (sector_start % 4096)*4096 - (sector_start % 4096) sector_start += ((sector_start % 4096)/4096 +1) * 4096 - (sector_start % 4096) @@ -144,21 +139,14 @@ def calculates_endofpart(self, start, size_in_gb, disk='sda'): Given the size of the partition, gives where it should end It doesn't get a conflictant sector, just as the function above ''' - #hd_info_dict = get_hd_info(disk) - print disk size = int(size_in_gb) * 1024 * 1024 *1024 #bytes if not self.verify_4k(disk): - print "not 4k" - #print start - print size end = int(start)*512 + size if (end % 512): #if end is not divisable by 512b, its wrong (?) #then, goes to the end of the next sector end += (end % 512)*512 - (end % 512) - print end return end / 512 else: #4096b - print "4k" end = int(start)*512 + size if (end % 4096): #if end is not divisable by 4k, its wrong #then, goes to the end of that sector @@ -202,7 +190,6 @@ def partionate_disk(self, part_list = [['sda1','ntfs','30',''], start = start, #length = length) end = end) - print "pqp" part_fs = None fs = part[1] print geometry From 020fff67ab32f9a271b6693ad864b04b3edf0c36 Mon Sep 17 00:00:00 2001 From: Bruno Date: Tue, 9 Apr 2013 10:47:30 -0300 Subject: [PATCH 13/99] Adicionado linha que desliga o alinhamento que o parted faz, deixando os setores das particoes que do jeito que foram criados. 'ok parted, I known what i'm doing'. --- carbono/disk_layout_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/carbono/disk_layout_manager.py b/carbono/disk_layout_manager.py index f4cb18b..2cf1664 100644 --- a/carbono/disk_layout_manager.py +++ b/carbono/disk_layout_manager.py @@ -68,6 +68,8 @@ def _wait_devices(self, disk): def restore_from_file(self, disk, expand=False): """ """ + + disk.unsetFlag(parted.DISK_CYLINDER_ALIGNMENT) with open(self.file_path, 'r') as f: layout = list() partitions = cPickle.load(f) From 035f27955c34b13fc90f3d57ac30f6e84d21c1e6 Mon Sep 17 00:00:00 2001 From: Bruno Date: Thu, 25 Apr 2013 15:23:23 -0300 Subject: [PATCH 14/99] Adicionado heads a info do caspart --- carbono/caspart/caspart.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/carbono/caspart/caspart.py b/carbono/caspart/caspart.py index 1079732..026437f 100644 --- a/carbono/caspart/caspart.py +++ b/carbono/caspart/caspart.py @@ -50,9 +50,9 @@ def fdisk_parser(self, hd='sda'): Pega informacoes da hd a partir do comando fdisk -l ''' - info = {'sector_size_physical': 0, 'sector_size_logical': 0, - 'cylinders_number': 0, 'sectors_per_track': 0, - 'total_sectors': 0, 'disk_size':0} + info = {'sector_size_physical' : 0, 'sector_size_logical' : 0, + 'cylinders_number' : 0, 'sectors_per_track' : 0, + 'total_sectors' : 0, 'disk_size' : 0, 'heads' : 0} out,err,ret = run_simple_command_echo("fdisk -l /dev/{0}".format(hd)) if ret is not 0: @@ -67,19 +67,22 @@ def fdisk_parser(self, hd='sda'): info['disk_size'] = re.findall('\d+',linha)[-1] elif "cylinders" in linha: - #[heads (nao usado), sectors/track, cylinders, total sectors] + #[heads, sectors/track, cylinders, total sectors] lst_aux = re.split('[,]',linha) for str_aux in lst_aux: - if "sectors/track" in str_aux: + if "heads" in str_aux: + info['heads'] = re.findall('\d+', str_aux)[-1] + elif "sectors/track" in str_aux: info['sectors_per_track'] = re.findall('\d+', str_aux)[-1] elif "cylinders" in str_aux: info['cylinders_number'] = re.findall('\d+', str_aux)[-1] elif "total" in str_aux: info['total_sectors'] = re.findall('\d+', str_aux)[-1] + elif "Sector size" in linha: #[logical, physical] info['sector_size_logical'] = re.findall('\d+',linha)[0] - #info['sector_size_physical'] = re.findall('\d+',linha)[-1] + info['sector_size_physical'] = re.findall('\d+',linha)[-1] return info @@ -97,16 +100,17 @@ def verify_gpt(self, hd = 'sda'): def get_hd_info(self, hd = 'sda'): - hd_info_dic = {'sector_size_physical': 0, 'sector_size_logical': 0, - 'cylinders_number': 0, 'sectors_per_track': 0, - 'total_sectors': 0, 'disk_size':0, 'ptable_type': None} + hd_info_dic = {'sector_size_physical' : 0, 'sector_size_logical' : 0, + 'cylinders_number' : 0, 'sectors_per_track' : 0, + 'total_sectors' : 0, 'disk_size' : 0, + 'heads' : 0, 'ptable_type' : None} parser_info = self.fdisk_parser(hd) for aux in parser_info: hd_info_dic[aux] = parser_info[aux] - hd_info_dic['sector_size_physical'] = 4096 if self.verify_4k(hd) else 512 + #hd_info_dic['sector_size_physical'] = 4096 if self.verify_4k(hd) else 512 hd_info_dic['ptable_type'] = 'gpt' if self.verify_gpt(hd) else 'mbr' return hd_info_dic @@ -117,13 +121,15 @@ def calculates_startofpart(self, end, size, disk='sda'): one, calculates where this one should start, returning a sector that don't give an cilinder conflict - where the end of last partition and the start of the new one is in the same cilinder - and also does not - get an wrong sector - where it could place a start/and of a part in the + get an wrong sector - where it could place a start/end of a part in the middle of a 4096b sector. ''' hd_info_dict = self.get_hd_info(disk) - cylinder_end = end/int(hd_info_dict['sectors_per_track']) + cylinder_end = end/(int(hd_info_dict['heads']) * int(hd_info_dict['sectors_per_track'])) cylinder_start = cylinder_end + 1 - sector_start = cylinder_start * int(hd_info_dict['sectors_per_track']) + sector_start = cylinder_start * \ + int(hd_info_dict['heads']) * \ + int(hd_info_dict['sectors_per_track']) if self.verify_4k(disk): sector_start *= 512 if (sector_start % 4096): @@ -163,7 +169,11 @@ def partionate_disk(self, part_list = [['sda1','ntfs','30',''], disk = "{0}".format(part[0][:3]) break disk_path = "/dev/{0}".format(disk) - device = Device(disk_path) + device = Device(disk_path) + ''' + print device.sectorSize + print device.physicalSectorSize + ''' disk_obj = Disk(device) start = end = 0 layout = list() #objs of parted.partition will be added in the list From 21f11791bcb7f830fd303944b9adeb6c35c752c5 Mon Sep 17 00:00:00 2001 From: Bruno Date: Thu, 25 Apr 2013 15:27:48 -0300 Subject: [PATCH 15/99] Adicionado linha que desliga o alinhamento que o parted faz, deixando os setores das particoes que do jeito que foram criados --- carbono/disk_layout_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/carbono/disk_layout_manager.py b/carbono/disk_layout_manager.py index 2cf1664..935952a 100644 --- a/carbono/disk_layout_manager.py +++ b/carbono/disk_layout_manager.py @@ -24,6 +24,7 @@ from carbono.exception import * from carbono.utils import * +pergunta pra ele se vai ser embaixo ou em cima. a gnt vai levar o bolo! __all__ = ["DiskLayoutManager"] class PartitionLayout(object): @@ -69,7 +70,9 @@ def _wait_devices(self, disk): def restore_from_file(self, disk, expand=False): """ """ + #Desativa o alinhamento automatico disk.unsetFlag(parted.DISK_CYLINDER_ALIGNMENT) + with open(self.file_path, 'r') as f: layout = list() partitions = cPickle.load(f) From 740608464110c7041eb19ad909fb893e4a6056c1 Mon Sep 17 00:00:00 2001 From: Bruno Date: Fri, 26 Apr 2013 13:57:10 -0300 Subject: [PATCH 16/99] Bolo eh bom, mas nao no meio do codigo. --- carbono/disk_layout_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/carbono/disk_layout_manager.py b/carbono/disk_layout_manager.py index 935952a..eb4559f 100644 --- a/carbono/disk_layout_manager.py +++ b/carbono/disk_layout_manager.py @@ -24,7 +24,6 @@ from carbono.exception import * from carbono.utils import * -pergunta pra ele se vai ser embaixo ou em cima. a gnt vai levar o bolo! __all__ = ["DiskLayoutManager"] class PartitionLayout(object): From 23d683a7824800ccda63724854a73ea227ac3b85 Mon Sep 17 00:00:00 2001 From: Bruno Date: Thu, 20 Jun 2013 15:57:19 -0300 Subject: [PATCH 17/99] Altera a tabela de particao antes de expandir a particao --- carbono/image_restorer.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 0749f31..0e730d2 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -30,8 +30,8 @@ from carbono.exception import * from carbono.utils import * from carbono.config import * - from carbono.log import log +from partition_expander import PartitionExpander class ImageRestorer: @@ -205,17 +205,23 @@ def restore_image(self): def expand_last_partition(self): # After all data is copied to the disk # we instance class again to reload + sync() device = Device(self.target_device) disk = Disk(device) partition = disk.get_last_partition() if partition is not None: if partition.type == PARTITION_NORMAL: - log.info("Expanding {0} filesystem".\ - format(partition.get_path())) - self.notify_status("expand", {"device": - partition.get_path()}) - partition.filesystem.resize() + log.info("Changing disk table") + expander = PartitionExpander(partition.get_path()) + new_size = expander.try_expand() + if new_size!= -1: + log.info("Expanding {0} filesystem".\ + format(partition.get_path())) + self.notify_status("expand", {"device": + partition.get_path()}) + partition.filesystem.resize() + log.info("Expasion finished") def stop(self): # When restoring only a swap partition, buffer_manager From ee7bc80f47f3fa8b0fc88b437f4f24310e78f104 Mon Sep 17 00:00:00 2001 From: Bruno Date: Thu, 20 Jun 2013 15:57:54 -0300 Subject: [PATCH 18/99] Classe que altera a tabela de particao, expandindo a ultima --- carbono/partition_expander.py | 109 ++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 carbono/partition_expander.py diff --git a/carbono/partition_expander.py b/carbono/partition_expander.py new file mode 100644 index 0000000..89e5550 --- /dev/null +++ b/carbono/partition_expander.py @@ -0,0 +1,109 @@ +#!/usr/bin/python +# coding: utf-8 + +# Author: MStech +# Lucas Alvares Gomes + +import os +import subprocess +import logging +import parted +import time +from carbono.log import log + +class PartitionExpander: + + def __init__(self, path): + self.path = path + + def try_expand(self): + """ """ + log.debug("Try expand") + device = parted.Device(self.path) + disk = parted.Disk(device) + + try: + last_partition = disk.partitions[-1] + except IndexError: + log.error("The disk has no partitions") + return -1 + + # A partição deve ser primária e conter NTFS como sistema + # de arquivos + + if last_partition.type != parted.PARTITION_NORMAL: + log.error("The partition must be primary") + return -1 + + if last_partition.fileSystem is not None: + if last_partition.fileSystem.type != "ntfs": + log.error("We only support ntfs filesystem for now") + return -1 + else: + log.error("The partititon has no filesystem") + return -1 + + # Recria a última partição do disco utilizando + # todo o espaço que desalocado + start = last_partition.geometry.start + fs = last_partition.fileSystem + ped = last_partition.getPedPartition() + + disk.removePartition(last_partition) + + new_geometry = None + for region in disk.getFreeSpaceRegions(): + if region.start == start: + new_geometry = region + + constraint = parted.Constraint(exactGeom=new_geometry) + new_partition = parted.Partition(disk = disk, + type = parted.PARTITION_NORMAL, + fs = fs, + geometry = new_geometry, + PedPartition = ped) + + disk.addPartition(partition=new_partition, constraint=constraint) + try: + disk.commit() + except: + disk.commitToDevice() + disk.commitToOS() + + # Após criar a tabela de partição temos que fazer + # com o kernel releia essa tabela. Será preciso + # fazer isso para dar continuidade ao processo + attempt = 0 + while True: + p = subprocess.Popen("sfdisk -R %s" % device.path, + shell = True, + stderr = subprocess.PIPE, + ) + + if not len(p.stderr.readlines()): + break + + if attempt >= 5: + return -1 + + attempt += 1 + time.sleep(2) + + # Apos o kernel re-ler a nova tabela de partição + # temos que esperar o dispositivo ficar pronto + attempt = 0 + while True: + if os.path.exists(new_partition.path): + break + + if attempt >= 5: + return -1 + + attempt += 1 + time.sleep(2) + + # Agora da um resize no sistema de arquivos + # Pegar o tamanho total do dispositivo da partição a ser redimensionada + size = new_partition.geometry.length * device.sectorSize + + return size From 0bac0cd8e4b6837c7e7c17e3f35e0e48e7c91625 Mon Sep 17 00:00:00 2001 From: Bruno Date: Thu, 20 Jun 2013 16:03:39 -0300 Subject: [PATCH 19/99] Abre grubinstall apos finalizar a imagem. Requisito de minas --- carbono/image_restorer.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 0e730d2..00e0404 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -201,27 +201,35 @@ def restore_image(self): else: self._finish() log.info("Restoration finished") + log.info("Iniciando gtk grubinstall") + cmd = "{0}".format(which("grubinstall")) + try: + self.process = RunCmd(cmd) + self.process.run() + except Exception as e: + log.error("Erro ao iniciar grubinstall. {0}".format(e)) def expand_last_partition(self): # After all data is copied to the disk # we instance class again to reload - + sync() device = Device(self.target_device) disk = Disk(device) partition = disk.get_last_partition() if partition is not None: if partition.type == PARTITION_NORMAL: - log.info("Changing disk table") expander = PartitionExpander(partition.get_path()) + log.info("vai expandir") new_size = expander.try_expand() + log.info("new_size {0}".format(new_size)) if new_size!= -1: log.info("Expanding {0} filesystem".\ format(partition.get_path())) self.notify_status("expand", {"device": partition.get_path()}) partition.filesystem.resize() - log.info("Expasion finished") + log.info("maoeee") def stop(self): # When restoring only a swap partition, buffer_manager From 6f25273c82883922f9452c6b26e0c44b8eaee607 Mon Sep 17 00:00:00 2001 From: Bruno Date: Wed, 26 Jun 2013 13:37:41 -0300 Subject: [PATCH 20/99] Adicionada funcao para formatar o filesystem. E ao inves de dar resize apos expandir, formata --- carbono/filesystem/__init__.py | 3 +++ carbono/filesystem/ext.py | 22 ++++++++++++++++++---- carbono/filesystem/ntfs.py | 19 +++++++++++++++++-- carbono/image_restorer.py | 10 +++++----- 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/carbono/filesystem/__init__.py b/carbono/filesystem/__init__.py index 3c2952b..6c56e34 100644 --- a/carbono/filesystem/__init__.py +++ b/carbono/filesystem/__init__.py @@ -111,6 +111,9 @@ def fill_with_zeros(self): def resize(self): return self._fs.resize() + + def format_filesystem(self): + return self._fs.format_filesystem() def stop(self): self._fs.stop() diff --git a/carbono/filesystem/ext.py b/carbono/filesystem/ext.py index dc3d79e..0f4d992 100644 --- a/carbono/filesystem/ext.py +++ b/carbono/filesystem/ext.py @@ -104,14 +104,28 @@ def check(self): return True return False + def format_filesystem(self): + + proc = subprocess.Popen([which("mkfs.ext3"), self.path], + stdout=subprocess.PIPE) + output = proc.stdout.read() + output = output.strip() + if output == 0: + return True + return False + + def resize(self): + if self.check(): - ret = run_simple_command("{0} {1}".format(which("resize2fs"), - self.path)) - if ret == 0: + proc = subprocess.Popen([which("resize2fs"), self.path], + stdout=subprocess.PIPE) + output = proc.stdout.read() + output = output.strip() + if output == 0: return True return False - + def read_label(self): proc = subprocess.Popen([which("e2label"), self.path], stdout=subprocess.PIPE) diff --git a/carbono/filesystem/ntfs.py b/carbono/filesystem/ntfs.py index df46616..47b8409 100644 --- a/carbono/filesystem/ntfs.py +++ b/carbono/filesystem/ntfs.py @@ -90,10 +90,25 @@ def check(self): self.process.run() return not self.process.wait() + def format_filesystem(self): + + proc = subprocess.Popen([which("mkfs.ntfs"), "--fast {0}".format(self.path)], + stdout=subprocess.PIPE) + output = proc.stdout.read() + output = output.strip() + if output == 0: + return True + return False + + def resize(self): + if self.check(): - ret = run_simple_command("ntfsresize -f {0}".format(self.path)) - if ret == 0: + proc = subprocess.Popen([which("ntfsresize"), self.path], + stdout=subprocess.PIPE) + output = proc.stdout.read() + output = output.strip() + if output == 0: return True return False diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 00e0404..b69b809 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -223,13 +223,13 @@ def expand_last_partition(self): log.info("vai expandir") new_size = expander.try_expand() log.info("new_size {0}".format(new_size)) - if new_size!= -1: - log.info("Expanding {0} filesystem".\ + #if new_size!= -1: + log.info("Expanding {0} filesystem".\ format(partition.get_path())) - self.notify_status("expand", {"device": + self.notify_status("format", {"device": partition.get_path()}) - partition.filesystem.resize() - log.info("maoeee") + #partition.filesystem.resize() + partition.filesystem.format_filesystem() def stop(self): # When restoring only a swap partition, buffer_manager From f72c3c59939c154c67262640d39c6fd1c8d7aedd Mon Sep 17 00:00:00 2001 From: Bruno Date: Tue, 23 Jul 2013 10:59:39 -0300 Subject: [PATCH 21/99] Mudado para usar o BUILD_ID. E versao mudada para 2.0.0 --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d602070..e4e3aaa 100755 --- a/setup.py +++ b/setup.py @@ -1,15 +1,18 @@ #!/usr/bin/python import os from setuptools import setup -from carbono.config import get_version def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() +VERSION_ID = "2.0.0" +BUILD_ID = "{0}-SNAPSHOT".format(VERSION_ID) +if 'BUILD_NUMBER' in os.environ: + BUILD_ID = "{0}.{1}".format(VERSION_ID, os.environ['BUILD_NUMBER']) setup( name = "carbono", - version = get_version(), + version = BUILD_ID, author = "Bruno Casella", author_email = "bruno.casella@gmail.com", description = ("A hard disk imaging and recovery application"), From d8a6bdab0704a61d9e4a928da6286f7c961499ff Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Fri, 26 Jul 2013 18:51:00 -0300 Subject: [PATCH 22/99] Expansao da ultima particao, sendo particoes formatadas em (ext4/ntfs) --- carbono/disk_layout_manager.py | 0 carbono/filesystem/ntfs.py | 18 ++++++++---------- carbono/image_restorer.py | 15 ++++++++------- carbono/partition_expander.py | 27 +++++++++++++++++---------- 4 files changed, 33 insertions(+), 27 deletions(-) mode change 100644 => 100755 carbono/disk_layout_manager.py mode change 100644 => 100755 carbono/filesystem/ntfs.py diff --git a/carbono/disk_layout_manager.py b/carbono/disk_layout_manager.py old mode 100644 new mode 100755 diff --git a/carbono/filesystem/ntfs.py b/carbono/filesystem/ntfs.py old mode 100644 new mode 100755 index 47b8409..559a982 --- a/carbono/filesystem/ntfs.py +++ b/carbono/filesystem/ntfs.py @@ -90,16 +90,14 @@ def check(self): self.process.run() return not self.process.wait() - def format_filesystem(self): - - proc = subprocess.Popen([which("mkfs.ntfs"), "--fast {0}".format(self.path)], - stdout=subprocess.PIPE) - output = proc.stdout.read() - output = output.strip() - if output == 0: - return True - return False - + #def format_filesystem(self): + # proc = subprocess.Popen([which("mkfs.ntfs"), "--fast {0}".format(self.path)], + # stdout=subprocess.PIPE) + # output = proc.stdout.read() + # output = output.strip() + # if output == 0: + # return True + # return False def resize(self): diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index b69b809..68d0e0f 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -223,13 +223,14 @@ def expand_last_partition(self): log.info("vai expandir") new_size = expander.try_expand() log.info("new_size {0}".format(new_size)) - #if new_size!= -1: - log.info("Expanding {0} filesystem".\ - format(partition.get_path())) - self.notify_status("format", {"device": - partition.get_path()}) - #partition.filesystem.resize() - partition.filesystem.format_filesystem() + if new_size!= -1: + #log.info("Expanding {0} filesystem".\ + # format(partition.get_path())) + #self.notify_status("format", {"device": + # partition.get_path()}) + log.debug(partition.filesystem) + partition.filesystem.resize() + #partition.filesystem.format_filesystem() def stop(self): # When restoring only a swap partition, buffer_manager diff --git a/carbono/partition_expander.py b/carbono/partition_expander.py index 89e5550..4ae59e4 100644 --- a/carbono/partition_expander.py +++ b/carbono/partition_expander.py @@ -35,12 +35,15 @@ def try_expand(self): log.error("The partition must be primary") return -1 - if last_partition.fileSystem is not None: - if last_partition.fileSystem.type != "ntfs": - log.error("We only support ntfs filesystem for now") - return -1 - else: - log.error("The partititon has no filesystem") + #if last_partition.fileSystem is not None: + # if last_partition.fileSystem.type != "ntfs": + # log.error("We only support ntfs filesystem for now") + # return -1 + #else: + # log.error("The partititon has no filesystem") + # return -1 + if last_partition.fileSystem is None: + log.error("The last Partition, is not typed") return -1 # Recria a última partição do disco utilizando @@ -72,7 +75,7 @@ def try_expand(self): # Após criar a tabela de partição temos que fazer # com o kernel releia essa tabela. Será preciso - # fazer isso para dar continuidade ao processo + # fazer isso para dar continuidade ao processoi attempt = 0 while True: p = subprocess.Popen("sfdisk -R %s" % device.path, @@ -84,7 +87,11 @@ def try_expand(self): break if attempt >= 5: - return -1 + break + + if hasattr(p, "process"): + p.process.wait() + p.process.returncode attempt += 1 time.sleep(2) @@ -97,7 +104,7 @@ def try_expand(self): break if attempt >= 5: - return -1 + break attempt += 1 time.sleep(2) @@ -105,5 +112,5 @@ def try_expand(self): # Agora da um resize no sistema de arquivos # Pegar o tamanho total do dispositivo da partição a ser redimensionada size = new_partition.geometry.length * device.sectorSize - + log.debug(size) return size From bb1552d784ece1271af4eae66354682df6fa6c92 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Mon, 29 Jul 2013 11:26:10 -0300 Subject: [PATCH 23/99] Mudanca no codigo para realizar a expansao da ultima particao com formatacao (NTFS/EXT) --- carbono/filesystem/ntfs.py | 16 ++++++++-------- carbono/image_restorer.py | 16 +++++++++------- carbono/partition.py | 0 carbono/partition_expander.py | 11 ++++++----- 4 files changed, 23 insertions(+), 20 deletions(-) mode change 100644 => 100755 carbono/partition.py diff --git a/carbono/filesystem/ntfs.py b/carbono/filesystem/ntfs.py index 559a982..6721a90 100755 --- a/carbono/filesystem/ntfs.py +++ b/carbono/filesystem/ntfs.py @@ -90,14 +90,14 @@ def check(self): self.process.run() return not self.process.wait() - #def format_filesystem(self): - # proc = subprocess.Popen([which("mkfs.ntfs"), "--fast {0}".format(self.path)], - # stdout=subprocess.PIPE) - # output = proc.stdout.read() - # output = output.strip() - # if output == 0: - # return True - # return False + def format_filesystem(self): + proc = subprocess.Popen([which("mkfs.ntfs"), "--fast {0}".format(self.path)], + stdout=subprocess.PIPE) + output = proc.stdout.read() + output = output.strip() + if output == 0: + return True + return False def resize(self): diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 68d0e0f..0090070 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -220,16 +220,18 @@ def expand_last_partition(self): if partition is not None: if partition.type == PARTITION_NORMAL: expander = PartitionExpander(partition.get_path()) - log.info("vai expandir") + log.info("checking and try expand {0}".format(partition.get_path())) + self.notify_status("expand", {"device": + partition.get_path()}) new_size = expander.try_expand() log.info("new_size {0}".format(new_size)) if new_size!= -1: - #log.info("Expanding {0} filesystem".\ - # format(partition.get_path())) - #self.notify_status("format", {"device": - # partition.get_path()}) - log.debug(partition.filesystem) - partition.filesystem.resize() + log.info("Expanding {0} filesystem".\ + format(partition.get_path())) + self.notify_status("expand", {"device": + partition.get_path()}) + result = partition.filesystem.resize() + log.info("Result of expanding {0} is {1}".format(partition.get_path(), result)) #partition.filesystem.format_filesystem() def stop(self): diff --git a/carbono/partition.py b/carbono/partition.py old mode 100644 new mode 100755 diff --git a/carbono/partition_expander.py b/carbono/partition_expander.py index 4ae59e4..0140590 100644 --- a/carbono/partition_expander.py +++ b/carbono/partition_expander.py @@ -42,8 +42,9 @@ def try_expand(self): #else: # log.error("The partititon has no filesystem") # return -1 + if last_partition.fileSystem is None: - log.error("The last Partition, is not typed") + log.error("The Partition, has no Filesystem") return -1 # Recria a última partição do disco utilizando @@ -75,7 +76,7 @@ def try_expand(self): # Após criar a tabela de partição temos que fazer # com o kernel releia essa tabela. Será preciso - # fazer isso para dar continuidade ao processoi + # fazer isso para dar continuidade ao processo attempt = 0 while True: p = subprocess.Popen("sfdisk -R %s" % device.path, @@ -86,7 +87,7 @@ def try_expand(self): if not len(p.stderr.readlines()): break - if attempt >= 5: + if attempt >= 10: break if hasattr(p, "process"): @@ -103,7 +104,7 @@ def try_expand(self): if os.path.exists(new_partition.path): break - if attempt >= 5: + if attempt >= 10: break attempt += 1 @@ -112,5 +113,5 @@ def try_expand(self): # Agora da um resize no sistema de arquivos # Pegar o tamanho total do dispositivo da partição a ser redimensionada size = new_partition.geometry.length * device.sectorSize - log.debug(size) + return size From de84cd383493c6c6f7181b2d55a6ca43ea122ced Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Mon, 29 Jul 2013 14:10:06 -0300 Subject: [PATCH 24/99] Codigo de expansao da ultima particao, com formatacao NTFS/EXT --- carbono/filesystem/ntfs.py | 4 ++-- carbono/image_restorer.py | 4 ++-- carbono/partition_expander.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/carbono/filesystem/ntfs.py b/carbono/filesystem/ntfs.py index 6721a90..f35c222 100755 --- a/carbono/filesystem/ntfs.py +++ b/carbono/filesystem/ntfs.py @@ -107,8 +107,8 @@ def resize(self): output = proc.stdout.read() output = output.strip() if output == 0: - return True - return False + return output, True + return output, False def read_label(self): proc = subprocess.Popen([which("ntfslabel"), "--force", self.path], diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 0090070..68dd678 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -230,8 +230,8 @@ def expand_last_partition(self): format(partition.get_path())) self.notify_status("expand", {"device": partition.get_path()}) - result = partition.filesystem.resize() - log.info("Result of expanding {0} is {1}".format(partition.get_path(), result)) + output,result = partition.filesystem.resize() + log.info("{0}".format(output)) #partition.filesystem.format_filesystem() def stop(self): diff --git a/carbono/partition_expander.py b/carbono/partition_expander.py index 0140590..b9195e3 100644 --- a/carbono/partition_expander.py +++ b/carbono/partition_expander.py @@ -87,7 +87,7 @@ def try_expand(self): if not len(p.stderr.readlines()): break - if attempt >= 10: + if attempt >= 5: break if hasattr(p, "process"): @@ -104,7 +104,7 @@ def try_expand(self): if os.path.exists(new_partition.path): break - if attempt >= 10: + if attempt >= 5: break attempt += 1 From 2140df6a8b12a9e2fb633233546465e4b7d0cd6a Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Mon, 29 Jul 2013 18:19:16 -0300 Subject: [PATCH 25/99] Implementacao da expansao da ultima partincao, com formatacao Ntfs/ext --- carbono/filesystem/ext.py | 8 +++++--- carbono/filesystem/ntfs.py | 8 +++++--- carbono/image_restorer.py | 4 ++-- carbono/partition_expander.py | 20 ++++---------------- 4 files changed, 16 insertions(+), 24 deletions(-) diff --git a/carbono/filesystem/ext.py b/carbono/filesystem/ext.py index 0f4d992..18e821b 100644 --- a/carbono/filesystem/ext.py +++ b/carbono/filesystem/ext.py @@ -120,11 +120,13 @@ def resize(self): if self.check(): proc = subprocess.Popen([which("resize2fs"), self.path], stdout=subprocess.PIPE) + proc.wait() output = proc.stdout.read() output = output.strip() - if output == 0: - return True - return False + returncode = proc.returncode + if returncode == 0: + return returncode, True + return returncode, False def read_label(self): proc = subprocess.Popen([which("e2label"), self.path], diff --git a/carbono/filesystem/ntfs.py b/carbono/filesystem/ntfs.py index f35c222..9a8c92a 100755 --- a/carbono/filesystem/ntfs.py +++ b/carbono/filesystem/ntfs.py @@ -104,11 +104,13 @@ def resize(self): if self.check(): proc = subprocess.Popen([which("ntfsresize"), self.path], stdout=subprocess.PIPE) + proc.wait() output = proc.stdout.read() output = output.strip() - if output == 0: - return output, True - return output, False + returncode = proc.returncode + if returncode == 0: + return returncode, True + return returncode, False def read_label(self): proc = subprocess.Popen([which("ntfslabel"), "--force", self.path], diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 68dd678..acfa113 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -219,7 +219,7 @@ def expand_last_partition(self): partition = disk.get_last_partition() if partition is not None: if partition.type == PARTITION_NORMAL: - expander = PartitionExpander(partition.get_path()) + expander = PartitionExpander(device.path) log.info("checking and try expand {0}".format(partition.get_path())) self.notify_status("expand", {"device": partition.get_path()}) @@ -231,7 +231,7 @@ def expand_last_partition(self): self.notify_status("expand", {"device": partition.get_path()}) output,result = partition.filesystem.resize() - log.info("{0}".format(output)) + log.info("{0}".format(result)) #partition.filesystem.format_filesystem() def stop(self): diff --git a/carbono/partition_expander.py b/carbono/partition_expander.py index b9195e3..afa029c 100644 --- a/carbono/partition_expander.py +++ b/carbono/partition_expander.py @@ -18,10 +18,9 @@ def __init__(self, path): def try_expand(self): """ """ - log.debug("Try expand") device = parted.Device(self.path) disk = parted.Disk(device) - + try: last_partition = disk.partitions[-1] except IndexError: @@ -35,14 +34,6 @@ def try_expand(self): log.error("The partition must be primary") return -1 - #if last_partition.fileSystem is not None: - # if last_partition.fileSystem.type != "ntfs": - # log.error("We only support ntfs filesystem for now") - # return -1 - #else: - # log.error("The partititon has no filesystem") - # return -1 - if last_partition.fileSystem is None: log.error("The Partition, has no Filesystem") return -1 @@ -88,11 +79,7 @@ def try_expand(self): break if attempt >= 5: - break - - if hasattr(p, "process"): - p.process.wait() - p.process.returncode + return -1 attempt += 1 time.sleep(2) @@ -101,11 +88,12 @@ def try_expand(self): # temos que esperar o dispositivo ficar pronto attempt = 0 while True: + if os.path.exists(new_partition.path): break if attempt >= 5: - break + return -1 attempt += 1 time.sleep(2) From e9714253f42329f64909389d87b5381aba365427 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Tue, 30 Jul 2013 12:26:47 -0300 Subject: [PATCH 26/99] Expansao da ultima particao, quando for NTFS e formatacao da ultima particao quando for EXT --- carbono/filesystem/ext.py | 3 ++- carbono/image_restorer.py | 23 +++++++++++++++-------- carbono/partition_expander.py | 9 +++++++-- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/carbono/filesystem/ext.py b/carbono/filesystem/ext.py index 18e821b..678548e 100644 --- a/carbono/filesystem/ext.py +++ b/carbono/filesystem/ext.py @@ -110,7 +110,8 @@ def format_filesystem(self): stdout=subprocess.PIPE) output = proc.stdout.read() output = output.strip() - if output == 0: + returncode = proc.returncode + if returncode == 0: return True return False diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index acfa113..3f6016e 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -225,14 +225,21 @@ def expand_last_partition(self): partition.get_path()}) new_size = expander.try_expand() log.info("new_size {0}".format(new_size)) - if new_size!= -1: - log.info("Expanding {0} filesystem".\ - format(partition.get_path())) - self.notify_status("expand", {"device": - partition.get_path()}) - output,result = partition.filesystem.resize() - log.info("{0}".format(result)) - #partition.filesystem.format_filesystem() + + if partition.fileSystem.type == "ntfs": + if new_size!= -1: + log.info("Expanding {0} filesystem".\ + format(partition.get_path())) + self.notify_status("expand", {"device": + partition.get_path()}) + output,result = partition.filesystem.resize() + else: + log.info("Formating last partition {0} typed as {1}".format(partition.get_path(), partition.fileSystem.type)) + returncode = partition.filesystem.format_filesystem() + if returncode == 0: + log.info("Formatation in {0} was made with sucess".format(partition.get_path())) + else: + log.info("Formatation in {0} failed".format(partition.get_path())) def stop(self): # When restoring only a swap partition, buffer_manager diff --git a/carbono/partition_expander.py b/carbono/partition_expander.py index afa029c..a588887 100644 --- a/carbono/partition_expander.py +++ b/carbono/partition_expander.py @@ -35,9 +35,14 @@ def try_expand(self): return -1 if last_partition.fileSystem is None: - log.error("The Partition, has no Filesystem") + log.error("The partition hasn't filesystem") return -1 - + + if last_partition.fileSystem is not None: + if last_partition.fileSystem.type != "ntfs": + log.error("We only support ntfs filesystem for now") + return -1 + # Recria a última partição do disco utilizando # todo o espaço que desalocado start = last_partition.geometry.start From 701d658bc1e317ead04f4cb2a3d36e250eb40eed Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Tue, 30 Jul 2013 16:08:28 -0300 Subject: [PATCH 27/99] Codigo de expansao da ultima particao, com formatacao NTFS/EXT --- carbono/image_restorer.py | 23 ++++++++--------------- carbono/partition_expander.py | 8 ++++---- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 3f6016e..3d4cbc8 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -221,25 +221,18 @@ def expand_last_partition(self): if partition.type == PARTITION_NORMAL: expander = PartitionExpander(device.path) log.info("checking and try expand {0}".format(partition.get_path())) - self.notify_status("expand", {"device": - partition.get_path()}) new_size = expander.try_expand() log.info("new_size {0}".format(new_size)) - - if partition.fileSystem.type == "ntfs": - if new_size!= -1: - log.info("Expanding {0} filesystem".\ - format(partition.get_path())) - self.notify_status("expand", {"device": - partition.get_path()}) - output,result = partition.filesystem.resize() - else: - log.info("Formating last partition {0} typed as {1}".format(partition.get_path(), partition.fileSystem.type)) - returncode = partition.filesystem.format_filesystem() + if new_size!= -1: + log.info("Expanding {0} filesystem".\ + format(partition.get_path())) + self.notify_status("expand", {"device": + partition.get_path()}) + returncode,result = partition.filesystem.resize() if returncode == 0: - log.info("Formatation in {0} was made with sucess".format(partition.get_path())) + log.info("Resize in {0} was made with sucess".format(partition.get_path())) else: - log.info("Formatation in {0} failed".format(partition.get_path())) + log.info("Resize in {0} failed".format(partition.get_path())) def stop(self): # When restoring only a swap partition, buffer_manager diff --git a/carbono/partition_expander.py b/carbono/partition_expander.py index a588887..8dc32e7 100644 --- a/carbono/partition_expander.py +++ b/carbono/partition_expander.py @@ -38,10 +38,10 @@ def try_expand(self): log.error("The partition hasn't filesystem") return -1 - if last_partition.fileSystem is not None: - if last_partition.fileSystem.type != "ntfs": - log.error("We only support ntfs filesystem for now") - return -1 + #if last_partition.fileSystem is not None: + # if last_partition.fileSystem.type != "ntfs": + # log.error("We only support ntfs filesystem for now") + # return -1 # Recria a última partição do disco utilizando # todo o espaço que desalocado From 046cd17a26fa61dd5fe4f9663a2bbdb67424ff38 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 20 Aug 2013 11:22:59 -0300 Subject: [PATCH 28/99] Version change to 2.0 to 2.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4e3aaa..f02468c 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() -VERSION_ID = "2.0.0" +VERSION_ID = "2.1.0" BUILD_ID = "{0}-SNAPSHOT".format(VERSION_ID) if 'BUILD_NUMBER' in os.environ: BUILD_ID = "{0}.{1}".format(VERSION_ID, os.environ['BUILD_NUMBER']) From cd7fd6167d8b0025236e26903d45555adbded77f Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Fri, 23 Aug 2013 09:20:55 -0300 Subject: [PATCH 29/99] Change in config for add size of the base system (used when creating iso's). And added egg folder to gitignore --- .gitignore | 1 + carbono/config.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 286cf05..8e28cda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ build/ dist/ +carbono.egg-info/ *.swp *.mo *.pyc diff --git a/carbono/config.py b/carbono/config.py index 134610c..ca620e3 100644 --- a/carbono/config.py +++ b/carbono/config.py @@ -21,7 +21,7 @@ FILE_PATTERN = "{name}-{partition}-{volume}.data" BLOCK_SIZE = 1048576 # 1MB -BASE_SYSTEM_SIZE = 0 #TODO (in bytes) +BASE_SYSTEM_SIZE = 1024 * 1024 * 250 #Em bytes. TODO: pegar tamanho real LOG_FILE = "/var/log/mstech/carbono.log" EOF = -1 VERSION = (0, 1, 0, "alpha", 0) From 3d7cb6ca1ada1220dc87f1912950507ea36f377d Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Fri, 23 Aug 2013 16:51:31 -0300 Subject: [PATCH 30/99] Mudanca na criacao de Iso --- carbono/iso_creator.py | 4 +- carbono/utils.py | 89 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/carbono/iso_creator.py b/carbono/iso_creator.py index c5d6ca4..987ff91 100644 --- a/carbono/iso_creator.py +++ b/carbono/iso_creator.py @@ -21,6 +21,7 @@ from carbono.utils import * from carbono.exception import * from carbono.config import * +from carbono.log import log CARBONO_FILES = ("initram.gz", "vmlinuz", "isolinux.cfg") @@ -83,11 +84,13 @@ def run(self): if first_volume: device = get_cdrom_device() + log.debug("Os device que tem o upimage eh {0}".format(device)) while True: error = False try: self.mount_point = self.mount_device(device) + log.debug("O Local que foi montado eh {0}".format(self.mount_point)) except ErrorMountingFilesystem: error = True @@ -96,7 +99,6 @@ def run(self): run_simple_command("umount {0}".\ format(self.mount_point)) error = True - if error: device = self.notify_callback("base_files_not_found", {"device": device}) diff --git a/carbono/utils.py b/carbono/utils.py index 9eda56f..cdbf28f 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -21,6 +21,7 @@ import random import errno import os +import parted from threading import Thread, Event from os.path import realpath @@ -138,15 +139,89 @@ def available_memory(percent=100): return free +def get_devices(): + disk_dict = {} + devices = parted.getAllDevices() + for device in devices: + dev_path = device.path + try: + disk = parted.Disk(device) + except: + continue + part_dict = {} + for p in disk.partitions: + part_path = p.path + part_type = "unkown" + if p.fileSystem: + part_type = p.fileSystem.type + if part_type == "fat32" or part_type == "fat16": + part_dict[part_path] = {"type":part_type} + disk_dict[dev_path] = {"partitions": part_dict} + return disk_dict + +CARBONO_FILES2 = ("initram.gz","vmlinuz","isolinux.cfg") + +def mount_pen(device): + tmpd = make_temp_dir() + ret = run_simple_command("mount {0} {1}".format(device, tmpd)) + if ret is not 0: + raise ErrorMountingFilesystem + return tmpd + +def find_carbono(path): + dev_files = os.listdir(path) + ret = True + if filter(lambda x:not x in dev_files, CARBONO_FILES2): + ret = False + return ret + + +def mount_point(device): + mounts = {} + for line in subprocess.check_output(['mount', '-l']).split('\n'): + parts = line.split(' ') + if len(parts) > 2: + mounts[parts[0]] = parts[2] + try: + if mounts[device]: + return mounts[device] + else: + raise ErrorWithoutConnectedDevice("Sem Pen-Drive conectado") + except: + raise ErrorIdentifyDevice("Erro na identificação do Pendrive") + +def get_upimage_device(): + devices = get_devices() + for dev in devices: + device = devices[dev]["partitions"].keys() + if is_mounted(device[0]): + mount_path = mount_point(device[0]) + else: + mount_path = mount_pen(device[0]) + ret = find_carbono(mount_path) + if ret: + run_simple_command("umount {0}".format(mount_path)) + return device[0], mount_path + else: + if ret == 0: + run_simple_command("umount {0}".format(mount_path)) + return -1,-1 + def get_cdrom_device(): device = None - with open("/proc/sys/dev/cdrom/info", 'r') as f: - for line in f: - if line.startswith("drive name:"): - try: - device = "/dev/" + line.split()[2] - except IndexError: - break + path = None + try: + device,path = get_upimage_device() + except: + pass + if (device and path) == -1: + with open("/proc/sys/dev/cdrom/info", 'r') as f: + for line in f: + if line.startswith("drive name:"): + try: + device = "/dev/" + line.split()[2] + except IndexError: + continue return device def which(program): From 740f8155306e5107b89db8d7770eab5343d51225 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Fri, 23 Aug 2013 17:51:09 -0300 Subject: [PATCH 31/99] image_info added to initial iso even when its a partittion recovery --- carbono/iso_creator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/carbono/iso_creator.py b/carbono/iso_creator.py index c5d6ca4..e25c667 100644 --- a/carbono/iso_creator.py +++ b/carbono/iso_creator.py @@ -115,13 +115,14 @@ def run(self): # Bootloader self.slices[volume].append("/usr/lib/syslinux/isolinux.bin") - + self.slices[volume].append("{0}image.info".format( + self.target_path)) if self.is_disk: # Add the rest of files needed to # restore the image map(lambda x: self.slices[volume].\ append(self.target_path + x), - ("mbr.bin", "disk.dl","image.info")) + ("mbr.bin", "disk.dl")) if first_volume: extra_params = "-joliet-long -b " + \ From bd762f535dec1ee280528acfe34744d7dcef7b9f Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 26 Aug 2013 11:04:41 -0300 Subject: [PATCH 32/99] Retirados logs do iso_creator --- carbono/iso_creator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/carbono/iso_creator.py b/carbono/iso_creator.py index f5688c7..273f72c 100644 --- a/carbono/iso_creator.py +++ b/carbono/iso_creator.py @@ -21,7 +21,6 @@ from carbono.utils import * from carbono.exception import * from carbono.config import * -from carbono.log import log CARBONO_FILES = ("initram.gz", "vmlinuz", "isolinux.cfg") @@ -84,13 +83,11 @@ def run(self): if first_volume: device = get_cdrom_device() - log.debug("Os device que tem o upimage eh {0}".format(device)) while True: error = False try: self.mount_point = self.mount_device(device) - log.debug("O Local que foi montado eh {0}".format(self.mount_point)) except ErrorMountingFilesystem: error = True From 1c6521c7d75878fbf2363a9044ecb5c3a8baccc5 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Fri, 13 Sep 2013 13:59:51 -0300 Subject: [PATCH 33/99] Otimizacao de leitura utilizacao Partclone para particoes EXT --- carbono/config.py | 6 ++++-- carbono/filesystem/ext.py | 15 ++++++++------- carbono/image_creator.py | 23 +++++++++++++++++++++-- 3 files changed, 33 insertions(+), 11 deletions(-) mode change 100644 => 100755 carbono/config.py mode change 100644 => 100755 carbono/filesystem/ext.py mode change 100644 => 100755 carbono/image_creator.py diff --git a/carbono/config.py b/carbono/config.py old mode 100644 new mode 100755 index ca620e3..5fb5418 --- a/carbono/config.py +++ b/carbono/config.py @@ -18,10 +18,12 @@ import os - +SUCESS_PARTCLONE = False +OLD_PERCENT = -2 +CURRENT_PARTITION_TYPE = "" FILE_PATTERN = "{name}-{partition}-{volume}.data" BLOCK_SIZE = 1048576 # 1MB -BASE_SYSTEM_SIZE = 1024 * 1024 * 250 #Em bytes. TODO: pegar tamanho real +BASE_SYSTEM_SIZE = 0 #TODO (in bytes) LOG_FILE = "/var/log/mstech/carbono.log" EOF = -1 VERSION = (0, 1, 0, "alpha", 0) diff --git a/carbono/filesystem/ext.py b/carbono/filesystem/ext.py old mode 100644 new mode 100755 index 678548e..67c9608 --- a/carbono/filesystem/ext.py +++ b/carbono/filesystem/ext.py @@ -100,6 +100,10 @@ def close(self): def check(self): ret = run_simple_command("{0} -f -y -v {1}".format(which("e2fsck"), self.path)) + proc = subprocess.Popen([which("resize2fs"), self.path], + stdout=subprocess.PIPE) + output = proc.stdout.read() + output = output.strip() if ret in (0, 1, 2, 256): return True return False @@ -110,8 +114,7 @@ def format_filesystem(self): stdout=subprocess.PIPE) output = proc.stdout.read() output = output.strip() - returncode = proc.returncode - if returncode == 0: + if output == 0: return True return False @@ -121,13 +124,11 @@ def resize(self): if self.check(): proc = subprocess.Popen([which("resize2fs"), self.path], stdout=subprocess.PIPE) - proc.wait() output = proc.stdout.read() output = output.strip() - returncode = proc.returncode - if returncode == 0: - return returncode, True - return returncode, False + if output == 0: + return True + return False def read_label(self): proc = subprocess.Popen([which("e2label"), self.path], diff --git a/carbono/image_creator.py b/carbono/image_creator.py old mode 100644 new mode 100755 index 2973bff..18607d4 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -17,7 +17,7 @@ import math import os - +import time from carbono.device import Device from carbono.disk import Disk from carbono.mbr import Mbr @@ -66,6 +66,22 @@ def notify_percent(self): self.current_percent = percent self.notify_status("progress", {"percent": percent}) + + global OLD_PERCENT + global CURRENT_PARTITION_TYPE + global SUCESS_PARTCLONE + + if (SUCESS_PARTCLONE == False) and (percent > 100) and (percent == OLD_PERCENT) and (CURRENT_PARTITION_TYPE != 'ntfs'): + log.info(CURRENT_PARTITION_TYPE) + time.sleep(18) + cmd = "kill -9 $(ps ax|grep '{0} -c -s {1} -o -'|awk '{2}')".format('/usr/sbin/partclone.extfs',self.device_path, '{print $1}') + try: + self.process = RunCmd(cmd) + self.process.run() + except Exception as e: + log.info(e) + OLD_PERCENT = percent + def create_image(self): """ """ if is_mounted(self.device_path): @@ -135,7 +151,8 @@ def create_image(self): uuid = part.filesystem.uuid() label = part.filesystem.read_label() type = part.filesystem.type - + global CURRENT_PARTITION_TYPE + CURRENT_PARTITION_TYPE = type part.filesystem.open_to_read() compact_callback = None @@ -229,6 +246,8 @@ def create_image(self): self.notify_status("canceled", {"operation": "Create image"}) else: + global SUCESS_PARTCLONE + SUCESS_PARTCLONE = True self.notify_status("finish") log.info("Creation finished") From 7ec323738f2afde1de2216160ff2f821fb2927f0 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Tue, 24 Sep 2013 13:55:04 -0300 Subject: [PATCH 34/99] Modificacao no carbono para formatar e expandir --- carbono/image_restorer.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 3d4cbc8..4afe3a0 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -37,7 +37,7 @@ class ImageRestorer: def __init__(self, image_folder, target_device, status_callback, partitions=None, - expand=False): + expand=2): assert check_if_root(), "You need to run this application as root" @@ -107,7 +107,10 @@ def restore_image(self): mbr = Mbr(self.image_path) mbr.restore_from_file(self.target_device) dlm = DiskLayoutManager(self.image_path) - dlm.restore_from_file(disk, self.expand) + if self.expand != 2: + dlm.restore_from_file(disk, True) + else: + dlm.restore_from_file(disk, False) self.timer.start() for part in partitions: @@ -190,10 +193,11 @@ def restore_image(self): partition.filesystem.close() self.timer.stop() + log.info(self.expand) - if self.expand: + if self.expand != 2: if information.get_image_is_disk(): - self.expand_last_partition() + self.expand_last_partition(self.expand) if self.canceled: self.notify_status("canceled", {"operation": @@ -209,7 +213,7 @@ def restore_image(self): except Exception as e: log.error("Erro ao iniciar grubinstall. {0}".format(e)) - def expand_last_partition(self): + def expand_last_partition(self,opt_expand): # After all data is copied to the disk # we instance class again to reload @@ -224,16 +228,22 @@ def expand_last_partition(self): new_size = expander.try_expand() log.info("new_size {0}".format(new_size)) if new_size!= -1: - log.info("Expanding {0} filesystem".\ - format(partition.get_path())) - self.notify_status("expand", {"device": - partition.get_path()}) - returncode,result = partition.filesystem.resize() - if returncode == 0: - log.info("Resize in {0} was made with sucess".format(partition.get_path())) - else: - log.info("Resize in {0} failed".format(partition.get_path())) + if opt_expand == 0: + log.info("Expanding {0} filesystem".format(partition.get_path())) + self.notify_status("expand", {"device": + partition.get_path()}) + returncode = partition.filesystem.resize() + if returncode == 0: + log.info("Resize in {0} was made with sucess".format(partition.get_path())) + else: + log.info("Resize in {0} failed".format(partition.get_path())) + else: + if opt_expand == 1: + log.info("Formating {0} filesystem".format(partition.get_path())) + self.notify_status("format", {"device": + partition.get_path()}) + partition.filesystem.format_filesystem() def stop(self): # When restoring only a swap partition, buffer_manager # isnt necessary From 1494ee2bdcc643aee76d84542c860115a6ae3a91 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Thu, 26 Sep 2013 17:23:12 -0300 Subject: [PATCH 35/99] Insercao do alerta de erro de disco cheio, durante extracao --- carbono/image_creator.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/carbono/image_creator.py b/carbono/image_creator.py index 18607d4..58caf2d 100755 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -96,7 +96,7 @@ def create_image(self): mbr.save_to_file(self.device_path) dlm = DiskLayoutManager(self.target_path) dlm.save_to_file(disk) - + partition_list = disk.get_valid_partitions(self.raw) if not partition_list: raise ErrorCreatingImage("Partition(s) hasn't a " +\ @@ -192,8 +192,14 @@ def create_image(self): slices[iso_volume] = list() slices[iso_volume].append(file_path) break + try: + fd.write(data) + except Exception as e: + log.info("Erro na escrita do disco {0}".format(e)) + self.notify_status("disk_full") + self.cancel() + break - fd.write(data) self.processed_blocks += 1 if self.split_size: From 33e50b0ed83dac3f438d973871e0cb1b73a887c8 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Wed, 9 Oct 2013 17:31:19 -0300 Subject: [PATCH 36/99] add diskinfo to utils *DiskInfo class get the disk information and returns a dict --- carbono/utils.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/carbono/utils.py b/carbono/utils.py index cdbf28f..d135694 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -27,6 +27,101 @@ from os.path import realpath from carbono.exception import * + + +class DiskInfo(): + + def __init__(self): + self.__DISK_DICT = {} + self.__PARTITION_DICT = {} + + def __get_devices(self): + ''' Filter = only partitions with a valid filesystem ''' + + disk_dict = {} + devices = parted.getAllDevices() + for device in devices: + dev_path = device.path + dev_model = device.model + dev_size = device.getSize('b') + try: + disk = parted.Disk(device) + except: + continue + + part_dict = {} + for p in disk.partitions: + + if p.type not in (parted.PARTITION_NORMAL, + parted.PARTITION_LOGICAL): + continue + + part_path = p.path + part_size = p.getSize('b') + part_type = "unknown" + if p.fileSystem: + part_type = p.fileSystem.type + + part_dict[part_path] = {"size": part_size, + "type": part_type} + + disk_dict[dev_path] = {"model": dev_model, + "size": dev_size, + "partitions": part_dict} + return disk_dict + + + + def __collect_information_about_devices(self): + ''' + Pega informacoes sobre os discos e particoes, + todas as telas que precisarem de informacoes sobre discos ou particoes + pegam elas dos dicionarios aqui criados + ''' + _dict = self.__get_devices() + for disk in _dict: + partitions = _dict[disk]["partitions"].keys() + for part in partitions: + self.__PARTITION_DICT[part] = {"type": _dict[disk]["partitions"][part]["type"], + "size": _dict[disk]["partitions"][part]["size"]} + self.__DISK_DICT[disk] = {"model": _dict[disk]["model"], + "size": _dict[disk]["size"], + "partitions": partitions} + + + def formated_partitions(self): + formated_partitions = [] + formated_partitions_dict = {} + self.__collect_information_about_devices() + for part in self.__PARTITION_DICT.keys(): + + part_type = self.__PARTITION_DICT[part]['type'] + size_bytes = self.__PARTITION_DICT[part]['size'] + size_mb = int(long(size_bytes)/(1024*1024.0)) + part_dict = {} + part_dict["type"] = part_type + part_dict["path"] = part + part_dict["size"] = size_mb + formated_partitions.append(part_dict) + + formated_partitions.sort(reverse=False) + temp_parts = [] + disk = formated_partitions[0]['path'][:8] + for aux in range(0,len(formated_partitions)): + temp_disk = formated_partitions[aux]['path'][:8] + if temp_disk == disk: + temp_parts.append(formated_partitions[aux]) + else: + formated_partitions_dict[disk] = temp_parts + disk = temp_disk + temp_parts = [] + formated_partitions_dict[disk] = temp_parts + + return(formated_partitions_dict) + + + + class Timer(Thread): def __init__(self, callback, timeout=2): Thread.__init__(self) From a00a9c25b1d375d4a3c4ebf8c3194248b981a812 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Fri, 18 Oct 2013 13:17:14 -0300 Subject: [PATCH 37/99] Adicao do comando de apagar a pasta da imagem, se o disco esta cheio Modificacao no carbono o qual foi adicionado o comando que apaga a pasta que foi criada para armazenar as imagens. --- carbono/image_creator.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/carbono/image_creator.py b/carbono/image_creator.py index 58caf2d..e4e65e8 100755 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -18,6 +18,7 @@ import math import os import time +import shutil from carbono.device import Device from carbono.disk import Disk from carbono.mbr import Mbr @@ -195,8 +196,15 @@ def create_image(self): try: fd.write(data) except Exception as e: - log.info("Erro na escrita do disco {0}".format(e)) + log.info("{0}".format(e)) self.notify_status("disk_full") + try: + shutil.rmtree(self.target_path) + log.info("The folder which cointain the images files were erased") + break + except Exception as e: + log.info(e) + break self.cancel() break From e85d136f78d62c5e292082af9cbc438790188852 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 21 Oct 2013 12:31:18 -0200 Subject: [PATCH 38/99] Escopo de diskpartion *Adcionando escopo de funcao para montar discos em utils.py --- carbono/utils.py | 87 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index d135694..a718cf4 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -33,15 +33,15 @@ class DiskInfo(): def __init__(self): self.__DISK_DICT = {} - self.__PARTITION_DICT = {} + self.__PARTITION_DICT = {} def __get_devices(self): ''' Filter = only partitions with a valid filesystem ''' - + disk_dict = {} devices = parted.getAllDevices() for device in devices: - dev_path = device.path + dev_path = device.path dev_model = device.model dev_size = device.getSize('b') try: @@ -53,7 +53,7 @@ def __get_devices(self): for p in disk.partitions: if p.type not in (parted.PARTITION_NORMAL, - parted.PARTITION_LOGICAL): + parted.PARTITION_LOGICAL): continue part_path = p.path @@ -69,18 +69,17 @@ def __get_devices(self): "size": dev_size, "partitions": part_dict} return disk_dict - def __collect_information_about_devices(self): - ''' + ''' Pega informacoes sobre os discos e particoes, todas as telas que precisarem de informacoes sobre discos ou particoes pegam elas dos dicionarios aqui criados ''' _dict = self.__get_devices() - for disk in _dict: - partitions = _dict[disk]["partitions"].keys() + for disk in _dict: + partitions = _dict[disk]["partitions"].keys() for part in partitions: self.__PARTITION_DICT[part] = {"type": _dict[disk]["partitions"][part]["type"], "size": _dict[disk]["partitions"][part]["size"]} @@ -93,14 +92,14 @@ def formated_partitions(self): formated_partitions = [] formated_partitions_dict = {} self.__collect_information_about_devices() - for part in self.__PARTITION_DICT.keys(): - + for part in self.__PARTITION_DICT.keys(): + part_type = self.__PARTITION_DICT[part]['type'] - size_bytes = self.__PARTITION_DICT[part]['size'] - size_mb = int(long(size_bytes)/(1024*1024.0)) + size_bytes = self.__PARTITION_DICT[part]['size'] + size_mb = int(long(size_bytes)/(1024*1024.0)) part_dict = {} part_dict["type"] = part_type - part_dict["path"] = part + part_dict["path"] = part part_dict["size"] = size_mb formated_partitions.append(part_dict) @@ -122,6 +121,50 @@ def formated_partitions(self): +class DiskPartition(): + + + def __init__(self): + + self.__temp_folder = "" + self.__partition = "" + + def __generate_temp_folder(self, destino = "/tmp/"): + + #Do generate the temp folder + self.__temp_folder = "pasta gerada" + + def __set_partition(self, partition): + + self.__partition = partition + + def get_partition(self): + + return self.__partition + + def umount_partition(self): + pass + #desmonta self.__partition + + def umount_all_partitions(self): + pass + + def mount_partition(self,destino = None): + + mounted_folder = "" + if destino is None: + pass + #gera pasta temporario + else: + pass + #monta no destino + return mounted_folder + + def get_mounted_devices(self): + + pass + + class Timer(Thread): def __init__(self, callback, timeout=2): Thread.__init__(self) @@ -236,8 +279,8 @@ def available_memory(percent=100): def get_devices(): disk_dict = {} - devices = parted.getAllDevices() - for device in devices: + devices = parted.getAllDevices() + for device in devices: dev_path = device.path try: disk = parted.Disk(device) @@ -268,7 +311,7 @@ def find_carbono(path): ret = True if filter(lambda x:not x in dev_files, CARBONO_FILES2): ret = False - return ret + return ret def mount_point(device): @@ -286,11 +329,11 @@ def mount_point(device): raise ErrorIdentifyDevice("Erro na identificação do Pendrive") def get_upimage_device(): - devices = get_devices() + devices = get_devices() for dev in devices: device = devices[dev]["partitions"].keys() if is_mounted(device[0]): - mount_path = mount_point(device[0]) + mount_path = mount_point(device[0]) else: mount_path = mount_pen(device[0]) ret = find_carbono(mount_path) @@ -350,12 +393,12 @@ def check_if_root(): if os.getuid() == 0: return True return False - + def verify_4k(hd = "sda"): ''' Retorna o tamanho fisico do setor ''' - try: + try: f = open("/sys/block/{0}/queue/physical_block_size".format(hd)) block = f.readline() if "4096" in block: @@ -364,5 +407,5 @@ def verify_4k(hd = "sda"): return(512) except Exception as e: #nao tem HD (uma vm sem hd, por exemplo) - return(512) - + return(512) + From f50a10828c23c390f2f2a3c763a7e300ae1fdf0b Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Fri, 25 Oct 2013 10:58:15 -0200 Subject: [PATCH 39/99] Implementacao das funcoes de Manipulacao de disco(carbono.utils) Implementacao das funcoes de disco, no codigo foi implementado a classe DiskPartition. Os metodos implementados sao: - get_partition, acessa o nome da particao - umount_partition, desmonta a particao, de todos os lugares montados - umount_all_partitions, desmonta todas as particoes - mount_partition, monta a particao em uma pasta temporaria se nao tiver o destino determinado ou monta a particao no destino se o mesmo estiver setado no parametro - get_mount_point, passado o nome do disco, a funcao retorna todos os pontos que a particao esta montada. - get_mounted_devices, retorna todos os mount point dos discos --- carbono/utils.py | 107 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index a718cf4..e300377 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -124,15 +124,16 @@ def formated_partitions(self): class DiskPartition(): - def __init__(self): + def __init__(self, partition = ""): self.__temp_folder = "" - self.__partition = "" + self.__partition = partition def __generate_temp_folder(self, destino = "/tmp/"): + + self.__temp_folder = adjust_path(tempfile.mkdtemp()) - #Do generate the temp folder - self.__temp_folder = "pasta gerada" + return self.__temp_folder def __set_partition(self, partition): @@ -142,28 +143,98 @@ def get_partition(self): return self.__partition - def umount_partition(self): - pass - #desmonta self.__partition + def umount_partition(self, device = None): + disk_mounted = self.get_mounted_devices() + disk_list = [] + result_disk_umounted = {} + if self.__partition not in disk_mounted.keys(): + return "{0} is already umounted".format(self.__partition) + else: + disk_list = disk_mounted[self.__partition] + for item in disk_list: + cmd = "umount {0}".format(item) + p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE, shell=True) + ret = os.waitpid(p.pid,0)[1] + if not ret: + result_disk_umounted[item] = 0 + print "A particao {0} foi desmontada do diretorio {1}".format(self.__partition, item) + else: + result_disk_umounted[item] = -1 + print "A particao {0} montada em {1} nao foi desmontada".format(self.__partition,item) + return result_disk_umounted def umount_all_partitions(self): - pass - + disk_mounted = self.get_mounted_devices() + result_part_umounted = {} + result_disk_umounted = {} + disk_list = [] + for item in disk_mounted.keys(): + disk_list = disk_mounted[item] + result_part_umounted = {} + for part in disk_list: + cmd = "umount {0}".format(item) + p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE, shell=True) + ret = os.waitpid(p.pid,0)[1] + if not ret: + result_part_umounted[part] = 0 + else: + result_part_umounted[part] = -1 + result_disk_umounted[item] = result_part_umounted + return result_disk_umounted + + def mount_partition(self,destino = None): mounted_folder = "" if destino is None: - pass - #gera pasta temporario - else: - pass - #monta no destino - return mounted_folder - - def get_mounted_devices(self): + mounted_folder = adjust_path(tempfile.mkdtemp()) + cmd = "mount {0} {1}".format(self.__partition, mounted_folder) + p = subprocess.Popen(cmd,stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + ret = os.waitpid(p.pid,0)[1] + if not ret: + return mounted_folder + else: + return False - pass + else: + cmd = "mount {0} {1}".format(self.__partition, destino) + p = subprocess.Popen(cmd,stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + ret = os.waitpid(p.pid,0)[1] + if not ret: + return destino + else: + return False + def get_mount_point(self, device = None): + if device is not None: + mounted_dest = self.get_mounted_devices() + if device not in mounted_dest.keys(): + return False + else: + return mounted_dest[device] + else: + return False + + def get_mounted_devices(self, device = None): + disk_mounted = {} + list_dest = [] + mount_command = subprocess.check_output(['mount','-l']).split('\n') + for lines in mount_command: + line = lines.split(' ') + if line[0].startswith('/dev/sd'): + if line[0] not in disk_mounted.keys(): + list_dest = [] + list_dest.append(line[2]) + disk_mounted[line[0]] = list_dest + list_dest = [] + else: + list_dest = [] + for element in xrange(len(disk_mounted[line[0]])): + list_dest.append(disk_mounted[line[0]][element]) + list_dest.append(line[2]) + disk_mounted[line[0]] = list_dest + list_dest = [] + return disk_mounted class Timer(Thread): def __init__(self, callback, timeout=2): From e8bd6585b0146477846b59e622fd95fedede8914 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Thu, 7 Nov 2013 15:22:46 -0200 Subject: [PATCH 40/99] Adicao do Tratamento de erro quando pretende-se restaurar Adicao do tratamento de erro quando a particao ou disco destino sao menores do que o a o HD que foi originalmente extraida a ima- gem, gerando assim um alerta na interface ao usuario informando o problema. --- carbono/image_restorer.py | 29 +++++++++++++++++++++++++++-- carbono/utils.py | 7 +++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 4afe3a0..0f0a1f6 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -30,6 +30,7 @@ from carbono.exception import * from carbono.utils import * from carbono.config import * + from carbono.log import log from partition_expander import PartitionExpander @@ -103,6 +104,17 @@ def restore_image(self): raise ErrorRestoringImage("Unrecognized disk label") if information.get_image_is_disk(): + #Get total disk target size + disk_size = get_disk_size(self.target_device) + if (total_bytes > disk_size): + log.info("Total size of image is {0}".format(total_bytes)) + log.info("Total size of {0} is {1}".format(self.target_device,disk_size)) + log.error("The size of {0} is {1}, is not enough to apply the selected image".format(self.target_device, disk_size)) + disk_space_info = [] + disk_space_info.append(total_bytes) + disk_space_info.append(disk_size) + self.notify_status("no_enough_space", {"disk_minimum_size":disk_space_info}) + raise ErrorRestoringImage("No enough space on disk") log.info("Restoring MBR and Disk Layout") mbr = Mbr(self.image_path) mbr.restore_from_file(self.target_device) @@ -111,7 +123,21 @@ def restore_image(self): dlm.restore_from_file(disk, True) else: dlm.restore_from_file(disk, False) - + else: + parent_path = get_parent_path(self.target_device) + parent_device = Device(parent_path) + parent_disk = Disk(parent_device) + partition = parent_disk.get_partition_by_path( + self.target_device, + part.type) + part_size = partition.getSize('b') + if (total_bytes > part_size): + part_space_info = [] + part_space_info.append(total_bytes) + part_space_info.append(part_size) + log.error("The partition selected is smaller than the image") + self.notify_status("no_enough_space_part", {"disk_minimum_size":part_space_info}) + raise ErrorRestoringImage("No enought space on partition") self.timer.start() for part in partitions: if not self.active: break @@ -193,7 +219,6 @@ def restore_image(self): partition.filesystem.close() self.timer.stop() - log.info(self.expand) if self.expand != 2: if information.get_image_is_disk(): diff --git a/carbono/utils.py b/carbono/utils.py index e300377..0afeec5 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -291,6 +291,13 @@ def run_simple_command(cmd): p.wait() return p.returncode +def get_disk_size(path): + devices = parted.getAllDevices() + for device in devices: + if (device.path == path): + return device.getSize('b') + return False + def random_string(length=5): return ''.join([random.choice(tempfile._RandomNameSequence.characters) \ for i in range(length)]) From 8e535c354a8d54fa585034c4f4fa1bf3abd89325 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Thu, 7 Nov 2013 15:50:39 -0200 Subject: [PATCH 41/99] Correcao da expansao da ultima particao sendo ela NTFS/EXT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foi adicionado o tratamento da memória livre, como eh nativo do sistema deixar alguns bytes entre as particoes, ao liberar a par- ticao que se quer expandir o sistema incorpora os bytes anterior a memória fazendo que dê erro na expansao. Com o tratamento se tem uma atribuicao para que o comeco da particao livre sempre seja o comeco da memoria que se quer expandir. --- carbono/filesystem/ext.py | 3 ++- carbono/filesystem/ntfs.py | 8 +++++--- carbono/image_restorer.py | 6 +++--- carbono/partition_expander.py | 15 +++++++++++---- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/carbono/filesystem/ext.py b/carbono/filesystem/ext.py index 67c9608..546d621 100755 --- a/carbono/filesystem/ext.py +++ b/carbono/filesystem/ext.py @@ -124,9 +124,10 @@ def resize(self): if self.check(): proc = subprocess.Popen([which("resize2fs"), self.path], stdout=subprocess.PIPE) + proc.wait() output = proc.stdout.read() output = output.strip() - if output == 0: + if proc.returncode == 0: return True return False diff --git a/carbono/filesystem/ntfs.py b/carbono/filesystem/ntfs.py index 9a8c92a..144e185 100755 --- a/carbono/filesystem/ntfs.py +++ b/carbono/filesystem/ntfs.py @@ -100,7 +100,8 @@ def format_filesystem(self): return False def resize(self): - + + returncode = -1 if self.check(): proc = subprocess.Popen([which("ntfsresize"), self.path], stdout=subprocess.PIPE) @@ -108,9 +109,10 @@ def resize(self): output = proc.stdout.read() output = output.strip() returncode = proc.returncode + if returncode == 0: - return returncode, True - return returncode, False + return True + return False def read_label(self): proc = subprocess.Popen([which("ntfslabel"), "--force", self.path], diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 0f0a1f6..4c3ecea 100644 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -249,9 +249,9 @@ def expand_last_partition(self,opt_expand): if partition is not None: if partition.type == PARTITION_NORMAL: expander = PartitionExpander(device.path) - log.info("checking and try expand {0}".format(partition.get_path())) + log.info("Checking and try expand {0}".format(partition.get_path())) new_size = expander.try_expand() - log.info("new_size {0}".format(new_size)) + log.info("The new_size of the disk will be {0}".format(new_size)) if new_size!= -1: if opt_expand == 0: @@ -259,7 +259,7 @@ def expand_last_partition(self,opt_expand): self.notify_status("expand", {"device": partition.get_path()}) returncode = partition.filesystem.resize() - if returncode == 0: + if returncode == True: log.info("Resize in {0} was made with sucess".format(partition.get_path())) else: log.info("Resize in {0} failed".format(partition.get_path())) diff --git a/carbono/partition_expander.py b/carbono/partition_expander.py index 8dc32e7..5af0449 100644 --- a/carbono/partition_expander.py +++ b/carbono/partition_expander.py @@ -52,10 +52,17 @@ def try_expand(self): disk.removePartition(last_partition) new_geometry = None - for region in disk.getFreeSpaceRegions(): - if region.start == start: - new_geometry = region - + if (len(disk.getFreeSpaceRegions()) == 1): + new_geometry = disk.getFreeSpaceRegions()[0] + new_geometry.start = start + else: + new_geometry = None + for region in disk.getFreeSpaceRegions(): + if region.start == start: + new_geometry = region + if (start > region.start and start < region.end): + new_geometry = region + new_geometry.start = start constraint = parted.Constraint(exactGeom=new_geometry) new_partition = parted.Partition(disk = disk, type = parted.PARTITION_NORMAL, From b483c4ae4344d26be74ce81c09fd47e5d633a1dc Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 12 Nov 2013 09:08:18 -0200 Subject: [PATCH 42/99] Change DiskInfo return msg * Change diskinfo return dictionay format, to have more info about the devices --- carbono/utils.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index e300377..01eeff7 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -35,6 +35,10 @@ def __init__(self): self.__DISK_DICT = {} self.__PARTITION_DICT = {} + # self.disk_info = {"/dev/sda":{"size":123123124, + # "label":model:self.__DISK_DICT, + # "partitions":[]} + def __get_devices(self): ''' Filter = only partitions with a valid filesystem ''' @@ -51,7 +55,6 @@ def __get_devices(self): part_dict = {} for p in disk.partitions: - if p.type not in (parted.PARTITION_NORMAL, parted.PARTITION_LOGICAL): continue @@ -90,10 +93,11 @@ def __collect_information_about_devices(self): def formated_partitions(self): formated_partitions = [] - formated_partitions_dict = {} + formated_partitions_dict = self.__DISK_DICT self.__collect_information_about_devices() - for part in self.__PARTITION_DICT.keys(): + device_info = {"size":None,"label":None,"partitions":None} + for part in self.__PARTITION_DICT.keys(): part_type = self.__PARTITION_DICT[part]['type'] size_bytes = self.__PARTITION_DICT[part]['size'] size_mb = int(long(size_bytes)/(1024*1024.0)) @@ -111,16 +115,15 @@ def formated_partitions(self): if temp_disk == disk: temp_parts.append(formated_partitions[aux]) else: - formated_partitions_dict[disk] = temp_parts - disk = temp_disk + formated_partitions_dict[disk]["partitions"] = temp_parts temp_parts = [] - formated_partitions_dict[disk] = temp_parts - + temp_parts.append(formated_partitions[aux]) + disk = temp_disk + formated_partitions_dict[disk]["partitions"] = temp_parts return(formated_partitions_dict) - class DiskPartition(): @@ -130,7 +133,7 @@ def __init__(self, partition = ""): self.__partition = partition def __generate_temp_folder(self, destino = "/tmp/"): - + self.__temp_folder = adjust_path(tempfile.mkdtemp()) return self.__temp_folder @@ -159,7 +162,7 @@ def umount_partition(self, device = None): result_disk_umounted[item] = 0 print "A particao {0} foi desmontada do diretorio {1}".format(self.__partition, item) else: - result_disk_umounted[item] = -1 + result_disk_umounted[item] = -1 print "A particao {0} montada em {1} nao foi desmontada".format(self.__partition,item) return result_disk_umounted @@ -171,18 +174,18 @@ def umount_all_partitions(self): for item in disk_mounted.keys(): disk_list = disk_mounted[item] result_part_umounted = {} - for part in disk_list: + for part in disk_list: cmd = "umount {0}".format(item) p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE, shell=True) ret = os.waitpid(p.pid,0)[1] if not ret: result_part_umounted[part] = 0 else: - result_part_umounted[part] = -1 - result_disk_umounted[item] = result_part_umounted + result_part_umounted[part] = -1 + result_disk_umounted[item] = result_part_umounted return result_disk_umounted - + def mount_partition(self,destino = None): mounted_folder = "" From 797d19292dbc7ed1d62bc58b33504880587ed001 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 19 Nov 2013 17:44:27 -0200 Subject: [PATCH 43/99] Add mount verification --- carbono/utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/carbono/utils.py b/carbono/utils.py index 7252166..a1b1db2 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -188,7 +188,13 @@ def umount_all_partitions(self): def mount_partition(self,destino = None): + mounted_folder = self.get_mount_point(self.__partition) + + if mounted_folder: + return mounted_folder + mounted_folder = "" + if destino is None: mounted_folder = adjust_path(tempfile.mkdtemp()) cmd = "mount {0} {1}".format(self.__partition, mounted_folder) @@ -295,7 +301,7 @@ def run_simple_command(cmd): return p.returncode def get_disk_size(path): - devices = parted.getAllDevices() + devices = parted.getAllDevices() for device in devices: if (device.path == path): return device.getSize('b') From 859c118a022354566fdf7f1b75664a85341e1c15 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 19 Nov 2013 17:57:39 -0200 Subject: [PATCH 44/99] Change mounted folder return --- carbono/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carbono/utils.py b/carbono/utils.py index a1b1db2..cdb7dac 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -191,7 +191,7 @@ def mount_partition(self,destino = None): mounted_folder = self.get_mount_point(self.__partition) if mounted_folder: - return mounted_folder + return mounted_folder[0] mounted_folder = "" From 4abc6c5ae9b1d5a3b37934587c39cd8f44b51ef3 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 2 Dec 2013 15:44:03 -0200 Subject: [PATCH 45/99] change disk_info in utils *Change partitions size return from MB to bytes --- carbono/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carbono/utils.py b/carbono/utils.py index cdb7dac..3f65b75 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -60,7 +60,7 @@ def __get_devices(self): continue part_path = p.path - part_size = p.getSize('b') + part_size =1024*1024*int(p.getSize('b')) part_type = "unknown" if p.fileSystem: part_type = p.fileSystem.type From a4193c23d1e635d496042a10e89232f0b71d7f14 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Wed, 4 Dec 2013 10:52:49 -0200 Subject: [PATCH 46/99] Adicao da chamada da interface caso ocorra erro na expansao Adicao da chamada do Alerta da interface caso ocorra erro na expansao da ultima particao. --- carbono/image_restorer.py | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 carbono/image_restorer.py diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py old mode 100644 new mode 100755 index 4c3ecea..76d1f39 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -263,6 +263,8 @@ def expand_last_partition(self,opt_expand): log.info("Resize in {0} was made with sucess".format(partition.get_path())) else: log.info("Resize in {0} failed".format(partition.get_path())) + self.notify_status("expand_last_partition_error", {"last_partition":partition.get_path()}) + self.canceled = True else: if opt_expand == 1: log.info("Formating {0} filesystem".format(partition.get_path())) From 5384b61cc71c16c0fda859e09a621dc233a8314c Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Fri, 6 Dec 2013 17:43:06 -0200 Subject: [PATCH 47/99] Implementacao de extracao com o partclone para sistema EXT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No comite foi implementado o contador de blocos que estao ocupados em cada particao. E considerando que o partclone faz a extracao apenas utilizando os blocos que contêm dados uteis, foi implementado a comparacao entre os blocos lidos no stdout do partclone e o numero de blocos que estão ocupados, e ao se diagnos- ticar que o numero de blocos lidos é maior que o numero de blocos ocupados, entao é parado o programa, impedindo assim, que o processo do partclone pare. --- carbono/buffer_manager/dummy_manager.py | 17 +++++++++-- carbono/config.py | 3 +- carbono/image_creator.py | 24 ++++------------ carbono/image_restorer.py | 1 + carbono/utils.py | 38 +++++++++++++++++++------ 5 files changed, 51 insertions(+), 32 deletions(-) diff --git a/carbono/buffer_manager/dummy_manager.py b/carbono/buffer_manager/dummy_manager.py index 6fdf2ed..9034cfb 100644 --- a/carbono/buffer_manager/dummy_manager.py +++ b/carbono/buffer_manager/dummy_manager.py @@ -20,6 +20,7 @@ from carbono.utils import * from carbono.config import * +from carbono.log import log class DummyManager(Thread): @@ -39,14 +40,26 @@ def put(self, data): def run(self): self.active = True + count_read_block = 0 while self.active: - try: - data = self.read_block() + try: + if get_part_type() == '': + data = self.read_block() + else: + if get_part_type() != 'ntfs': + if (get_block_used()+3) >= count_read_block: + count_read_block = count_read_block +1 + data = self.read_block() + else: + data = '' + else: + data = self.read_block() except ErrorReadingFromDevice, e: self.stop() raise e if not data: + set_part_type('') self.stop() break self.put(data) diff --git a/carbono/config.py b/carbono/config.py index 5fb5418..48c76df 100755 --- a/carbono/config.py +++ b/carbono/config.py @@ -18,8 +18,7 @@ import os -SUCESS_PARTCLONE = False -OLD_PERCENT = -2 +BLOCK_USED = 0 CURRENT_PARTITION_TYPE = "" FILE_PATTERN = "{name}-{partition}-{volume}.data" BLOCK_SIZE = 1048576 # 1MB diff --git a/carbono/image_creator.py b/carbono/image_creator.py index e4e65e8..845af8a 100755 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -67,22 +67,6 @@ def notify_percent(self): self.current_percent = percent self.notify_status("progress", {"percent": percent}) - - global OLD_PERCENT - global CURRENT_PARTITION_TYPE - global SUCESS_PARTCLONE - - if (SUCESS_PARTCLONE == False) and (percent > 100) and (percent == OLD_PERCENT) and (CURRENT_PARTITION_TYPE != 'ntfs'): - log.info(CURRENT_PARTITION_TYPE) - time.sleep(18) - cmd = "kill -9 $(ps ax|grep '{0} -c -s {1} -o -'|awk '{2}')".format('/usr/sbin/partclone.extfs',self.device_path, '{print $1}') - try: - self.process = RunCmd(cmd) - self.process.run() - except Exception as e: - log.info(e) - OLD_PERCENT = percent - def create_image(self): """ """ if is_mounted(self.device_path): @@ -144,9 +128,12 @@ def create_image(self): remaining_size -= BASE_SYSTEM_SIZE slices = dict() # Used when creating iso iso_volume = 1 # Used when creating iso + global BLOCK_USED for part in partition_list: if not self.active: break - + total_bytes = part.filesystem.get_used_size() + BLOCK_USED = long(math.ceil(total_bytes/float(BLOCK_SIZE))) + set_block_used(BLOCK_USED) log.info("Creating image of {0}".format(part.get_path())) number = part.get_number() uuid = part.filesystem.uuid() @@ -154,6 +141,7 @@ def create_image(self): type = part.filesystem.type global CURRENT_PARTITION_TYPE CURRENT_PARTITION_TYPE = type + set_part_type(type) part.filesystem.open_to_read() compact_callback = None @@ -260,8 +248,6 @@ def create_image(self): self.notify_status("canceled", {"operation": "Create image"}) else: - global SUCESS_PARTCLONE - SUCESS_PARTCLONE = True self.notify_status("finish") log.info("Creation finished") diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 76d1f39..b30b4a0 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -219,6 +219,7 @@ def restore_image(self): partition.filesystem.close() self.timer.stop() + log.info(self.expand) if self.expand != 2: if information.get_image_is_disk(): diff --git a/carbono/utils.py b/carbono/utils.py index 0afeec5..887462f 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -26,6 +26,7 @@ from threading import Thread, Event from os.path import realpath from carbono.exception import * +from carbono.config import * @@ -357,8 +358,8 @@ def available_memory(percent=100): def get_devices(): disk_dict = {} - devices = parted.getAllDevices() - for device in devices: + devices = parted.getAllDevices() + for device in devices: dev_path = device.path try: disk = parted.Disk(device) @@ -389,7 +390,7 @@ def find_carbono(path): ret = True if filter(lambda x:not x in dev_files, CARBONO_FILES2): ret = False - return ret + return ret def mount_point(device): @@ -407,11 +408,11 @@ def mount_point(device): raise ErrorIdentifyDevice("Erro na identificação do Pendrive") def get_upimage_device(): - devices = get_devices() + devices = get_devices() for dev in devices: device = devices[dev]["partitions"].keys() if is_mounted(device[0]): - mount_path = mount_point(device[0]) + mount_path = mount_point(device[0]) else: mount_path = mount_pen(device[0]) ret = find_carbono(mount_path) @@ -457,6 +458,25 @@ def is_exe(fpath): raise CommandNotFound("{0}: command not found".\ format(program)) +def set_block_used(value): + global BLOCK_USED + BLOCK_USED = value + return value + +def get_block_used(): + global BLOCK_USED + return BLOCK_USED + +global CURRENT_PARTITION_TYPE +def set_part_type(value): + global CURRENT_PARTITION_TYPE + CURRENT_PARTITION_TYPE = value + return CURRENT_PARTITION_TYPE + +def get_part_type(): + global CURRENT_PARTITION_TYPE + return CURRENT_PARTITION_TYPE + def sync(): run_simple_command("sync") @@ -471,12 +491,12 @@ def check_if_root(): if os.getuid() == 0: return True return False - + def verify_4k(hd = "sda"): ''' Retorna o tamanho fisico do setor ''' - try: + try: f = open("/sys/block/{0}/queue/physical_block_size".format(hd)) block = f.readline() if "4096" in block: @@ -485,5 +505,5 @@ def verify_4k(hd = "sda"): return(512) except Exception as e: #nao tem HD (uma vm sem hd, por exemplo) - return(512) - + return(512) + From 9231235e13ac35e7ce43022cc013c79d593140e8 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Wed, 18 Dec 2013 17:00:15 -0200 Subject: [PATCH 48/99] [ERROR_MISSED_FILE] Adicao da chamada para imagens incompletas No codigo do carbono, no arquivo image_restore.py foi adicionado as chamadas da interface relativo as imagens que estiverem com arquivos [mbr.bin, disk.dl] faltando. Antes apenas era registrado no log o IOError e travava a interface. --- carbono/image_restorer.py | 40 +++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index b30b4a0..407d8f7 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -115,14 +115,25 @@ def restore_image(self): disk_space_info.append(disk_size) self.notify_status("no_enough_space", {"disk_minimum_size":disk_space_info}) raise ErrorRestoringImage("No enough space on disk") + log.info("Restoring MBR and Disk Layout") mbr = Mbr(self.image_path) - mbr.restore_from_file(self.target_device) + try: + mbr.restore_from_file(self.target_device) + except Exception as e: + log.error("Error to restore the Mbr file") + image_path = self.image_path.split("/")[3] + "/mbr.bin" + self.notify_status("file_not_found",{"file_not_found":image_path}) dlm = DiskLayoutManager(self.image_path) - if self.expand != 2: - dlm.restore_from_file(disk, True) - else: - dlm.restore_from_file(disk, False) + try: + if self.expand != 2: + dlm.restore_from_file(disk, True) + else: + dlm.restore_from_file(disk, False) + except Exception as e: + log.error("Error to restore the disk.dl file") + image_path = self.image_path.split("/")[3] + "/disk.dl" + self.notify_status("file_not_found",{"file_not_found":image_path}) else: parent_path = get_parent_path(self.target_device) parent_device = Device(parent_path) @@ -138,6 +149,7 @@ def restore_image(self): log.error("The partition selected is smaller than the image") self.notify_status("no_enough_space_part", {"disk_minimum_size":part_space_info}) raise ErrorRestoringImage("No enought space on partition") + self.timer.start() for part in partitions: if not self.active: break @@ -219,7 +231,6 @@ def restore_image(self): partition.filesystem.close() self.timer.stop() - log.info(self.expand) if self.expand != 2: if information.get_image_is_disk(): @@ -230,14 +241,15 @@ def restore_image(self): "Restore image"}) else: self._finish() - log.info("Restoration finished") - log.info("Iniciando gtk grubinstall") - cmd = "{0}".format(which("grubinstall")) - try: - self.process = RunCmd(cmd) - self.process.run() - except Exception as e: - log.error("Erro ao iniciar grubinstall. {0}".format(e)) + log.info("Restoration finished") + log.info("Iniciando gtk grubinstall") + cmd = "{0}".format(which("grubinstall")) + try: + self.process = RunCmd(cmd) + self.process.run() + self.process.wait() + except Exception as e: + log.error("Erro ao iniciar grubinstall. {0}".format(e)) def expand_last_partition(self,opt_expand): # After all data is copied to the disk From b511c16d114bc0bbfb5046f57d6c1a4c4c402368 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Wed, 18 Dec 2013 17:37:16 -0200 Subject: [PATCH 49/99] [ERROR_MOUNTED_ON_CREATE] Tratamento de erro na extracao Adicionado o tratamento de erro no image_creator.py quando a particao o qual sera extraida a imagem esta montada, impedindo o carbono inicie o procedimento. Anteriormente era adicionado o log a informacao de que a particao estava montada podem, nao havia resposta a interface, o qual fica- va travada. --- carbono/image_creator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/carbono/image_creator.py b/carbono/image_creator.py index 845af8a..4973019 100755 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -70,6 +70,8 @@ def notify_percent(self): def create_image(self): """ """ if is_mounted(self.device_path): + log.error("The partition {0} is mounted, please umount first, and try again".format(self.device_path)) + self.notify_status("mounted_partition_error",{"mounted_partition_error":self.device_path}) raise DeviceIsMounted("Please umount first") self.active = True From add6d6e131f5e10ccc2b8309ef6f92b1cd0ed0fd Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Fri, 20 Dec 2013 08:38:07 -0200 Subject: [PATCH 50/99] Adicao da chamada de erro se a particao o qual vai ser aplicada a imagem esta montada. --- carbono/image_restorer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 407d8f7..de8b5d1 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -68,6 +68,8 @@ def notify_percent(self): def restore_image(self): """ """ if is_mounted(self.target_device): + log.error("The partition {0} is mounted, please umount first, and try again".format(self.target_device)) + self.notify_status("mounted_partition_error",{"mounted_partition_error":self.target_device}) raise DeviceIsMounted("Please umount first") self.active = True From 6aed60be4df1ce38bc4431eb8e8ab649b255a3cb Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Mon, 30 Dec 2013 17:05:08 -0200 Subject: [PATCH 51/99] [PARTCLONE] Modificacao e teste do partclone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foi adicionado todas as modificações para o funcionamento do partclone integrado com o carbono. O problema identificado de leitura do sico foi solucionado, pois quando se utiliza o subprocess o partclone fornece saida (stdout) para a leitura do disco, alem disso o partclone oferece a resposta do programa em outra saida (stderr) que até entao nao era considerada pelo carbono. --- carbono/buffer_manager/dummy_manager.py | 19 +- carbono/config.py | 2 - carbono/filesystem/__init__.py | 5 +- carbono/filesystem/ext.py | 1 + carbono/filesystem/generic.py | 1 + carbono/image_creator.py | 28 ++- carbono/partclone.py | 262 ++++++++++++++++++++++++ carbono/utils.py | 19 -- 8 files changed, 288 insertions(+), 49 deletions(-) mode change 100755 => 100644 carbono/filesystem/ext.py mode change 100755 => 100644 carbono/image_creator.py create mode 100644 carbono/partclone.py diff --git a/carbono/buffer_manager/dummy_manager.py b/carbono/buffer_manager/dummy_manager.py index 9034cfb..b132e49 100644 --- a/carbono/buffer_manager/dummy_manager.py +++ b/carbono/buffer_manager/dummy_manager.py @@ -40,26 +40,9 @@ def put(self, data): def run(self): self.active = True - count_read_block = 0 while self.active: - try: - if get_part_type() == '': - data = self.read_block() - else: - if get_part_type() != 'ntfs': - if (get_block_used()+3) >= count_read_block: - count_read_block = count_read_block +1 - data = self.read_block() - else: - data = '' - else: - data = self.read_block() - except ErrorReadingFromDevice, e: - self.stop() - raise e - + data = self.read_block() if not data: - set_part_type('') self.stop() break self.put(data) diff --git a/carbono/config.py b/carbono/config.py index 48c76df..80c2f8b 100755 --- a/carbono/config.py +++ b/carbono/config.py @@ -18,8 +18,6 @@ import os -BLOCK_USED = 0 -CURRENT_PARTITION_TYPE = "" FILE_PATTERN = "{name}-{partition}-{volume}.data" BLOCK_SIZE = 1048576 # 1MB BASE_SYSTEM_SIZE = 0 #TODO (in bytes) diff --git a/carbono/filesystem/__init__.py b/carbono/filesystem/__init__.py index 6c56e34..4f5dc58 100644 --- a/carbono/filesystem/__init__.py +++ b/carbono/filesystem/__init__.py @@ -53,7 +53,10 @@ def _get_filesystem_instance(self): def open_to_read(self): self._fs.open_to_read() - + + def get_error_ext(self): + return self._fs.fderr + def open_to_write(self, uuid=None): if uuid is not None: self._fs.open_to_write(uuid) diff --git a/carbono/filesystem/ext.py b/carbono/filesystem/ext.py old mode 100755 new mode 100644 index 546d621..a15826e --- a/carbono/filesystem/ext.py +++ b/carbono/filesystem/ext.py @@ -58,6 +58,7 @@ def open_to_read(self): self.process = RunCmd(cmd) self.process.run() self._fd = self.process.stdout + self.fderr = self.process.stderr except: raise ErrorOpenToRead("Cannot open {0} to read".format(self.path)) diff --git a/carbono/filesystem/generic.py b/carbono/filesystem/generic.py index 06cd47b..552dcaf 100644 --- a/carbono/filesystem/generic.py +++ b/carbono/filesystem/generic.py @@ -28,6 +28,7 @@ def __init__(self, path, type, geometry): self.type = type self.geometry = geometry self._fd = None + self.fderr = None self.process = None def get_size(self): diff --git a/carbono/image_creator.py b/carbono/image_creator.py old mode 100755 new mode 100644 index 4973019..08c97e4 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -60,13 +60,21 @@ def __init__(self, source_device, output_folder, self.current_percent = -1 self.active = False self.canceled = False + self.partclone_stderr = None + self.partclone_sucess = False def notify_percent(self): + #refresh the interface percentage percent = (self.processed_blocks/float(self.total_blocks)) * 100 if percent > self.current_percent: self.current_percent = percent self.notify_status("progress", {"percent": percent}) + #verify stderr from parclone + if self.partclone_stderr != None: + if self.partclone_stderr.readline().startswith("Partclone successfully cloned the device"): + self.partclone_sucess = True + def create_image(self): """ """ if is_mounted(self.device_path): @@ -113,9 +121,8 @@ def create_image(self): total_bytes = 0 for part in partition_list: total_bytes += part.filesystem.get_used_size() - - self.total_blocks = long(math.ceil(total_bytes/float(BLOCK_SIZE))) + self.total_blocks = long(math.ceil(total_bytes/float(BLOCK_SIZE))) information = Information(self.target_path) information.set_image_is_disk(device.is_disk()) information.set_image_name(self.image_name) @@ -130,22 +137,20 @@ def create_image(self): remaining_size -= BASE_SYSTEM_SIZE slices = dict() # Used when creating iso iso_volume = 1 # Used when creating iso - global BLOCK_USED + for part in partition_list: if not self.active: break total_bytes = part.filesystem.get_used_size() - BLOCK_USED = long(math.ceil(total_bytes/float(BLOCK_SIZE))) - set_block_used(BLOCK_USED) - log.info("Creating image of {0}".format(part.get_path())) number = part.get_number() uuid = part.filesystem.uuid() label = part.filesystem.read_label() type = part.filesystem.type - global CURRENT_PARTITION_TYPE - CURRENT_PARTITION_TYPE = type - set_part_type(type) part.filesystem.open_to_read() + #check if partclone is running + if type in ("ext2","ext3","ext4"): + self.partclone_stderr = part.filesystem.get_error_ext() + compact_callback = None if self.compressor_level: compressor = Compressor(self.compressor_level) @@ -176,6 +181,11 @@ def create_image(self): break if data == EOF: + if (self.partclone_stderr != None): + while self.partclone_sucess == False: + time.sleep(0.5) + self.notify_status("waiting_partclone") + time.sleep(0.5) next_partition = True if self.create_iso: remaining_size -= total_written diff --git a/carbono/partclone.py b/carbono/partclone.py new file mode 100644 index 0000000..e8dd2a2 --- /dev/null +++ b/carbono/partclone.py @@ -0,0 +1,262 @@ +#!/usr/bin/python +# coding: utf-8 +# Copyright (C) 2010 Lucas Alvares Gomes +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +import math +import os +import time +import shutil +from carbono.device import Device +from carbono.disk import Disk +from carbono.mbr import Mbr +from carbono.disk_layout_manager import DiskLayoutManager +from carbono.information import Information +from carbono.compressor import Compressor +from carbono.buffer_manager import BufferManagerFactory +from carbono.iso_creator import IsoCreator +from carbono.exception import * +from carbono.utils import * +from carbono.config import * + +from carbono.log import log + +class Partclone: + + def __init__(self, source_device, output_folder, + status_callback, image_name="image", compressor_level=6, + raw=False, split_size=0, create_iso=False, + fill_with_zeros=False): + + assert check_if_root(), "You need to run this application as root" + assert os.path.isdir(output_folder), "{0} folder is invalid".\ + format(output_folder) + + self.image_name = image_name + self.device_path = source_device + self.target_path = adjust_path(output_folder) + self.notify_status = status_callback + self.compressor_level = compressor_level + self.raw = raw + self.split_size = split_size + self.create_iso = create_iso + self.fill_with_zeros = fill_with_zeros + + self.timer = Timer(self.notify_percent) + self.total_blocks = 0 + self.processed_blocks = 0 + self.current_percent = -1 + self.active = False + self.canceled = False + + def notify_percent(self): + percent = (self.processed_blocks/float(self.total_blocks)) * 100 + if percent > self.current_percent: + self.current_percent = percent + self.notify_status("progress", {"percent": percent}) + + def create_image(self): + """ """ + if is_mounted(self.device_path): + log.error("The partition {0} is mounted, please umount first, and try again".format(self.device_path)) + self.notify_status("mounted_partition_error",{"mounted_partition_error":self.device_path}) + raise DeviceIsMounted("Please umount first") + + self.active = True + device = Device(self.device_path) + disk = Disk(device) + + if device.is_disk(): + mbr = Mbr(self.target_path) + mbr.save_to_file(self.device_path) + dlm = DiskLayoutManager(self.target_path) + dlm.save_to_file(disk) + + partition_list = disk.get_valid_partitions(self.raw) + if not partition_list: + raise ErrorCreatingImage("Partition(s) hasn't a " +\ + "valid filesystem") + + # check partitions filesystem + if not self.raw: + for part in partition_list: + log.info("Checking filesystem of {0}".format(part.get_path())) + self.notify_status("checking_filesystem", + {"device": part.get_path()}) + if not part.filesystem.check(): + log.error("{0} Filesystem is not clean".\ + format(part.get_path())) + raise ErrorCreatingImage("{0} Filesystem is not clean".\ + format(part.get_path())) + + # fill partitions with zeroes + if self.raw and self.fill_with_zeros: + for part in partition_list: + log.info("{0} Filling with zeros".format(part.get_path())) + self.notify_status("filling_with_zeros", {"device": + part.get_path()}) + part.filesystem.fill_with_zeros() + + # get total size + total_bytes = 0 + for part in partition_list: + total_bytes += part.filesystem.get_used_size() + + self.total_blocks = long(math.ceil(total_bytes/float(BLOCK_SIZE))) + + information = Information(self.target_path) + information.set_image_is_disk(device.is_disk()) + information.set_image_name(self.image_name) + information.set_image_compressor_level(self.compressor_level) + + # TODO: Abstract this whole part, when creating isos, + # splitting in files, etc... + + self.timer.start() + remaining_size = self.split_size + if self.create_iso: + remaining_size -= BASE_SYSTEM_SIZE + slices = dict() # Used when creating iso + iso_volume = 1 # Used when creating iso + global BLOCK_USED + for part in partition_list: + if not self.active: break + total_bytes = part.filesystem.get_used_size() + number = part.get_number() + uuid = part.filesystem.uuid() + label = part.filesystem.read_label() + type = part.filesystem.type + part.filesystem.open_to_read() + error_file = part.filesystem.get_error_ext() + compact_callback = None + if self.compressor_level: + compressor = Compressor(self.compressor_level) + compact_callback = compressor.compact + + self.buffer_manager = BufferManagerFactory( + part.filesystem.read_block, + compact_callback) + self.buffer_manager.start() + + buffer = self.buffer_manager.output_buffer + volumes = 1 + while self.active: + total_written = 0 # Used to help splitting the file + pattern = FILE_PATTERN.format(name=self.image_name, + partition=number, + volume=volumes) + file_path = self.target_path + pattern + fd = open(file_path, "wb") + + next_partition = False + while self.active: + try: + data = buffer.get() + except IOError, e: + if e.errno == errno.EINTR: + self.cancel() + break + log.info(error_file.read(4096)) + if data == EOF: + next_partition = True + if self.create_iso: + remaining_size -= total_written + if not slices.has_key(iso_volume): + slices[iso_volume] = list() + slices[iso_volume].append(file_path) + break + try: + fd.write(data) + except Exception as e: + log.info("{0}".format(e)) + self.notify_status("disk_full") + try: + shutil.rmtree(self.target_path) + log.info("The folder which cointain the images files were erased") + break + except Exception as e: + log.info(e) + break + self.cancel() + break + + self.processed_blocks += 1 + + if self.split_size: + bytes_written = len(data) + total_written += bytes_written + + if (total_written + bytes_written) >= remaining_size: + volumes += 1 + remaining_size = self.split_size + if self.create_iso: + if not slices.has_key(iso_volume): + slices[iso_volume] = list() + slices[iso_volume].append(file_path) + iso_volume += 1 + break + + fd.close() + if next_partition: break + + self.buffer_manager.join() + part.filesystem.close() + information.add_partition(number, type, volumes, + part.filesystem.get_used_size(), + uuid, label) + + # We dont need to save the data of the swap partition + # we just copy the informations and re-create when + # restoring + swap = disk.get_swap_partition() + if swap is not None: + log.info("Swap path {0}".format(swap.get_path())) + number = swap.get_number() + uuid = swap.filesystem.uuid() + type = swap.filesystem.type + information.add_partition(number, type, 0, 0, uuid) + + information.save() + self.stop() + + if self.create_iso: + log.info("Starting create ISO operation") + iso_creator = IsoCreator(self.target_path, slices, + self.image_name, + self.notify_status, + device.is_disk()) + iso_creator.run() + + if self.canceled: + log.info("Creation canceled") + self.notify_status("canceled", {"operation": + "Create image"}) + else: + self.notify_status("finish") + log.info("Creation finished") + + def stop(self): + if self.active: + if hasattr(self, "buffer_manager"): + self.buffer_manager.stop() + self.active = False + self.timer.stop() + log.info("Create image stopped") + + def cancel(self): + if not self.canceled: + log.info("Create image canceled") + self.canceled = True + self.stop() diff --git a/carbono/utils.py b/carbono/utils.py index 887462f..5dcc101 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -458,25 +458,6 @@ def is_exe(fpath): raise CommandNotFound("{0}: command not found".\ format(program)) -def set_block_used(value): - global BLOCK_USED - BLOCK_USED = value - return value - -def get_block_used(): - global BLOCK_USED - return BLOCK_USED - -global CURRENT_PARTITION_TYPE -def set_part_type(value): - global CURRENT_PARTITION_TYPE - CURRENT_PARTITION_TYPE = value - return CURRENT_PARTITION_TYPE - -def get_part_type(): - global CURRENT_PARTITION_TYPE - return CURRENT_PARTITION_TYPE - def sync(): run_simple_command("sync") From e43f5d903e3f38443fa69f202c68b9ab4f450501 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Mon, 30 Dec 2013 17:11:01 -0200 Subject: [PATCH 52/99] [Partclone] Adicao da operacao para o funcionamento do partclone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit relativo ao funcionamento do partclone, foi solucionado o erro de congelamento da leitura. Até entao o carbono ignorava a saida STDERR do partclone, nesse stderr é a resposta padrao do partclone identificando o status da extracao. apartir desse commit o carbono foi adaptado para receber duas saidas o STDOUT e o STDERR para o perfeito funcionamento do partclone --- carbono/partclone.py | 262 ------------------------------------------- 1 file changed, 262 deletions(-) delete mode 100644 carbono/partclone.py diff --git a/carbono/partclone.py b/carbono/partclone.py deleted file mode 100644 index e8dd2a2..0000000 --- a/carbono/partclone.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/python -# coding: utf-8 -# Copyright (C) 2010 Lucas Alvares Gomes -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - -import math -import os -import time -import shutil -from carbono.device import Device -from carbono.disk import Disk -from carbono.mbr import Mbr -from carbono.disk_layout_manager import DiskLayoutManager -from carbono.information import Information -from carbono.compressor import Compressor -from carbono.buffer_manager import BufferManagerFactory -from carbono.iso_creator import IsoCreator -from carbono.exception import * -from carbono.utils import * -from carbono.config import * - -from carbono.log import log - -class Partclone: - - def __init__(self, source_device, output_folder, - status_callback, image_name="image", compressor_level=6, - raw=False, split_size=0, create_iso=False, - fill_with_zeros=False): - - assert check_if_root(), "You need to run this application as root" - assert os.path.isdir(output_folder), "{0} folder is invalid".\ - format(output_folder) - - self.image_name = image_name - self.device_path = source_device - self.target_path = adjust_path(output_folder) - self.notify_status = status_callback - self.compressor_level = compressor_level - self.raw = raw - self.split_size = split_size - self.create_iso = create_iso - self.fill_with_zeros = fill_with_zeros - - self.timer = Timer(self.notify_percent) - self.total_blocks = 0 - self.processed_blocks = 0 - self.current_percent = -1 - self.active = False - self.canceled = False - - def notify_percent(self): - percent = (self.processed_blocks/float(self.total_blocks)) * 100 - if percent > self.current_percent: - self.current_percent = percent - self.notify_status("progress", {"percent": percent}) - - def create_image(self): - """ """ - if is_mounted(self.device_path): - log.error("The partition {0} is mounted, please umount first, and try again".format(self.device_path)) - self.notify_status("mounted_partition_error",{"mounted_partition_error":self.device_path}) - raise DeviceIsMounted("Please umount first") - - self.active = True - device = Device(self.device_path) - disk = Disk(device) - - if device.is_disk(): - mbr = Mbr(self.target_path) - mbr.save_to_file(self.device_path) - dlm = DiskLayoutManager(self.target_path) - dlm.save_to_file(disk) - - partition_list = disk.get_valid_partitions(self.raw) - if not partition_list: - raise ErrorCreatingImage("Partition(s) hasn't a " +\ - "valid filesystem") - - # check partitions filesystem - if not self.raw: - for part in partition_list: - log.info("Checking filesystem of {0}".format(part.get_path())) - self.notify_status("checking_filesystem", - {"device": part.get_path()}) - if not part.filesystem.check(): - log.error("{0} Filesystem is not clean".\ - format(part.get_path())) - raise ErrorCreatingImage("{0} Filesystem is not clean".\ - format(part.get_path())) - - # fill partitions with zeroes - if self.raw and self.fill_with_zeros: - for part in partition_list: - log.info("{0} Filling with zeros".format(part.get_path())) - self.notify_status("filling_with_zeros", {"device": - part.get_path()}) - part.filesystem.fill_with_zeros() - - # get total size - total_bytes = 0 - for part in partition_list: - total_bytes += part.filesystem.get_used_size() - - self.total_blocks = long(math.ceil(total_bytes/float(BLOCK_SIZE))) - - information = Information(self.target_path) - information.set_image_is_disk(device.is_disk()) - information.set_image_name(self.image_name) - information.set_image_compressor_level(self.compressor_level) - - # TODO: Abstract this whole part, when creating isos, - # splitting in files, etc... - - self.timer.start() - remaining_size = self.split_size - if self.create_iso: - remaining_size -= BASE_SYSTEM_SIZE - slices = dict() # Used when creating iso - iso_volume = 1 # Used when creating iso - global BLOCK_USED - for part in partition_list: - if not self.active: break - total_bytes = part.filesystem.get_used_size() - number = part.get_number() - uuid = part.filesystem.uuid() - label = part.filesystem.read_label() - type = part.filesystem.type - part.filesystem.open_to_read() - error_file = part.filesystem.get_error_ext() - compact_callback = None - if self.compressor_level: - compressor = Compressor(self.compressor_level) - compact_callback = compressor.compact - - self.buffer_manager = BufferManagerFactory( - part.filesystem.read_block, - compact_callback) - self.buffer_manager.start() - - buffer = self.buffer_manager.output_buffer - volumes = 1 - while self.active: - total_written = 0 # Used to help splitting the file - pattern = FILE_PATTERN.format(name=self.image_name, - partition=number, - volume=volumes) - file_path = self.target_path + pattern - fd = open(file_path, "wb") - - next_partition = False - while self.active: - try: - data = buffer.get() - except IOError, e: - if e.errno == errno.EINTR: - self.cancel() - break - log.info(error_file.read(4096)) - if data == EOF: - next_partition = True - if self.create_iso: - remaining_size -= total_written - if not slices.has_key(iso_volume): - slices[iso_volume] = list() - slices[iso_volume].append(file_path) - break - try: - fd.write(data) - except Exception as e: - log.info("{0}".format(e)) - self.notify_status("disk_full") - try: - shutil.rmtree(self.target_path) - log.info("The folder which cointain the images files were erased") - break - except Exception as e: - log.info(e) - break - self.cancel() - break - - self.processed_blocks += 1 - - if self.split_size: - bytes_written = len(data) - total_written += bytes_written - - if (total_written + bytes_written) >= remaining_size: - volumes += 1 - remaining_size = self.split_size - if self.create_iso: - if not slices.has_key(iso_volume): - slices[iso_volume] = list() - slices[iso_volume].append(file_path) - iso_volume += 1 - break - - fd.close() - if next_partition: break - - self.buffer_manager.join() - part.filesystem.close() - information.add_partition(number, type, volumes, - part.filesystem.get_used_size(), - uuid, label) - - # We dont need to save the data of the swap partition - # we just copy the informations and re-create when - # restoring - swap = disk.get_swap_partition() - if swap is not None: - log.info("Swap path {0}".format(swap.get_path())) - number = swap.get_number() - uuid = swap.filesystem.uuid() - type = swap.filesystem.type - information.add_partition(number, type, 0, 0, uuid) - - information.save() - self.stop() - - if self.create_iso: - log.info("Starting create ISO operation") - iso_creator = IsoCreator(self.target_path, slices, - self.image_name, - self.notify_status, - device.is_disk()) - iso_creator.run() - - if self.canceled: - log.info("Creation canceled") - self.notify_status("canceled", {"operation": - "Create image"}) - else: - self.notify_status("finish") - log.info("Creation finished") - - def stop(self): - if self.active: - if hasattr(self, "buffer_manager"): - self.buffer_manager.stop() - self.active = False - self.timer.stop() - log.info("Create image stopped") - - def cancel(self): - if not self.canceled: - log.info("Create image canceled") - self.canceled = True - self.stop() From ffb180b49368e3697f16baf5ba3e7dde40f68121 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Tue, 31 Dec 2013 10:05:27 -0200 Subject: [PATCH 53/99] Adicao do tamanho dos discos para geracao de isos de tamanhos corretos --- carbono/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carbono/config.py b/carbono/config.py index 80c2f8b..dc71359 100755 --- a/carbono/config.py +++ b/carbono/config.py @@ -20,7 +20,7 @@ FILE_PATTERN = "{name}-{partition}-{volume}.data" BLOCK_SIZE = 1048576 # 1MB -BASE_SYSTEM_SIZE = 0 #TODO (in bytes) +BASE_SYSTEM_SIZE = 1024 * 1024 * 250 #Em bytes. TODO: pegar tamanho real LOG_FILE = "/var/log/mstech/carbono.log" EOF = -1 VERSION = (0, 1, 0, "alpha", 0) From c9b3d19d0f8829ed2499eb529435083403d458e9 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Tue, 31 Dec 2013 11:54:01 -0200 Subject: [PATCH 54/99] =?UTF-8?q?[MIDIA=5FRESTORE]=20Correcao=20na=20numer?= =?UTF-8?q?acao=20dos=20volumes=20na=20restaura=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correcao na exibiçao dos volumes quando for restaurar a partir do CD/DVD. Anteriormente caso colocasse o cd errado o contador ia incrementando o numero de volumes, causando um erro. --- carbono/image_reader/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carbono/image_reader/generic.py b/carbono/image_reader/generic.py index 4fc23e1..f06e464 100644 --- a/carbono/image_reader/generic.py +++ b/carbono/image_reader/generic.py @@ -45,7 +45,7 @@ def open(self): self._fd = open(file_path, 'rb') else: self.image_path = self.notify_callback("file_not_found", - {"path": self.image_path, "file": file_pattern}) + {"path": self.image_path, "file": file_pattern, "current_volume": self.current_volume}) if self.image_path: self.image_path = adjust_path(self.image_path) From 143ee544429654c6f1f050288388f7375dfb6ca9 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Thu, 2 Jan 2014 11:03:59 -0200 Subject: [PATCH 55/99] [MIDIA_RESTORE] Correcao na numeracao dos volumes na restauracao Correcao no Image_reader para apresentar o numero do volume correto para qual o carbono esta requisitado para continuar a extracao da imagem --- carbono/image_reader/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carbono/image_reader/generic.py b/carbono/image_reader/generic.py index f06e464..6944c8c 100644 --- a/carbono/image_reader/generic.py +++ b/carbono/image_reader/generic.py @@ -44,7 +44,7 @@ def open(self): if os.path.exists(file_path): self._fd = open(file_path, 'rb') else: - self.image_path = self.notify_callback("file_not_found", + self.image_path = self.notify_callback("file_not_found_cd", {"path": self.image_path, "file": file_pattern, "current_volume": self.current_volume}) if self.image_path: From c6c8cc1a7e85389a2605b10430de98eca9748183 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Thu, 2 Jan 2014 18:29:50 -0200 Subject: [PATCH 56/99] Chamada da porcentagem do partclone na interface Modificacao no arquivo image_creator.py para a chamada da porcenta- gem do partclone na interface, melhorando a usabilidade do usuario --- carbono/image_creator.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/carbono/image_creator.py b/carbono/image_creator.py index 08c97e4..acc818b 100644 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -62,6 +62,7 @@ def __init__(self, source_device, output_folder, self.canceled = False self.partclone_stderr = None self.partclone_sucess = False + self.data_is_eof = False def notify_percent(self): #refresh the interface percentage @@ -70,10 +71,20 @@ def notify_percent(self): self.current_percent = percent self.notify_status("progress", {"percent": percent}) - #verify stderr from parclone + #verify stderr from partclone if self.partclone_stderr != None: - if self.partclone_stderr.readline().startswith("Partclone successfully cloned the device"): + partclone_status = self.partclone_stderr.readline() + if partclone_status.startswith("Partclone successfully cloned the device"): self.partclone_sucess = True + else: + if self.data_is_eof: + part_list = partclone_status.split() + if part_list[0] == "current": + if len(part_list) >= 13: + try: + self.notify_status("waiting_partclone",{"partclone_percent":partclone_status.split()[13].split(",")[0]}) + except Exception as e: + log.info(e) def create_image(self): """ """ @@ -183,9 +194,12 @@ def create_image(self): if data == EOF: if (self.partclone_stderr != None): while self.partclone_sucess == False: - time.sleep(0.5) - self.notify_status("waiting_partclone") - time.sleep(0.5) + self.data_is_eof = True + + self.partclone_stderr = None + self.partclone_sucess = False + self.data_is_eof = False + next_partition = True if self.create_iso: remaining_size -= total_written From 068ad37caf3e48cca9c50aa2a3a8a40d2d7010ba Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Fri, 3 Jan 2014 10:09:11 -0200 Subject: [PATCH 57/99] Otimizacao da interface partclone Exibicao da porcentagem do partclone na interface do upimage e correcao para a extracao de discos que contenham mais de duas particoes ext --- carbono/image_creator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) mode change 100644 => 100755 carbono/image_creator.py diff --git a/carbono/image_creator.py b/carbono/image_creator.py old mode 100644 new mode 100755 index acc818b..63c38ba --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -151,7 +151,8 @@ def create_image(self): for part in partition_list: if not self.active: break - total_bytes = part.filesystem.get_used_size() + + log.info("Creating image of {0}".format(part.get_path())) number = part.get_number() uuid = part.filesystem.uuid() label = part.filesystem.read_label() @@ -193,8 +194,9 @@ def create_image(self): if data == EOF: if (self.partclone_stderr != None): + self.data_is_eof = True while self.partclone_sucess == False: - self.data_is_eof = True + pass self.partclone_stderr = None self.partclone_sucess = False From fe9706c956ef8f413af5794756635ccd40fc5e3b Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 14 Jan 2014 14:50:14 -0200 Subject: [PATCH 58/99] Adds halinfo *halinfo added to utils, to get info of a list of external devices *run_simple_command_echo added to utils --- carbono/utils.py | 106 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 10 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index d81cc1a..6d4ff99 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -30,6 +30,76 @@ + +class HalInfo(): + ''' + Get info of the disk using HAL + ''' + + def __init__(self): + self.external_devices = [] + + def get_storages_udis(self): + ''' + Get the dbus path of all disk(/dev/sda) + Returns a list + ''' + out,err,ret = run_simple_command_echo('{0} --capability "storage"'.format( + which('hal-find-by-capability')),False) + + return out.split() + + + def get_volumes_udis(self): + ''' + Get the dbus path of all devices (/dev/sda1) + Returns a list + ''' + out,err,ret = run_simple_command_echo('{0} --capability "volume"'.format( + which('hal-find-by-capability')),False) + + + return out.split() + + + def is_storage_removable(self, storage_udi): + '''Verifies if given udi is a removable storage''' + out,err,ret = run_simple_command_echo('{0} --udi {1} \ + --key storage.removable'.format( + which('hal-get-property'),storage_udi), False) + if 'true' in out: + return True + return False + + + def get_block_device(self, udi): + ''' + Get the block of the given udi + Returns /dev/sda or /dev/sda1, for example + ''' + out,err,ret = run_simple_command_echo('{0} --udi {1} \ + --key block.device'.format( + which('hal-get-property'),udi), False + ) + return out + + + def get_external_devices(self): + storages_udis = self.get_storages_udis() + volumes_udis = self.get_volumes_udis() + for s in storages_udis: + if self.is_storage_removable(s): + device = self.get_block_device(s).split()[0] + if "sr" in device: + self.external_devices.append(device) + else: + for v in volumes_udis: + vol = self.get_block_device(v).split()[0] + if device in vol: + self.external_devices.append(vol) + return self.external_devices + + class DiskInfo(): def __init__(self): @@ -301,6 +371,22 @@ def run_simple_command(cmd): p.wait() return p.returncode +def run_simple_command_echo(cmd, echo=False): + ''' + run a given command + returns the output, errors (if any) and returncode + ''' + if echo: + print "{0}".format(cmd) + p = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE) + p.wait() + output, err = p.communicate() + ret = p.returncode + return output,err,ret + def get_disk_size(path): devices = parted.getAllDevices() for device in devices: @@ -367,8 +453,8 @@ def available_memory(percent=100): def get_devices(): disk_dict = {} - devices = parted.getAllDevices() - for device in devices: + devices = parted.getAllDevices() + for device in devices: dev_path = device.path try: disk = parted.Disk(device) @@ -399,7 +485,7 @@ def find_carbono(path): ret = True if filter(lambda x:not x in dev_files, CARBONO_FILES2): ret = False - return ret + return ret def mount_point(device): @@ -417,11 +503,11 @@ def mount_point(device): raise ErrorIdentifyDevice("Erro na identificação do Pendrive") def get_upimage_device(): - devices = get_devices() + devices = get_devices() for dev in devices: device = devices[dev]["partitions"].keys() if is_mounted(device[0]): - mount_path = mount_point(device[0]) + mount_path = mount_point(device[0]) else: mount_path = mount_pen(device[0]) ret = find_carbono(mount_path) @@ -478,15 +564,15 @@ def is_mounted(device): return False def check_if_root(): - if os.getuid() == 0: + if os.getudi() == 0: return True return False - + def verify_4k(hd = "sda"): ''' Retorna o tamanho fisico do setor ''' - try: + try: f = open("/sys/block/{0}/queue/physical_block_size".format(hd)) block = f.readline() if "4096" in block: @@ -495,5 +581,5 @@ def verify_4k(hd = "sda"): return(512) except Exception as e: #nao tem HD (uma vm sem hd, por exemplo) - return(512) - + return(512) + From ad601e37f8803ca5c4e22e5e31b55d8e3ea065b8 Mon Sep 17 00:00:00 2001 From: Bruno 'dropped' Criado Date: Wed, 15 Jan 2014 08:55:10 -0200 Subject: [PATCH 59/99] Fixes os.getuid() --- carbono/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carbono/utils.py b/carbono/utils.py index 6d4ff99..114c112 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -564,7 +564,7 @@ def is_mounted(device): return False def check_if_root(): - if os.getudi() == 0: + if os.getuid() == 0: return True return False From 9deab2a00daeb2ca499e4e451c54935ad56d5c0a Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Fri, 31 Jan 2014 17:05:44 -0200 Subject: [PATCH 60/99] Change Hal *When returns external devices, returns all fat32 devices too - need to get exeternal hard disk --- carbono/utils.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/carbono/utils.py b/carbono/utils.py index 114c112..b1c3309 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -71,6 +71,14 @@ def is_storage_removable(self, storage_udi): return True return False + def is_device_fat(self, storage_udi): + '''Verifies if given udi ''' + out,err,ret = run_simple_command_echo('{0} --udi {1} \ + --key volume.fstype'.format( + which('hal-get-property'),storage_udi), False) + if 'fat' in out: + return True + return False def get_block_device(self, udi): ''' @@ -87,6 +95,10 @@ def get_block_device(self, udi): def get_external_devices(self): storages_udis = self.get_storages_udis() volumes_udis = self.get_volumes_udis() + for s in volumes_udis: + if self.is_device_fat(s): + dev = self.get_block_device(s).split()[0] + self.external_devices.append(dev) for s in storages_udis: if self.is_storage_removable(s): device = self.get_block_device(s).split()[0] @@ -95,7 +107,7 @@ def get_external_devices(self): else: for v in volumes_udis: vol = self.get_block_device(v).split()[0] - if device in vol: + if ( device in vol ) and ( vol not in self.external_devices ): self.external_devices.append(vol) return self.external_devices From da20c76384c7fa0cf2f3792b78eb9fbf0465fbd8 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Fri, 14 Feb 2014 10:49:15 -0200 Subject: [PATCH 61/99] Alteracao no carbono para parar a extracao ou aplicacao se ocorre erro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alteração no carbono de maneira que se ocorrer um erro de extracao ou aplicacao da imagem, se tenha uma correta notificação a interface. --- carbono/exception.py | 11 +++++++ carbono/image_creator.py | 65 ++++++++++++++++++++++++++++++--------- carbono/image_restorer.py | 18 ++++++++++- 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/carbono/exception.py b/carbono/exception.py index be7653f..ed48836 100644 --- a/carbono/exception.py +++ b/carbono/exception.py @@ -66,3 +66,14 @@ class ExpandingPartitionError(Exception): class DeviceIsMounted(Exception): pass +class InvalidFolder(Exception): + pass + +class ErrorOpenToWrite(Exception): + pass + +class ErrorCloseToWrite(Exception): + pass + +class ErrorFileNotFound(Exception): + pass diff --git a/carbono/image_creator.py b/carbono/image_creator.py index 63c38ba..d9a44cf 100755 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -40,10 +40,6 @@ def __init__(self, source_device, output_folder, raw=False, split_size=0, create_iso=False, fill_with_zeros=False): - assert check_if_root(), "You need to run this application as root" - assert os.path.isdir(output_folder), "{0} folder is invalid".\ - format(output_folder) - self.image_name = image_name self.device_path = source_device self.target_path = adjust_path(output_folder) @@ -64,6 +60,17 @@ def __init__(self, source_device, output_folder, self.partclone_sucess = False self.data_is_eof = False + if not check_if_root(): + log.info("You need to run this application as root") + self.notify_status("not_root", \ + {"not_root":"You dont't have permission"}) + + if not os.path.isdir(output_folder): + log.info("The folder is invalid") + self.notify_status("invalid_folder",\ + {"invalid_folder":output_folder}) + raise InvalidFolder("Invalid folder {0}".format(output_folder)) + def notify_percent(self): #refresh the interface percentage percent = (self.processed_blocks/float(self.total_blocks)) * 100 @@ -85,6 +92,7 @@ def notify_percent(self): self.notify_status("waiting_partclone",{"partclone_percent":partclone_status.split()[13].split(",")[0]}) except Exception as e: log.info(e) + raise ErrorCreatingImage("Create image wasn't made with success") def create_image(self): """ """ @@ -98,10 +106,16 @@ def create_image(self): disk = Disk(device) if device.is_disk(): - mbr = Mbr(self.target_path) - mbr.save_to_file(self.device_path) - dlm = DiskLayoutManager(self.target_path) - dlm.save_to_file(disk) + try: + mbr = Mbr(self.target_path) + mbr.save_to_file(self.device_path) + dlm = DiskLayoutManager(self.target_path) + dlm.save_to_file(disk) + except Exception as e: + log.info(e) + self.notify_status("write_error", \ + {"write_error":self.target_path}) + raise ErrorWritingToDevice("Write error in {0}".format(self.target_path)) partition_list = disk.get_valid_partitions(self.raw) if not partition_list: @@ -153,6 +167,8 @@ def create_image(self): if not self.active: break log.info("Creating image of {0}".format(part.get_path())) + self.notify_status("image_creator", \ + {"image_creator ": part.get_path()}) number = part.get_number() uuid = part.filesystem.uuid() label = part.filesystem.read_label() @@ -181,13 +197,21 @@ def create_image(self): partition=number, volume=volumes) file_path = self.target_path + pattern - fd = open(file_path, "wb") - + try: + fd = open(file_path, "wb") + except Exception as e: + log.info(e) + self.notify_status("open_file", \ + {"open_file":file_path}) + raise ImageNotFound("The file wasn't found {0}". \ + format(file_path)) next_partition = False while self.active: try: data = buffer.get() except IOError, e: + self.notify_status("read_buffer_error", \ + {"read_buffer_error":"Erro buffer"}) if e.errno == errno.EINTR: self.cancel() break @@ -221,8 +245,8 @@ def create_image(self): except Exception as e: log.info(e) break - self.cancel() - break + raise ErrorWritingToDevice("Error in write file {0}".\ + format(file_path)) self.processed_blocks += 1 @@ -239,8 +263,13 @@ def create_image(self): slices[iso_volume].append(file_path) iso_volume += 1 break + try: + fd.close() + except Exception as e: + log.info(e) + self.notify_status("write_error",{"write_error":e}) + raise ErrorCloseToWrite("Close Error {0}".format(e)) - fd.close() if next_partition: break self.buffer_manager.join() @@ -259,12 +288,20 @@ def create_image(self): uuid = swap.filesystem.uuid() type = swap.filesystem.type information.add_partition(number, type, 0, 0, uuid) + try: + information.save() + except Exception as e: + log.info(e) + self.notify_status("write_error",{"write_error":e}) + raise ErrorWritingToDevice("Write Error {0}".format(e)) - information.save() self.stop() if self.create_iso: log.info("Starting create ISO operation") + self.notify_status("create_iso", \ + {"create_iso":self.create_iso}) + iso_creator = IsoCreator(self.target_path, slices, self.image_name, self.notify_status, diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index de8b5d1..dda6d43 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -55,6 +55,12 @@ def __init__(self, image_folder, target_device, self.active = False self.canceled = False + if not os.path.isdir(self.image_path): + log.info("The folder is invalid") + self.notify_status("invalid_folder", \ + {"invalid_folder":self.image_path}) + raise InvalidFolder("Invalid folder {0}".format(output_folder)) + def notify_percent(self): # Total blocks can be 0 when restoring only a swap partition # for example @@ -93,6 +99,8 @@ def restore_image(self): if device.is_disk() != \ information.get_image_is_disk(): log.error("Invalid target device %s" % device.path) + self.notify_status("write_error", \ + {"write_error":device_path}) raise ErrorRestoringImage("Invalid target device") try: @@ -126,6 +134,8 @@ def restore_image(self): log.error("Error to restore the Mbr file") image_path = self.image_path.split("/")[3] + "/mbr.bin" self.notify_status("file_not_found",{"file_not_found":image_path}) + raise ErrorFileNotFound("File not Found {0}".format(image_path)) + dlm = DiskLayoutManager(self.image_path) try: if self.expand != 2: @@ -135,7 +145,9 @@ def restore_image(self): except Exception as e: log.error("Error to restore the disk.dl file") image_path = self.image_path.split("/")[3] + "/disk.dl" - self.notify_status("file_not_found",{"file_not_found":image_path}) + self.notify_status("file_not_found",{"file_not_found":image_path}) + raise ErrorFileNotFound("File not found {0}".format(image_path)) + else: parent_path = get_parent_path(self.target_device) parent_device = Device(parent_path) @@ -168,8 +180,12 @@ def restore_image(self): part.type) log.info("Restoring partition {0}".format(partition.get_path())) + self.notify_status("restore_partition",\ + {"restoring_partition":partition.get_path()}) if partition is None: + self.notify_status("no_valid_partitions", \ + {"no_valid_partitions":partitions.get_path()}) raise ErrorRestoringImage("No valid partitions found") if hasattr(part, "uuid"): From 6922197864b230369cb786dc3a879be3ffc30d4a Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 17 Feb 2014 15:48:47 -0300 Subject: [PATCH 62/99] Fix bug #394 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit *Bug: Depois de extrair uma imagem inteira com ISO e outra apenas parte da imagem (linux) com ISO, ao tentar restaurar, a aplicação não bootou corretamente. *Problem: a file need by default isolinux.cfg was not copied when the cd was being built *Resolution: Now it just copy initram.gz and vmlinuz from the upimage cd isolinux.cfg is generated to boot this files --- carbono/iso_creator.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/carbono/iso_creator.py b/carbono/iso_creator.py index 273f72c..f8f35ac 100644 --- a/carbono/iso_creator.py +++ b/carbono/iso_creator.py @@ -22,7 +22,8 @@ from carbono.exception import * from carbono.config import * -CARBONO_FILES = ("initram.gz", "vmlinuz", "isolinux.cfg") +CARBONO_FILES = ("initram.gz", "vmlinuz") + class IsoCreator: def __init__(self, target_path, slices, @@ -38,6 +39,20 @@ def __init__(self, target_path, slices, self.process = None self.mount_point = None + def __generate_isolinux_file(self): + isolinux_tmp_file = "/tmp/isolinux.cfg" + tmp_file = open(isolinux_tmp_file,"w") + tmp_file.write("prompt 0\n"+ + " default upimage\n"+ + "label upimage\n"+ + " kernel vmlinuz\n"+ + " append initrd=initram.gz rdinit=/sbin/init "+ + "ramdisk_size=512000\n") + tmp_file.close() + + return isolinux_tmp_file + + def notify_percent(self): if self.process is not None: if self.process.stderr is not None: @@ -61,7 +76,7 @@ def mount_device(self, device): format(device, tmpd)) if ret is not 0: raise ErrorMountingFilesystem - return tmpd + return tmpd def find_carbono_files(self, path): dev_files = os.listdir(path) @@ -108,10 +123,14 @@ def run(self): break - # Add carbono files + # Add carbono files map(lambda x: self.slices[volume].\ append(self.mount_point + x), CARBONO_FILES) + # Add isolinux.cfg file + isolinux_file = self.__generate_isolinux_file() + self.slices[volume].append(isolinux_file) + # Bootloader self.slices[volume].append("/usr/lib/syslinux/isolinux.bin") self.slices[volume].append("{0}image.info".format( @@ -122,7 +141,7 @@ def run(self): map(lambda x: self.slices[volume].\ append(self.target_path + x), ("mbr.bin", "disk.dl")) - + if first_volume: extra_params = "-joliet-long -b " + \ "isolinux.bin -c " + \ From 2f7f88620c60cfa5e873b3c295789283eb24ff43 Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Tue, 18 Feb 2014 15:50:42 -0300 Subject: [PATCH 63/99] =?UTF-8?q?Modifica=C3=A7=C3=A3o=20do=20aleta=20do?= =?UTF-8?q?=20carbono=20para=20o=20tratamento=20de=20erro=20de=20escrita?= =?UTF-8?q?=20de=20disco?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adicao da chamada do alerta caso ocorra erro de escrita no Carbono, O erro ocorria quando se aplicava uma imagem do partclone em uma partição maior que a imagem e menor do que a partição original que se extraiu a imagem. --- carbono/image_restorer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index dda6d43..ec4efb3 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -239,6 +239,7 @@ def restore_image(self): try: partition.filesystem.write_block(data) except ErrorWritingToDevice, e: + self.notify_status("write_partition_error") if not self.canceled: self.stop() raise e From 408ffa0025c59d41355cbbf471a5acc4646a274c Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Mon, 24 Feb 2014 10:07:33 -0300 Subject: [PATCH 64/99] [QoS #397]Alteracao na chamada do alerta da numeracao dos CDs Alteracao no carbono para a chamada do alerta de proximo disco e tratamento do alerta de cancelar no carbono. --- carbono/image_reader/generic.py | 4 ++-- carbono/image_restorer.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/carbono/image_reader/generic.py b/carbono/image_reader/generic.py index 6944c8c..2f6e817 100644 --- a/carbono/image_reader/generic.py +++ b/carbono/image_reader/generic.py @@ -51,8 +51,8 @@ def open(self): self.image_path = adjust_path(self.image_path) continue else: - self.notify_callback("canceled", - {"operation": "Restoring image"}) + raise Exception("canceled") + break break def close(self): diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index ec4efb3..7d7eda4 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -205,10 +205,12 @@ def restore_image(self): volumes = 1 if hasattr(part, "volumes"): volumes = part.volumes - - image_reader = ImageReaderFactory(self.image_path, pattern, - volumes, compressor_level, - self.notify_status) + try: + image_reader = ImageReaderFactory(self.image_path, pattern, + volumes, compressor_level, + self.notify_status) + except Exception as e: + log.info(e) extract_callback = None if compressor_level: From 2f3531168fe689067d0417316335dfebfc245def Mon Sep 17 00:00:00 2001 From: leandro takahashi Date: Mon, 24 Mar 2014 10:16:01 -0300 Subject: [PATCH 65/99] =?UTF-8?q?[Defeito=20#837]=20Aplica=C3=A7=C3=A3o=20?= =?UTF-8?q?n=C3=A3o=20exibe=20mensagem=20de=20erro=20quando=20HD=20contend?= =?UTF-8?q?o=20imagem=20de=20origem=20=C3=A9=20desconectado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolucao do erro de alerta quando o HD que contem a imagem de origem é desconectado. Modificou-se o codigo, adicionando mais um parâmetro a classe buffer manager, que foi a "status_notify" o qual responde a interface caso ocorra algum erro no buffer. --- carbono/buffer_manager/__init__.py | 10 +++++++--- carbono/buffer_manager/simple_manager.py | 10 +++++++--- carbono/buffer_manager/work_manager.py | 10 ++++++++-- carbono/image_creator.py | 10 ++++++++-- carbono/image_restorer.py | 4 ++++ 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/carbono/buffer_manager/__init__.py b/carbono/buffer_manager/__init__.py index 6c0e66f..cfee403 100644 --- a/carbono/buffer_manager/__init__.py +++ b/carbono/buffer_manager/__init__.py @@ -23,12 +23,16 @@ class BufferManagerFactory: - def __init__(self, read_callback, job_callback=None): + def __init__(self, read_callback,notify_status, job_callback=None): if job_callback: if available_processors() <= 2:# and is_hyperthreading(): - self._manager = SimpleManager(read_callback, job_callback) + self._manager = SimpleManager(read_callback, + job_callback, + notify_status) else: - self._manager = WorkManager(read_callback, job_callback) + self._manager = WorkManager(read_callback, + job_callback, + notify_status) else: self._manager = DummyManager(read_callback) diff --git a/carbono/buffer_manager/simple_manager.py b/carbono/buffer_manager/simple_manager.py index b126bbe..2c450e2 100644 --- a/carbono/buffer_manager/simple_manager.py +++ b/carbono/buffer_manager/simple_manager.py @@ -19,10 +19,14 @@ class SimpleManager(DummyManager): - def __init__(self, read_callback, job_callback): + def __init__(self, read_callback, job_callback, notify_status): DummyManager.__init__(self, read_callback) self.job = job_callback + self.notify_status = notify_status def put(self, data): - worked_data = self.job(data) - self.output_buffer.put(worked_data) + try: + worked_data = self.job(data) + self.output_buffer.put(worked_data) + except Exception as e: + self.notify_status("read_buffer_error",{"read_buffer_error":str(e)}) diff --git a/carbono/buffer_manager/work_manager.py b/carbono/buffer_manager/work_manager.py index 2242187..6bbe96a 100644 --- a/carbono/buffer_manager/work_manager.py +++ b/carbono/buffer_manager/work_manager.py @@ -28,11 +28,12 @@ from carbono.exception import * class Worker(Process): - def __init__(self, buffer, reorder_buffer, job): + def __init__(self, buffer, reorder_buffer, job, status_callback): Process.__init__(self) self.buffer = buffer self.reorder_buffer = reorder_buffer self.job = job + self.status_callback = status_callback self.event = Event() def run(self): @@ -47,7 +48,12 @@ def run(self): if data == EOF: self.stop() break - worked_data = self.job(data) + + try: + worked_data = self.job(data) + except Exception as e: + self.notify_status("read_buffer_error",{"read_buffer_error":str(e)}) + raise ErrorReadingToDevice(e) while self.event.is_set(): try: diff --git a/carbono/image_creator.py b/carbono/image_creator.py index d9a44cf..41d447e 100755 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -186,6 +186,7 @@ def create_image(self): self.buffer_manager = BufferManagerFactory( part.filesystem.read_block, + self.notify_status, compact_callback) self.buffer_manager.start() @@ -210,11 +211,16 @@ def create_image(self): try: data = buffer.get() except IOError, e: - self.notify_status("read_buffer_error", \ - {"read_buffer_error":"Erro buffer"}) + #self.notify_status("read_buffer_error", \ + # {"read_buffer_error":str(e)}) if e.errno == errno.EINTR: + self.notify_status("read_buffer_error", \ + {"read_buffer_error":str(e)}) + data = "" self.cancel() + raise ErrorReadingFromDevice(e) break + if data == EOF: if (self.partclone_stderr != None): diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 7d7eda4..f9811c9 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -218,6 +218,7 @@ def restore_image(self): extract_callback = compressor.extract self.buffer_manager = BufferManagerFactory(image_reader.read_block, + self.notify_status, extract_callback) # open the file after instantiating BufferManager, cause of a @@ -232,7 +233,10 @@ def restore_image(self): data = buffer.get() except IOError, e: if e.errno == errno.EINTR: + self.notify_status("read_buffer_error",{"read_buffer_error":str(e)}) + data = "" self.cancel() + raise ErrorReadingFromDevice(e) break if data == EOF: From 579f35df25e640d4aa196272dce09b83cf126b25 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 14 Apr 2014 09:44:42 -0300 Subject: [PATCH 66/99] Muda modo de restauracao da part. de dados *Adiciona modo aonde a particao de dados nao eh aplicada --- carbono/image_restorer.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index 7d7eda4..135d672 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -39,6 +39,10 @@ class ImageRestorer: def __init__(self, image_folder, target_device, status_callback, partitions=None, expand=2): + # expand -> 0 Expande a ultima particao + # -> 1 Formata a ultima particao + # -> 2 Nenhuma operacao (ele aplica a imagem e nao faz mais nada) + # -> 3 Ele nao aplica a ultima particao assert check_if_root(), "You need to run this application as root" @@ -147,7 +151,7 @@ def restore_image(self): image_path = self.image_path.split("/")[3] + "/disk.dl" self.notify_status("file_not_found",{"file_not_found":image_path}) raise ErrorFileNotFound("File not found {0}".format(image_path)) - + else: parent_path = get_parent_path(self.target_device) parent_device = Device(parent_path) @@ -165,9 +169,15 @@ def restore_image(self): raise ErrorRestoringImage("No enought space on partition") self.timer.start() + + total_partitions = len(partitions) for part in partitions: + total_partitions -= 1 + if not self.active: break - + + if (self.expand == 3) and (total_partitions == 0): break + if information.get_image_is_disk(): partition = disk.get_partition_by_number(part.number, part.type) @@ -258,7 +268,7 @@ def restore_image(self): self.expand_last_partition(self.expand) if self.canceled: - self.notify_status("canceled", {"operation": + self.notify_status("canceled", {"operation": "Restore image"}) else: self._finish() @@ -275,7 +285,7 @@ def restore_image(self): def expand_last_partition(self,opt_expand): # After all data is copied to the disk # we instance class again to reload - + sync() device = Device(self.target_device) disk = Disk(device) From bcf2b90d8d40ce692b6e199c7cf564b35a13d073 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 28 Apr 2014 16:37:08 -0300 Subject: [PATCH 67/99] Change how grubinstall is opened *Change to opening of grubinstall (after the restorating) from a subprocesses to a os.system call with "&" background argument. This way the main interface does no stop working. --- carbono/image_restorer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index a58b273..be2458b 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -17,6 +17,8 @@ import math import _ped +import os + from parted import PARTITION_NORMAL from carbono.device import Device @@ -280,10 +282,8 @@ def restore_image(self): log.info("Iniciando gtk grubinstall") cmd = "{0}".format(which("grubinstall")) try: - self.process = RunCmd(cmd) - self.process.run() - self.process.wait() - except Exception as e: + os.system("{0} &".format(cmd)) + except: log.error("Erro ao iniciar grubinstall. {0}".format(e)) def expand_last_partition(self,opt_expand): From 4b1f1ce591861d3d10a859071c582f12f06a66de Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Thu, 15 May 2014 10:04:26 -0300 Subject: [PATCH 68/99] Change buffer_manager and version *Only use one processor --- carbono/buffer_manager/__init__.py | 16 ++++++++-------- setup.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/carbono/buffer_manager/__init__.py b/carbono/buffer_manager/__init__.py index cfee403..0f42d73 100644 --- a/carbono/buffer_manager/__init__.py +++ b/carbono/buffer_manager/__init__.py @@ -25,14 +25,14 @@ class BufferManagerFactory: def __init__(self, read_callback,notify_status, job_callback=None): if job_callback: - if available_processors() <= 2:# and is_hyperthreading(): - self._manager = SimpleManager(read_callback, - job_callback, - notify_status) - else: - self._manager = WorkManager(read_callback, - job_callback, - notify_status) + #if available_processors() <= 2:# and is_hyperthreading(): + self._manager = SimpleManager(read_callback, + job_callback, + notify_status) + #else: + # self._manager = WorkManager(read_callback, + # job_callback, + # notify_status) else: self._manager = DummyManager(read_callback) diff --git a/setup.py b/setup.py index f02468c..f22a582 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() -VERSION_ID = "2.1.0" +VERSION_ID = "2.2.0" BUILD_ID = "{0}-SNAPSHOT".format(VERSION_ID) if 'BUILD_NUMBER' in os.environ: BUILD_ID = "{0}.{1}".format(VERSION_ID, os.environ['BUILD_NUMBER']) @@ -23,4 +23,4 @@ def read(fname): "carbono.filesystem", "carbono.ui", "carbono.image_reader"], scripts = ["scripts/carbono"], -) +) From 62cf0df6257448b5a12d5319a590d235136f2def Mon Sep 17 00:00:00 2001 From: root Date: Fri, 30 May 2014 14:02:07 -0300 Subject: [PATCH 69/99] [Adicao do modulo detach] Adicao no modulo do disco barramento Adicao do modulo para verificar se o barramento dos dispositivos sao moveis(USB) ou nao moveis(HDs). Foi Adicionado um parametro de retorno, que descrever o tipo dos barramentos, pelo dicionario "detach" Retorno: {"result": {"/dev/sdb": {"model": "SanDisk Cruzer Blade", "partitions": [{"path": "/dev/sdb1", "type": "fat32", "size": 8004288512}], "detach": 1, "size": 8004304896.0}, "/dev/sda": {"model": "ATA ST3500413AS", "partitions": [{"path": "/dev/sda1", "type": "ext4", "size": 9999220736}, {"path": "/dev/sda5", "type": "linux-swap(v1)", "size": 8514437120}, {"path": "/dev/sda6", "type": "ext4", "size": 481589985280}], "detach": 0, "size": 500107862016.0}}, "error": null} --- carbono/utils.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index b1c3309..69afd76 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -22,15 +22,13 @@ import errno import os import parted +import dbus from threading import Thread, Event from os.path import realpath from carbono.exception import * from carbono.config import * - - - class HalInfo(): ''' Get info of the disk using HAL @@ -171,8 +169,23 @@ def __collect_information_about_devices(self): "size": _dict[disk]["partitions"][part]["size"]} self.__DISK_DICT[disk] = {"model": _dict[disk]["model"], "size": _dict[disk]["size"], - "partitions": partitions} - + "partitions": partitions, + "detach": self.device_detachable(disk)} + + def device_detachable(self, device): + bus = dbus.SystemBus() + ud_manager_obj = bus.get_object('org.freedesktop.UDisks', + '/org/freedesktop/UDisks') + proplist = [] + ud_manager = dbus.Interface(ud_manager_obj, 'org.freedesktop.UDisks') + for device_pc in ud_manager.EnumerateDevices(): + device_obj = bus.get_object('org.freedesktop.UDisks', device_pc) + device_props = dbus.Interface(device_obj, dbus.PROPERTIES_IFACE) + proplist.append(device_props.GetAll('org.freedesktop.UDisks.Device')) + + for device_props in proplist: + if device_props["DeviceFile"] == device: + return device_props['DriveCanDetach'] def formated_partitions(self): formated_partitions = [] @@ -206,7 +219,6 @@ def formated_partitions(self): return(formated_partitions_dict) - class DiskPartition(): From 27e6dbfbe944cd2333e9af19ae29687fbcb73e0f Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Fri, 30 May 2014 16:20:49 -0300 Subject: [PATCH 70/99] Hotfix part_expansion * Fix iorerro on exanding a big partition --- carbono/partition_expander.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/carbono/partition_expander.py b/carbono/partition_expander.py index 5af0449..70c417f 100644 --- a/carbono/partition_expander.py +++ b/carbono/partition_expander.py @@ -20,7 +20,7 @@ def try_expand(self): """ """ device = parted.Device(self.path) disk = parted.Disk(device) - + try: last_partition = disk.partitions[-1] except IndexError: @@ -74,8 +74,12 @@ def try_expand(self): try: disk.commit() except: - disk.commitToDevice() - disk.commitToOS() + try: + disk.commitToDevice() + disk.commitToOS() + except Exception as e: + log.error("PartitionExpander : {0}".format(e)) + # Após criar a tabela de partição temos que fazer # com o kernel releia essa tabela. Será preciso @@ -100,7 +104,7 @@ def try_expand(self): # temos que esperar o dispositivo ficar pronto attempt = 0 while True: - + if os.path.exists(new_partition.path): break @@ -113,5 +117,5 @@ def try_expand(self): # Agora da um resize no sistema de arquivos # Pegar o tamanho total do dispositivo da partição a ser redimensionada size = new_partition.geometry.length * device.sectorSize - + return size From 91a2e03978534236223f75be37301980afaba24f Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Fri, 4 Jul 2014 16:08:11 -0300 Subject: [PATCH 71/99] Added info to the disk information *Add total, used and free space to partition_info --- carbono/utils.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index b1c3309..aa3c4f9 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -28,14 +28,11 @@ from carbono.exception import * from carbono.config import * - - - class HalInfo(): ''' Get info of the disk using HAL ''' - + def __init__(self): self.external_devices = [] @@ -49,7 +46,6 @@ def get_storages_udis(self): return out.split() - def get_volumes_udis(self): ''' Get the dbus path of all devices (/dev/sda1) @@ -58,10 +54,8 @@ def get_volumes_udis(self): out,err,ret = run_simple_command_echo('{0} --capability "volume"'.format( which('hal-find-by-capability')),False) - return out.split() - def is_storage_removable(self, storage_udi): '''Verifies if given udi is a removable storage''' out,err,ret = run_simple_command_echo('{0} --udi {1} \ @@ -91,7 +85,6 @@ def get_block_device(self, udi): ) return out - def get_external_devices(self): storages_udis = self.get_storages_udis() volumes_udis = self.get_volumes_udis() @@ -172,6 +165,14 @@ def __collect_information_about_devices(self): self.__DISK_DICT[disk] = {"model": _dict[disk]["model"], "size": _dict[disk]["size"], "partitions": partitions} + def disk_usage(self, path): + st = os.statvfs(path) + disk_usage = {} + disk_usage["free"] = st.f_bavail * st.f_frsize + disk_usage["total"] = st.f_blocks * st.f_frsize + disk_usage["used"] = (st.f_blocks - st.f_bfree) * st.f_frsize + + return disk_usage def formated_partitions(self): @@ -181,10 +182,18 @@ def formated_partitions(self): device_info = {"size":None,"label":None,"partitions":None} for part in self.__PARTITION_DICT.keys(): + part_dict = {} + disk_part = DiskPartition(part) + tmp_folder_part = disk_part.mount_partition() + if tmp_folder_part: + part_usage = self.disk_usage(tmp_folder_part) + part_dict["total_size"] = part_usage["total"] + part_dict["used_size"] = part_usage["used"] + part_dict["free_size"] = part_usage["free"] + part_type = self.__PARTITION_DICT[part]['type'] size_bytes = self.__PARTITION_DICT[part]['size'] size_mb = int(long(size_bytes)/(1024*1024.0)) - part_dict = {} part_dict["type"] = part_type part_dict["path"] = part part_dict["size"] = size_mb @@ -328,6 +337,7 @@ def get_mounted_devices(self, device = None): list_dest = [] return disk_mounted + class Timer(Thread): def __init__(self, callback, timeout=2): Thread.__init__(self) @@ -384,10 +394,10 @@ def run_simple_command(cmd): return p.returncode def run_simple_command_echo(cmd, echo=False): - ''' + ''' run a given command returns the output, errors (if any) and returncode - ''' + ''' if echo: print "{0}".format(cmd) p = subprocess.Popen(cmd, shell=True, From 064a69b32010a413157478027bf5da2d699e0569 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 7 Jul 2014 16:31:48 -0300 Subject: [PATCH 72/99] Remove error msg from expand_last_partition_error --- carbono/image_restorer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index be2458b..874cec2 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -310,9 +310,9 @@ def expand_last_partition(self,opt_expand): if returncode == True: log.info("Resize in {0} was made with sucess".format(partition.get_path())) else: - log.info("Resize in {0} failed".format(partition.get_path())) - self.notify_status("expand_last_partition_error", {"last_partition":partition.get_path()}) - self.canceled = True + log.info("Resize in {0} failed - Versao sem mensagem!".format(partition.get_path())) + #self.notify_status("expand_last_partition_error", {"last_partition":partition.get_path()}) + #self.canceled = True else: if opt_expand == 1: log.info("Formating {0} filesystem".format(partition.get_path())) From 94c151d95fb9fed7ac6fc074c5197150c2707d2d Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 8 Jul 2014 15:49:14 -0300 Subject: [PATCH 73/99] Add read-only mount option * Adiciona opcao para montagem read-only na montagem de particao em pasta temporaria --- carbono/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index aa3c4f9..68be0cf 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -184,7 +184,7 @@ def formated_partitions(self): for part in self.__PARTITION_DICT.keys(): part_dict = {} disk_part = DiskPartition(part) - tmp_folder_part = disk_part.mount_partition() + tmp_folder_part = disk_part.mount_partition(ro = True) if tmp_folder_part: part_usage = self.disk_usage(tmp_folder_part) part_dict["total_size"] = part_usage["total"] @@ -278,7 +278,10 @@ def umount_all_partitions(self): return result_disk_umounted - def mount_partition(self,destino = None): + def mount_partition(self, destino = None, ro = False): + mount_options = "" + if not ro: + mount_options += "-o ro" mounted_folder = self.get_mount_point(self.__partition) @@ -289,7 +292,7 @@ def mount_partition(self,destino = None): if destino is None: mounted_folder = adjust_path(tempfile.mkdtemp()) - cmd = "mount {0} {1}".format(self.__partition, mounted_folder) + cmd = "mount {0} {1} {2}".format(mount_options, self.__partition, mounted_folder) p = subprocess.Popen(cmd,stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) ret = os.waitpid(p.pid,0)[1] if not ret: @@ -298,7 +301,7 @@ def mount_partition(self,destino = None): return False else: - cmd = "mount {0} {1}".format(self.__partition, destino) + cmd = "mount {0} {1} {2}".format(mount_options, self.__partition, destino) p = subprocess.Popen(cmd,stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) ret = os.waitpid(p.pid,0)[1] if not ret: From 78d42993c3e9addc1f428204c9200be2df3b785e Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Fri, 11 Jul 2014 14:05:16 -0300 Subject: [PATCH 74/99] Fix device_info dict return --- carbono/utils.py | 57 +++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index 68be0cf..0fd0824 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -119,6 +119,7 @@ def __get_devices(self): ''' Filter = only partitions with a valid filesystem ''' disk_dict = {} + devices = parted.getAllDevices() for device in devices: dev_path = device.path @@ -164,7 +165,8 @@ def __collect_information_about_devices(self): "size": _dict[disk]["partitions"][part]["size"]} self.__DISK_DICT[disk] = {"model": _dict[disk]["model"], "size": _dict[disk]["size"], - "partitions": partitions} + "partitions" : [] + } def disk_usage(self, path): st = os.statvfs(path) disk_usage = {} @@ -174,8 +176,7 @@ def disk_usage(self, path): return disk_usage - - def formated_partitions(self): + def formated_partitions(self, filter_disk=None): formated_partitions = [] formated_partitions_dict = self.__DISK_DICT self.__collect_information_about_devices() @@ -184,47 +185,53 @@ def formated_partitions(self): for part in self.__PARTITION_DICT.keys(): part_dict = {} disk_part = DiskPartition(part) - tmp_folder_part = disk_part.mount_partition(ro = True) + tmp_folder_part = disk_part.mount_partition(ro=True) + part_dict["total_size"] = None + part_dict["used_size"] = None + part_dict["free_size"] = None + if tmp_folder_part: - part_usage = self.disk_usage(tmp_folder_part) - part_dict["total_size"] = part_usage["total"] - part_dict["used_size"] = part_usage["used"] - part_dict["free_size"] = part_usage["free"] + part_usage = self.disk_usage(tmp_folder_part) + part_dict["total_size"] = part_usage["total"] + part_dict["used_size"] = part_usage["used"] + part_dict["free_size"] = part_usage["free"] part_type = self.__PARTITION_DICT[part]['type'] size_bytes = self.__PARTITION_DICT[part]['size'] - size_mb = int(long(size_bytes)/(1024*1024.0)) + size_mb = int(long(size_bytes) / (1024 * 1024.0)) part_dict["type"] = part_type part_dict["path"] = part part_dict["size"] = size_mb formated_partitions.append(part_dict) formated_partitions.sort(reverse=False) - temp_parts = [] disk = formated_partitions[0]['path'][:8] - for aux in range(0,len(formated_partitions)): + formated_partitions_dict[disk]["partitions"] = [] + for aux in range(0, len(formated_partitions)): temp_disk = formated_partitions[aux]['path'][:8] - if temp_disk == disk: - temp_parts.append(formated_partitions[aux]) - else: - formated_partitions_dict[disk]["partitions"] = temp_parts - temp_parts = [] - temp_parts.append(formated_partitions[aux]) + if formated_partitions_dict[temp_disk]["partitions"] is None: + formated_partitions_dict[temp_disk]["partitions"] = [] + if temp_disk != disk: disk = temp_disk - formated_partitions_dict[disk]["partitions"] = temp_parts + formated_partitions_dict[temp_disk]["partitions"].append( + formated_partitions[aux]) + if filter_disk is not None: + formated_partitions_dict = dict((key,value) + for key, value + in formated_partitions_dict.iteritems() + if key == filter_disk) return(formated_partitions_dict) - class DiskPartition(): - def __init__(self, partition = ""): + def __init__(self, partition=""): self.__temp_folder = "" self.__partition = partition - def __generate_temp_folder(self, destino = "/tmp/"): + def __generate_temp_folder(self, destino="/tmp/"): self.__temp_folder = adjust_path(tempfile.mkdtemp()) @@ -238,7 +245,7 @@ def get_partition(self): return self.__partition - def umount_partition(self, device = None): + def umount_partition(self, device=None): disk_mounted = self.get_mounted_devices() disk_list = [] result_disk_umounted = {} @@ -477,7 +484,7 @@ def available_memory(percent=100): return free def get_devices(): - disk_dict = {} + disk_dic = {} devices = parted.getAllDevices() for device in devices: dev_path = device.path @@ -493,8 +500,8 @@ def get_devices(): part_type = p.fileSystem.type if part_type == "fat32" or part_type == "fat16": part_dict[part_path] = {"type":part_type} - disk_dict[dev_path] = {"partitions": part_dict} - return disk_dict + disk_dic[dev_path] = {"partitions": part_dict} + return disk_dic CARBONO_FILES2 = ("initram.gz","vmlinuz","isolinux.cfg") From 7aeab7a66b14de784d7f8be93115f3131410638e Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 14 Jul 2014 16:36:24 -0300 Subject: [PATCH 75/99] Change 0/1 of detachable to False/True --- carbono/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index 48854dc..90e9675 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -190,9 +190,11 @@ def device_detachable(self, device): proplist.append(device_props.GetAll('org.freedesktop.UDisks.Device')) for device_props in proplist: - if device_props["DeviceFile"] == device: - return device_props['DriveCanDetach'] + if (device_props["DeviceFile"] == device and + device_props['DriveCanDetach']): + return True + return False def formated_partitions(self, filter_disk=None): formated_partitions = [] formated_partitions_dict = self.__DISK_DICT From f4c2c874b1c497a1c067c7237c5c7b0129bc2eef Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Thu, 17 Jul 2014 11:01:50 -0300 Subject: [PATCH 76/99] Add disk_size to info file * Add, to the .info file, info about the total size of the original disk --- carbono/image_creator.py | 23 ++++++++++++--------- carbono/information.py | 16 ++++++++++++--- carbono/utils.py | 44 ++++++++++++++++++++++++++-------------- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/carbono/image_creator.py b/carbono/image_creator.py index 41d447e..6732aaa 100755 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -78,7 +78,7 @@ def notify_percent(self): self.current_percent = percent self.notify_status("progress", {"percent": percent}) - #verify stderr from partclone + #verify stderr from partclone if self.partclone_stderr != None: partclone_status = self.partclone_stderr.readline() if partclone_status.startswith("Partclone successfully cloned the device"): @@ -147,12 +147,15 @@ def create_image(self): for part in partition_list: total_bytes += part.filesystem.get_used_size() - self.total_blocks = long(math.ceil(total_bytes/float(BLOCK_SIZE))) + self.total_blocks = long(math.ceil(total_bytes/float(BLOCK_SIZE))) information = Information(self.target_path) information.set_image_is_disk(device.is_disk()) information.set_image_name(self.image_name) information.set_image_compressor_level(self.compressor_level) - + if device.is_disk(): + disk_info = DiskInfo() + disk_dict = disk_info.formated_disk(self.device_path) + information.set_disk_size(disk_dict["size"]) # TODO: Abstract this whole part, when creating isos, # splitting in files, etc... @@ -176,7 +179,7 @@ def create_image(self): part.filesystem.open_to_read() #check if partclone is running - if type in ("ext2","ext3","ext4"): + if type in ("ext2","ext3","ext4"): self.partclone_stderr = part.filesystem.get_error_ext() compact_callback = None @@ -190,7 +193,7 @@ def create_image(self): compact_callback) self.buffer_manager.start() - buffer = self.buffer_manager.output_buffer + buffer = self.buffer_manager.output_buffer volumes = 1 while self.active: total_written = 0 # Used to help splitting the file @@ -220,13 +223,13 @@ def create_image(self): self.cancel() raise ErrorReadingFromDevice(e) break - + if data == EOF: if (self.partclone_stderr != None): self.data_is_eof = True while self.partclone_sucess == False: - pass + pass self.partclone_stderr = None self.partclone_sucess = False @@ -250,7 +253,7 @@ def create_image(self): break except Exception as e: log.info(e) - break + break raise ErrorWritingToDevice("Error in write file {0}".\ format(file_path)) @@ -316,7 +319,7 @@ def create_image(self): if self.canceled: log.info("Creation canceled") - self.notify_status("canceled", {"operation": + self.notify_status("canceled", {"operation": "Create image"}) else: self.notify_status("finish") @@ -327,7 +330,7 @@ def stop(self): if hasattr(self, "buffer_manager"): self.buffer_manager.stop() self.active = False - self.timer.stop() + self.timer.stop() log.info("Create image stopped") def cancel(self): diff --git a/carbono/information.py b/carbono/information.py index 403a63e..5ef0f51 100644 --- a/carbono/information.py +++ b/carbono/information.py @@ -21,6 +21,7 @@ from carbono.utils import * from carbono.exception import * + class Information: def __init__(self, target_path): @@ -56,6 +57,10 @@ def set_image_is_disk(self, is_disk): """ """ self._doc.update({"is_disk": is_disk}) + def set_disk_size(self, size): + """ """ + self._doc.update({"disk_size": size}) + def add_partition(self, number, type, volumes, size, uuid=None, label=None): """ """ @@ -79,7 +84,13 @@ def add_partition(self, number, type, volumes, size, def get_image_name(self): """ """ return self._doc["name"] - + + def get_disk_size(self): + """ """ + if hasattr(self, _doc["disk_size"]): + return self._doc["disk_size"] + return None + def get_image_compressor_level(self): """ """ return self._doc["compressor_level"] @@ -90,7 +101,7 @@ def get_image_is_disk(self): def get_partitions(self): """ """ - if hasattr(self,"_partitions"): + if hasattr(self, "_partitions"): partitions = self._partitions else: partitions = self._doc["partitions"] @@ -114,4 +125,3 @@ def load(self): except IOError, e: if e.errno == errno.ENOENT: raise ImageNotFound("Image not found") - diff --git a/carbono/utils.py b/carbono/utils.py index 90e9675..a38dbfb 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -179,22 +179,36 @@ def disk_usage(self, path): return disk_usage def device_detachable(self, device): - bus = dbus.SystemBus() - ud_manager_obj = bus.get_object('org.freedesktop.UDisks', - '/org/freedesktop/UDisks') - proplist = [] - ud_manager = dbus.Interface(ud_manager_obj, 'org.freedesktop.UDisks') - for device_pc in ud_manager.EnumerateDevices(): - device_obj = bus.get_object('org.freedesktop.UDisks', device_pc) - device_props = dbus.Interface(device_obj, dbus.PROPERTIES_IFACE) - proplist.append(device_props.GetAll('org.freedesktop.UDisks.Device')) - - for device_props in proplist: - if (device_props["DeviceFile"] == device and - device_props['DriveCanDetach']): - return True - + try: + bus = dbus.SystemBus() + ud_manager_obj = bus.get_object('org.freedesktop.UDisks', + '/org/freedesktop/UDisks') + proplist = [] + ud_manager = dbus.Interface(ud_manager_obj, 'org.freedesktop.UDisks') + for device_pc in ud_manager.EnumerateDevices(): + device_obj = bus.get_object('org.freedesktop.UDisks', device_pc) + device_props = dbus.Interface(device_obj, dbus.PROPERTIES_IFACE) + proplist.append(device_props.GetAll('org.freedesktop.UDisks.Device')) + + for device_props in proplist: + if (device_props["DeviceFile"] == device and + device_props['DriveCanDetach']): + return True + except Exception as e: + print e return False + + def formated_disk(self, filter_disk=None): + """ """ + self.__collect_information_about_devices() + formated_partitions_list = self.__DISK_DICT + formated_partitions_dict = {} + if filter_disk is not None: + return formated_partitions_list[filter_disk] + else: + for disk in formated_partitions_list: + formated_partitions_dict[disk] = formated_partitions_list[disk] + return formated_partitions_dict def formated_partitions(self, filter_disk=None): formated_partitions = [] formated_partitions_dict = self.__DISK_DICT From 2c648b44b2d6c9571ace23b772cce97cd7005155 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 22 Jul 2014 15:35:43 -0300 Subject: [PATCH 77/99] Add cdrom info to utils * Adiciona funcoes para retorna info sobre drive de cdrom * muda versao de 2.2.0 para 2.2.1 --- carbono/utils.py | 36 ++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/carbono/utils.py b/carbono/utils.py index a38dbfb..a9f45ba 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -57,6 +57,16 @@ def get_volumes_udis(self): return out.split() + def is_storage_cdrom(self, storage_udi): + '''Verifies if given udi is a cdrom storage''' + out,err,ret = run_simple_command_echo('{0} --udi {1} \ + --key storage.cdrom.cdr'.format( + which('hal-get-property'),storage_udi), False) + if 'true' in out: + return True + return False + + def is_storage_removable(self, storage_udi): '''Verifies if given udi is a removable storage''' out,err,ret = run_simple_command_echo('{0} --udi {1} \ @@ -86,7 +96,18 @@ def get_block_device(self, udi): ) return out + def get_storage_model(self, storage_udi): + '''Returns model of given storage''' + out,err,ret = run_simple_command_echo('{0} --udi {1} \ + --key storage.model'.format( + which('hal-get-property'),storage_udi), False) + return out + def get_external_devices(self): + ''' + get all external devices + returns a list of /dev/xxx. ie.: ['/dev/sdd','/dev/sr0'] + ''' storages_udis = self.get_storages_udis() volumes_udis = self.get_volumes_udis() for s in volumes_udis: @@ -105,6 +126,21 @@ def get_external_devices(self): self.external_devices.append(vol) return self.external_devices + def get_cdrom_devices(self): + ''' + get cdroms devices + returns a dict with device path and it model string + ie.: {"/dev/sr0":{'model': "dvdrw"} } + ''' + storage_udis = self.get_storages_udis() + cdrom_dict = {} + for st in storage_udis: + if self.is_storage_cdrom(st): + device = self.get_block_device(st).split()[0] + model = self.get_storage_model(st) + cdrom_dict[device] = {"model": model} + return cdrom_dict + class DiskInfo(): diff --git a/setup.py b/setup.py index f22a582..2650cde 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() -VERSION_ID = "2.2.0" +VERSION_ID = "2.2.1" BUILD_ID = "{0}-SNAPSHOT".format(VERSION_ID) if 'BUILD_NUMBER' in os.environ: BUILD_ID = "{0}.{1}".format(VERSION_ID, os.environ['BUILD_NUMBER']) From 45353c1ea43d43ae62e5200668204f455a128a69 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Wed, 30 Jul 2014 16:52:57 -0300 Subject: [PATCH 78/99] fix #2040 * Umount folder after it has been mounted to tkae info of (free) size --- carbono/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/carbono/utils.py b/carbono/utils.py index a9f45ba..71f22dd 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -245,6 +245,7 @@ def formated_disk(self, filter_disk=None): for disk in formated_partitions_list: formated_partitions_dict[disk] = formated_partitions_list[disk] return formated_partitions_dict + def formated_partitions(self, filter_disk=None): formated_partitions = [] formated_partitions_dict = self.__DISK_DICT @@ -272,6 +273,7 @@ def formated_partitions(self, filter_disk=None): part_dict["path"] = part part_dict["size"] = size_mb formated_partitions.append(part_dict) + disk_part.umount_partition() formated_partitions.sort(reverse=False) disk = formated_partitions[0]['path'][:8] From 107794ce6190db9471aa638a824beacb58fca01d Mon Sep 17 00:00:00 2001 From: leandro Date: Wed, 30 Jul 2014 18:05:03 -0300 Subject: [PATCH 79/99] Modificacao para retorno de valores inteiros na porcentagem do Partclone --- carbono/image_creator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/carbono/image_creator.py b/carbono/image_creator.py index 6732aaa..ff2c937 100755 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -89,7 +89,9 @@ def notify_percent(self): if part_list[0] == "current": if len(part_list) >= 13: try: - self.notify_status("waiting_partclone",{"partclone_percent":partclone_status.split()[13].split(",")[0]}) + status = partclone_status.split()[13].split(",")[0] + status2 = status.split("%")[0] + self.notify_status("waiting_partclone",{"partclone_percent":float(status2)}) except Exception as e: log.info(e) raise ErrorCreatingImage("Create image wasn't made with success") From ed3778ccd3bb376176d6fdc1497e83829b2b1556 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 4 Aug 2014 15:33:16 -0300 Subject: [PATCH 80/99] fix #2070 @2h --- carbono/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carbono/utils.py b/carbono/utils.py index 71f22dd..86e23e5 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -358,7 +358,7 @@ def umount_all_partitions(self): def mount_partition(self, destino = None, ro = False): mount_options = "" - if not ro: + if ro: mount_options += "-o ro" mounted_folder = self.get_mount_point(self.__partition) From 57cba0b04cdbc86889a45865440687097b688496 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Wed, 6 Aug 2014 10:48:37 -0300 Subject: [PATCH 81/99] Change how umount is done Ao listar as info de discos, deixa montado os discos que ja estavam montados, e tenta desmontar duas vezes os dispositivos a desmontar --- carbono/utils.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/carbono/utils.py b/carbono/utils.py index 86e23e5..26f4eda 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -255,7 +255,11 @@ def formated_partitions(self, filter_disk=None): for part in self.__PARTITION_DICT.keys(): part_dict = {} disk_part = DiskPartition(part) - tmp_folder_part = disk_part.mount_partition(ro=True) + tmp_folder_part = disk_part.get_mounted_folder() + was_mounted = True + if not tmp_folder_part: + was_mounted = False + tmp_folder_part = disk_part.mount_partition(ro=True) part_dict["total_size"] = None part_dict["used_size"] = None part_dict["free_size"] = None @@ -273,7 +277,8 @@ def formated_partitions(self, filter_disk=None): part_dict["path"] = part part_dict["size"] = size_mb formated_partitions.append(part_dict) - disk_part.umount_partition() + if not was_mounted: + disk_part.umount_partition() formated_partitions.sort(reverse=False) disk = formated_partitions[0]['path'][:8] @@ -332,8 +337,15 @@ def umount_partition(self, device=None): result_disk_umounted[item] = 0 print "A particao {0} foi desmontada do diretorio {1}".format(self.__partition, item) else: - result_disk_umounted[item] = -1 - print "A particao {0} montada em {1} nao foi desmontada".format(self.__partition,item) + cmd = "umount {0}".format(item) + p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE, shell=True) + ret = os.waitpid(p.pid,0)[1] + if not ret: + result_disk_umounted[item] = 0 + print "A particao {0} foi desmontada do diretorio {1}".format(self.__partition, item) + else: + result_disk_umounted[item] = -1 + print "A particao {0} montada em {1} nao foi desmontada".format(self.__partition,item) return result_disk_umounted def umount_all_partitions(self): @@ -355,6 +367,12 @@ def umount_all_partitions(self): result_disk_umounted[item] = result_part_umounted return result_disk_umounted + def get_mounted_folder(self): + mounted_folder = self.get_mount_point(self.__partition) + + if mounted_folder: + return mounted_folder[0] + return False def mount_partition(self, destino = None, ro = False): mount_options = "" From a7142014c4a97487f64bf6856eb0ac03a61846b1 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Wed, 3 Sep 2014 10:33:08 -0300 Subject: [PATCH 82/99] Add ntfs label replication. closes #2750 closes #2748 * Ao clonar particoes ntfs sem compactacao, tb faz a replicacao de label --- carbono/filesystem/generic.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/carbono/filesystem/generic.py b/carbono/filesystem/generic.py index 552dcaf..7a529b3 100644 --- a/carbono/filesystem/generic.py +++ b/carbono/filesystem/generic.py @@ -142,8 +142,25 @@ def resize(self): return True def read_label(self): + proc = subprocess.Popen([which("ntfslabel"), "--force", self.path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = proc.stdout.read() + err = proc.stdout.read() + if "mount" in err: + return None + if output: + output = output.strip() + return output return None def write_label(self, label): + try: + ret = run_simple_command('{0} --force {1} "{2}"'.\ + format(which("ntfslabel"), self.path, label)) + if ret == 0: + return True + return False + except Exception as e: + pass return True From 3a4ee74cf4133d90fc2aef86c4d243253f167b2d Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Thu, 18 Sep 2014 16:57:41 -0300 Subject: [PATCH 83/99] Change iso creation method * Metodo de criacao mudado pois initram nao existem mais * Iso ira bootar a partir de squashfs * Pega a pasta /casper da pendrive, pois eh onde estao os arquivos base * inclui "menu.c32" com 1ms de espera para opcao de botar via ram funcionar por padrao * Gera arquivo isolinux.cfg para boot squashfs pela ram --- carbono/iso_creator.py | 26 +++++++++++++++----------- carbono/utils.py | 10 ++++++++-- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/carbono/iso_creator.py b/carbono/iso_creator.py index f8f35ac..0a6432b 100644 --- a/carbono/iso_creator.py +++ b/carbono/iso_creator.py @@ -22,7 +22,9 @@ from carbono.exception import * from carbono.config import * -CARBONO_FILES = ("initram.gz", "vmlinuz") +CARBONO_FILES = ("filesystem.squashfs", + "vmlinuz", + "initrd.lz") class IsoCreator: @@ -43,11 +45,11 @@ def __generate_isolinux_file(self): isolinux_tmp_file = "/tmp/isolinux.cfg" tmp_file = open(isolinux_tmp_file,"w") tmp_file.write("prompt 0\n"+ - " default upimage\n"+ + "timeout 1\n"+ + " default menu.c32\n"+ "label upimage\n"+ - " kernel vmlinuz\n"+ - " append initrd=initram.gz rdinit=/sbin/init "+ - "ramdisk_size=512000\n") + " kernel /casper/vmlinuz boot=casper toram\n"+ + " append initrd=/casper/initrd.lz file=/cdrom/preseed/ubuntu.seed boot=casper") tmp_file.close() return isolinux_tmp_file @@ -79,7 +81,7 @@ def mount_device(self, device): return tmpd def find_carbono_files(self, path): - dev_files = os.listdir(path) + dev_files = os.listdir(os.path.join(path,"casper")) ret = True if filter(lambda x: not x in dev_files, CARBONO_FILES): @@ -123,9 +125,11 @@ def run(self): break - # Add carbono files - map(lambda x: self.slices[volume].\ - append(self.mount_point + x), CARBONO_FILES) + add = os.path.join(self.mount_point,"casper/") + add="/casper="+add + + self.slices[volume].append(add) + self.slices[volume].append(os.path.join(self.mount_point,"menu.c32")) # Add isolinux.cfg file isolinux_file = self.__generate_isolinux_file() @@ -146,10 +150,10 @@ def run(self): extra_params = "-joliet-long -b " + \ "isolinux.bin -c " + \ "boot.cat -no-emul-boot " + \ - "-boot-load-size 4 -boot-info-table" + "-boot-load-size 4 -boot-info-table -graft-points" slist = ' '.join(self.slices[volume]) - cmd = "{0} -R -J -o {1}{2}{3}.iso {4} {5}".format( + cmd = "{0} -R -J {4} {5} > {1}{2}{3}.iso".format( which("mkisofs"), self.target_path, self.name, diff --git a/carbono/utils.py b/carbono/utils.py index 26f4eda..ad52306 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -592,7 +592,7 @@ def get_devices(): disk_dic[dev_path] = {"partitions": part_dict} return disk_dic -CARBONO_FILES2 = ("initram.gz","vmlinuz","isolinux.cfg") +CARBONO_FILES2 = ("initrd.lz","vmlinuz","filesystem.squashfs") def mount_pen(device): tmpd = make_temp_dir() @@ -602,8 +602,14 @@ def mount_pen(device): return tmpd def find_carbono(path): - dev_files = os.listdir(path) + try: + dev_files = os.listdir(os.path.join(path, "casper")) + except Exception as e: + print e + return False ret = True + if dev_files is None: + ret = False if filter(lambda x:not x in dev_files, CARBONO_FILES2): ret = False return ret From 18bffdd21154605fb0a4f9ba0c717cb3e0217795 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 23 Sep 2014 16:08:15 -0300 Subject: [PATCH 84/99] Change file_not_found status message * Caso seja a nova interface, continua procurando na pasta inicial pelo proximo arquivo da imagem --- carbono/image_reader/generic.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/carbono/image_reader/generic.py b/carbono/image_reader/generic.py index 2f6e817..989f92c 100644 --- a/carbono/image_reader/generic.py +++ b/carbono/image_reader/generic.py @@ -36,7 +36,7 @@ def _check_fd(self): def open(self): if self._check_fd(): file_pattern = self.pattern.format(volume=self.current_volume) - + image_path_aux = None # Loop untill find the the next slice or # cancel the operation while True: @@ -44,15 +44,19 @@ def open(self): if os.path.exists(file_path): self._fd = open(file_path, 'rb') else: - self.image_path = self.notify_callback("file_not_found_cd", + image_path_aux = self.notify_callback("file_not_found_cd", {"path": self.image_path, "file": file_pattern, "current_volume": self.current_volume}) - - if self.image_path: - self.image_path = adjust_path(self.image_path) - continue + if image_path_aux: + self.image_path = image_path_aux + if self.image_path: + self.image_path = adjust_path(self.image_path) + continue + else: + raise Exception("canceled") + break else: - raise Exception("canceled") - break + continue + break def close(self): From 570fd6cd7102366e416c2eae6006517c372a2070 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 17 Nov 2014 17:48:57 -0200 Subject: [PATCH 85/99] Adds a kill to partclone.extfs @fix #5721 * On close a applying image of ext, kills partclone --- carbono/filesystem/ext.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/carbono/filesystem/ext.py b/carbono/filesystem/ext.py index a15826e..46eb711 100644 --- a/carbono/filesystem/ext.py +++ b/carbono/filesystem/ext.py @@ -53,23 +53,24 @@ def get_used_size(self): def open_to_read(self): """ """ #cmd = "{0} -q -c -s {1} -o -".format(which("partclone.extfs"), self.path) - cmd = "{0} -c -s {1} -o -".format(which("partclone.extfs"), self.path) + cmd = "{0} -c -s {1} -o -".format(which("partclone.extfs"), self.path) try: self.process = RunCmd(cmd) self.process.run() self._fd = self.process.stdout - self.fderr = self.process.stderr + self.fderr = self.process.stderr except: raise ErrorOpenToRead("Cannot open {0} to read".format(self.path)) def open_to_write(self, uuid=None): """ """ #cmd = "{0} -q -r -o {1} -s - ".format(which("partclone.extfs"), self.path) - cmd = "{0} -r -o {1} -s - ".format(which("partclone.extfs"), self.path) + cmd = "{0} -r -o {1} -s - ".format(which("partclone.extfs"), self.path) try: self.process = RunCmd(cmd) self.process.run() self._fd = self.process.stdin + self.fderr = self.process.stderr except: raise ErrorOpenToWrite("Cannot open {0} to write".format(self.path)) @@ -82,7 +83,7 @@ def uuid(self): if not len(output): return uuid - + try: uuid = re.search('(?<=UUID=")\w+-\w+-\w+-\w+-\w+', output).group(0) except AttributeError: @@ -92,6 +93,7 @@ def uuid(self): def close(self): """ """ + os.system("pkill -9 partclone.extfs") if self._fd is None or \ self._fd.closed: return @@ -117,11 +119,11 @@ def format_filesystem(self): output = output.strip() if output == 0: return True - return False - + return False + def resize(self): - + if self.check(): proc = subprocess.Popen([which("resize2fs"), self.path], stdout=subprocess.PIPE) @@ -131,7 +133,7 @@ def resize(self): if proc.returncode == 0: return True return False - + def read_label(self): proc = subprocess.Popen([which("e2label"), self.path], stdout=subprocess.PIPE) From 7fff8c89682ea59723d5ac1b37d8094252b84cb4 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 24 Nov 2014 18:34:11 -0200 Subject: [PATCH 86/99] fixes resize function of ntfs and ext --- carbono/filesystem/ext.py | 2 +- carbono/filesystem/ntfs.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/carbono/filesystem/ext.py b/carbono/filesystem/ext.py index 46eb711..3b6b718 100644 --- a/carbono/filesystem/ext.py +++ b/carbono/filesystem/ext.py @@ -103,7 +103,7 @@ def close(self): def check(self): ret = run_simple_command("{0} -f -y -v {1}".format(which("e2fsck"), self.path)) - proc = subprocess.Popen([which("resize2fs"), self.path], + proc = subprocess.Popen([which("resize2fs"), "-f", self.path], stdout=subprocess.PIPE) output = proc.stdout.read() output = output.strip() diff --git a/carbono/filesystem/ntfs.py b/carbono/filesystem/ntfs.py index 144e185..e3b1115 100755 --- a/carbono/filesystem/ntfs.py +++ b/carbono/filesystem/ntfs.py @@ -58,12 +58,12 @@ def get_size(self): size = long(l.split()[3]) except ValueError: raise ErrorGettingSize - + if size is None: raise ErrorGettingSize return size - + def get_used_size(self): """ """ proc = subprocess.Popen("{0} -i {1} -f".format(which("ntfsresize"), self.path), @@ -77,7 +77,7 @@ def get_used_size(self): size = long(l.split()[4]) except ValueError: raise ErrorGettingUsedSize - + if size is None: raise ErrorGettingUsedSize @@ -97,19 +97,19 @@ def format_filesystem(self): output = output.strip() if output == 0: return True - return False + return False def resize(self): - + returncode = -1 if self.check(): - proc = subprocess.Popen([which("ntfsresize"), self.path], + proc = subprocess.Popen([which("ntfsresize"), "--force", self.path], stdout=subprocess.PIPE) proc.wait() output = proc.stdout.read() output = output.strip() returncode = proc.returncode - + if returncode == 0: return True return False From 63f3c879426edbc7141879e1f849fb70f138c986 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Thu, 4 Dec 2014 10:51:36 -0200 Subject: [PATCH 87/99] Change cdrom device find --- carbono/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/carbono/utils.py b/carbono/utils.py index ad52306..01bb063 100644 --- a/carbono/utils.py +++ b/carbono/utils.py @@ -421,7 +421,9 @@ def get_mounted_devices(self, device = None): mount_command = subprocess.check_output(['mount','-l']).split('\n') for lines in mount_command: line = lines.split(' ') - if line[0].startswith('/dev/sd'): + if (line[0].startswith('/dev/sd') or + line[0].startswith('/dev/sr') or + line[0].startswith('/dev/cd')): if line[0] not in disk_mounted.keys(): list_dest = [] list_dest.append(line[2]) From ecbb574d59b0815f15113f07d2ef382ade1c5075 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 15 Dec 2014 16:54:24 -0200 Subject: [PATCH 88/99] Change mount to read-only on iso_Creator @fix #6729 --- carbono/iso_creator.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/carbono/iso_creator.py b/carbono/iso_creator.py index 0a6432b..2bcec37 100644 --- a/carbono/iso_creator.py +++ b/carbono/iso_creator.py @@ -21,6 +21,7 @@ from carbono.utils import * from carbono.exception import * from carbono.config import * +from carbono.utils import DiskPartition CARBONO_FILES = ("filesystem.squashfs", "vmlinuz", @@ -73,12 +74,11 @@ def notify_percent(self): return None def mount_device(self, device): - tmpd = make_temp_dir() - ret = run_simple_command("mount {0} {1}".\ - format(device, tmpd)) - if ret is not 0: - raise ErrorMountingFilesystem - return tmpd + ''' ''' + disk_part = DiskPartition(device) + folder = disk_part.mount_partition(ro=True) + + return folder def find_carbono_files(self, path): dev_files = os.listdir(os.path.join(path,"casper")) @@ -145,6 +145,11 @@ def run(self): map(lambda x: self.slices[volume].\ append(self.target_path + x), ("mbr.bin", "disk.dl")) + if os.path.exists("{0}UpImageAuto.json".format( + self.target_path)): + + self.slices[volume].append("{0}UpImageAuto.json".format( + self.target_path)) if first_volume: extra_params = "-joliet-long -b " + \ From a39ffff3d72cbc23b596135ecd8743b9c6d87f04 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Thu, 22 Jan 2015 16:25:30 -0200 Subject: [PATCH 89/99] Fix #7112 #6548 @4h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bug: 'Ao realizar aplicação da imagem máquina travou em "99,97%"' * Causa: Partição invalida causava partition = None Tentava acessar metodo de partition antes de verificar * Solucao: Verifica se é None antes de acessar metodo de partition Se todas partitions forem None, nao acessar metodos do obj nenhuma vez, retornando e dando raise na exception gerada --- carbono/image_restorer.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/carbono/image_restorer.py b/carbono/image_restorer.py index be2458b..82a3cae 100755 --- a/carbono/image_restorer.py +++ b/carbono/image_restorer.py @@ -79,6 +79,8 @@ def notify_percent(self): def restore_image(self): """ """ + invalid_partitions = False + if is_mounted(self.target_device): log.error("The partition {0} is mounted, please umount first, and try again".format(self.target_device)) self.notify_status("mounted_partition_error",{"mounted_partition_error":self.target_device}) @@ -191,14 +193,15 @@ def restore_image(self): self.target_device, part.type) + if partition is None: + invalid_partitions = True + continue + log.info("Restoring partition {0}".format(partition.get_path())) self.notify_status("restore_partition",\ {"restoring_partition":partition.get_path()}) - if partition is None: - self.notify_status("no_valid_partitions", \ - {"no_valid_partitions":partitions.get_path()}) - raise ErrorRestoringImage("No valid partitions found") + invalid_partitions = False if hasattr(part, "uuid"): partition.filesystem.open_to_write(part.uuid) @@ -266,6 +269,10 @@ def restore_image(self): self.buffer_manager.join() partition.filesystem.close() + if invalid_partitions: + self.notify_status("no_valid_partitions", \ + {"no_valid_partitions":partitions}) + raise ErrorRestoringImage("No valid partitions found") self.timer.stop() From 0afa394767451552b5ea2d41330ab215a2f77339 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Wed, 18 Feb 2015 18:43:54 -0200 Subject: [PATCH 90/99] Removes the delete folder if error. @refs #6392 * If an writing error occurs 'cause there was no space left, is was send a error message and se folder of target_path was deleted. THIS WAS NOT GOOD (imagine if the target_path=/media/ !) * Removes the deleting target_path folder thing. * Adds an self.cancel to cancel operation the above error occurs * and do not solve the problem #6392 --- carbono/image_creator.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/carbono/image_creator.py b/carbono/image_creator.py index ff2c937..68ce632 100755 --- a/carbono/image_creator.py +++ b/carbono/image_creator.py @@ -249,13 +249,7 @@ def create_image(self): except Exception as e: log.info("{0}".format(e)) self.notify_status("disk_full") - try: - shutil.rmtree(self.target_path) - log.info("The folder which cointain the images files were erased") - break - except Exception as e: - log.info(e) - break + self.cancel() raise ErrorWritingToDevice("Error in write file {0}".\ format(file_path)) From 559972622e66ec5ae01026de3278503f2e62dc50 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 23 Feb 2015 18:10:22 -0300 Subject: [PATCH 91/99] Modificacoes de pasta para MPS.BR @refs #7330 @1h - Muda todas pastas para uma pasta src/ , incluindo o setup.py - Cria pata build/ para scripts de build - Cria pasta tests/ para testes unitarios --- .gitignore | 6 +++--- build/.gitkeep | 0 COPYING => src/COPYING | 0 README.md => src/README.md | 0 TODO => src/TODO | 0 {carbono => src/carbono}/__init__.py | 0 {carbono => src/carbono}/boot_manager/__init__.py | 0 .../carbono}/boot_manager/boot_manager.py | 0 {carbono => src/carbono}/boot_manager/disk_utils.py | 0 {carbono => src/carbono}/boot_manager/grub2.py | 0 .../carbono}/boot_manager/grub_legacy.py | 0 {carbono => src/carbono}/boot_manager/syslinux.py | 0 {carbono => src/carbono}/boot_manager/utils_misc.py | 0 {carbono => src/carbono}/buffer_manager/__init__.py | 0 .../carbono}/buffer_manager/dummy_manager.py | 0 .../carbono}/buffer_manager/reorder_buffer.py | 0 .../carbono}/buffer_manager/simple_manager.py | 0 .../carbono}/buffer_manager/work_manager.py | 0 {carbono => src/carbono}/caspart/__init__.py | 0 {carbono => src/carbono}/caspart/caspart.py | 0 {carbono => src/carbono}/caspart/utils_misc.py | 0 {carbono => src/carbono}/compressor.py | 0 {carbono => src/carbono}/config.py | 0 {carbono => src/carbono}/device.py | 0 {carbono => src/carbono}/disk.py | 0 {carbono => src/carbono}/disk_layout_manager.py | 0 {carbono => src/carbono}/exception.py | 0 {carbono => src/carbono}/filesystem/__init__.py | 0 {carbono => src/carbono}/filesystem/btrfs.py | 0 {carbono => src/carbono}/filesystem/ext.py | 0 {carbono => src/carbono}/filesystem/generic.py | 0 {carbono => src/carbono}/filesystem/linux_swap.py | 0 {carbono => src/carbono}/filesystem/ntfs.py | 0 {carbono => src/carbono}/image_creator.py | 0 {carbono => src/carbono}/image_reader/__init__.py | 0 {carbono => src/carbono}/image_reader/compressed.py | 0 {carbono => src/carbono}/image_reader/generic.py | 0 {carbono => src/carbono}/image_restorer.py | 0 {carbono => src/carbono}/information.py | 0 {carbono => src/carbono}/iso_creator.py | 0 {carbono => src/carbono}/log.py | 0 {carbono => src/carbono}/mbr.py | 0 {carbono => src/carbono}/partition.py | 0 {carbono => src/carbono}/partition_expander.py | 0 {carbono => src/carbono}/ui/__init__.py | 0 {carbono => src/carbono}/ui/cli.py | 0 {carbono => src/carbono}/utils.py | 0 {scripts => src/scripts}/carbono | 0 setup.py => src/setup.py | 0 {test => src/test}/gui/carbono.glade | 0 {test => src/test}/gui/gui.py | 0 {test => src/test}/gui/main_bg.png | Bin {test => src/test}/gui/refs | 0 {test => src/test}/gui/select_file.glade | 0 {test => src/test}/gui/treeview_disks.py | 0 tests/.gitkeep | 0 56 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 build/.gitkeep rename COPYING => src/COPYING (100%) rename README.md => src/README.md (100%) rename TODO => src/TODO (100%) rename {carbono => src/carbono}/__init__.py (100%) rename {carbono => src/carbono}/boot_manager/__init__.py (100%) rename {carbono => src/carbono}/boot_manager/boot_manager.py (100%) rename {carbono => src/carbono}/boot_manager/disk_utils.py (100%) rename {carbono => src/carbono}/boot_manager/grub2.py (100%) rename {carbono => src/carbono}/boot_manager/grub_legacy.py (100%) rename {carbono => src/carbono}/boot_manager/syslinux.py (100%) rename {carbono => src/carbono}/boot_manager/utils_misc.py (100%) rename {carbono => src/carbono}/buffer_manager/__init__.py (100%) rename {carbono => src/carbono}/buffer_manager/dummy_manager.py (100%) rename {carbono => src/carbono}/buffer_manager/reorder_buffer.py (100%) rename {carbono => src/carbono}/buffer_manager/simple_manager.py (100%) rename {carbono => src/carbono}/buffer_manager/work_manager.py (100%) rename {carbono => src/carbono}/caspart/__init__.py (100%) rename {carbono => src/carbono}/caspart/caspart.py (100%) rename {carbono => src/carbono}/caspart/utils_misc.py (100%) rename {carbono => src/carbono}/compressor.py (100%) rename {carbono => src/carbono}/config.py (100%) rename {carbono => src/carbono}/device.py (100%) rename {carbono => src/carbono}/disk.py (100%) rename {carbono => src/carbono}/disk_layout_manager.py (100%) rename {carbono => src/carbono}/exception.py (100%) rename {carbono => src/carbono}/filesystem/__init__.py (100%) rename {carbono => src/carbono}/filesystem/btrfs.py (100%) rename {carbono => src/carbono}/filesystem/ext.py (100%) rename {carbono => src/carbono}/filesystem/generic.py (100%) rename {carbono => src/carbono}/filesystem/linux_swap.py (100%) rename {carbono => src/carbono}/filesystem/ntfs.py (100%) rename {carbono => src/carbono}/image_creator.py (100%) rename {carbono => src/carbono}/image_reader/__init__.py (100%) rename {carbono => src/carbono}/image_reader/compressed.py (100%) rename {carbono => src/carbono}/image_reader/generic.py (100%) rename {carbono => src/carbono}/image_restorer.py (100%) rename {carbono => src/carbono}/information.py (100%) rename {carbono => src/carbono}/iso_creator.py (100%) rename {carbono => src/carbono}/log.py (100%) rename {carbono => src/carbono}/mbr.py (100%) rename {carbono => src/carbono}/partition.py (100%) rename {carbono => src/carbono}/partition_expander.py (100%) rename {carbono => src/carbono}/ui/__init__.py (100%) rename {carbono => src/carbono}/ui/cli.py (100%) rename {carbono => src/carbono}/utils.py (100%) rename {scripts => src/scripts}/carbono (100%) rename setup.py => src/setup.py (100%) rename {test => src/test}/gui/carbono.glade (100%) rename {test => src/test}/gui/gui.py (100%) rename {test => src/test}/gui/main_bg.png (100%) rename {test => src/test}/gui/refs (100%) rename {test => src/test}/gui/select_file.glade (100%) rename {test => src/test}/gui/treeview_disks.py (100%) create mode 100644 tests/.gitkeep diff --git a/.gitignore b/.gitignore index 8e28cda..f166069 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -build/ -dist/ -carbono.egg-info/ +src/build/ +src/dist/ +src/carbono.egg-info/ *.swp *.mo *.pyc diff --git a/build/.gitkeep b/build/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/COPYING b/src/COPYING similarity index 100% rename from COPYING rename to src/COPYING diff --git a/README.md b/src/README.md similarity index 100% rename from README.md rename to src/README.md diff --git a/TODO b/src/TODO similarity index 100% rename from TODO rename to src/TODO diff --git a/carbono/__init__.py b/src/carbono/__init__.py similarity index 100% rename from carbono/__init__.py rename to src/carbono/__init__.py diff --git a/carbono/boot_manager/__init__.py b/src/carbono/boot_manager/__init__.py similarity index 100% rename from carbono/boot_manager/__init__.py rename to src/carbono/boot_manager/__init__.py diff --git a/carbono/boot_manager/boot_manager.py b/src/carbono/boot_manager/boot_manager.py similarity index 100% rename from carbono/boot_manager/boot_manager.py rename to src/carbono/boot_manager/boot_manager.py diff --git a/carbono/boot_manager/disk_utils.py b/src/carbono/boot_manager/disk_utils.py similarity index 100% rename from carbono/boot_manager/disk_utils.py rename to src/carbono/boot_manager/disk_utils.py diff --git a/carbono/boot_manager/grub2.py b/src/carbono/boot_manager/grub2.py similarity index 100% rename from carbono/boot_manager/grub2.py rename to src/carbono/boot_manager/grub2.py diff --git a/carbono/boot_manager/grub_legacy.py b/src/carbono/boot_manager/grub_legacy.py similarity index 100% rename from carbono/boot_manager/grub_legacy.py rename to src/carbono/boot_manager/grub_legacy.py diff --git a/carbono/boot_manager/syslinux.py b/src/carbono/boot_manager/syslinux.py similarity index 100% rename from carbono/boot_manager/syslinux.py rename to src/carbono/boot_manager/syslinux.py diff --git a/carbono/boot_manager/utils_misc.py b/src/carbono/boot_manager/utils_misc.py similarity index 100% rename from carbono/boot_manager/utils_misc.py rename to src/carbono/boot_manager/utils_misc.py diff --git a/carbono/buffer_manager/__init__.py b/src/carbono/buffer_manager/__init__.py similarity index 100% rename from carbono/buffer_manager/__init__.py rename to src/carbono/buffer_manager/__init__.py diff --git a/carbono/buffer_manager/dummy_manager.py b/src/carbono/buffer_manager/dummy_manager.py similarity index 100% rename from carbono/buffer_manager/dummy_manager.py rename to src/carbono/buffer_manager/dummy_manager.py diff --git a/carbono/buffer_manager/reorder_buffer.py b/src/carbono/buffer_manager/reorder_buffer.py similarity index 100% rename from carbono/buffer_manager/reorder_buffer.py rename to src/carbono/buffer_manager/reorder_buffer.py diff --git a/carbono/buffer_manager/simple_manager.py b/src/carbono/buffer_manager/simple_manager.py similarity index 100% rename from carbono/buffer_manager/simple_manager.py rename to src/carbono/buffer_manager/simple_manager.py diff --git a/carbono/buffer_manager/work_manager.py b/src/carbono/buffer_manager/work_manager.py similarity index 100% rename from carbono/buffer_manager/work_manager.py rename to src/carbono/buffer_manager/work_manager.py diff --git a/carbono/caspart/__init__.py b/src/carbono/caspart/__init__.py similarity index 100% rename from carbono/caspart/__init__.py rename to src/carbono/caspart/__init__.py diff --git a/carbono/caspart/caspart.py b/src/carbono/caspart/caspart.py similarity index 100% rename from carbono/caspart/caspart.py rename to src/carbono/caspart/caspart.py diff --git a/carbono/caspart/utils_misc.py b/src/carbono/caspart/utils_misc.py similarity index 100% rename from carbono/caspart/utils_misc.py rename to src/carbono/caspart/utils_misc.py diff --git a/carbono/compressor.py b/src/carbono/compressor.py similarity index 100% rename from carbono/compressor.py rename to src/carbono/compressor.py diff --git a/carbono/config.py b/src/carbono/config.py similarity index 100% rename from carbono/config.py rename to src/carbono/config.py diff --git a/carbono/device.py b/src/carbono/device.py similarity index 100% rename from carbono/device.py rename to src/carbono/device.py diff --git a/carbono/disk.py b/src/carbono/disk.py similarity index 100% rename from carbono/disk.py rename to src/carbono/disk.py diff --git a/carbono/disk_layout_manager.py b/src/carbono/disk_layout_manager.py similarity index 100% rename from carbono/disk_layout_manager.py rename to src/carbono/disk_layout_manager.py diff --git a/carbono/exception.py b/src/carbono/exception.py similarity index 100% rename from carbono/exception.py rename to src/carbono/exception.py diff --git a/carbono/filesystem/__init__.py b/src/carbono/filesystem/__init__.py similarity index 100% rename from carbono/filesystem/__init__.py rename to src/carbono/filesystem/__init__.py diff --git a/carbono/filesystem/btrfs.py b/src/carbono/filesystem/btrfs.py similarity index 100% rename from carbono/filesystem/btrfs.py rename to src/carbono/filesystem/btrfs.py diff --git a/carbono/filesystem/ext.py b/src/carbono/filesystem/ext.py similarity index 100% rename from carbono/filesystem/ext.py rename to src/carbono/filesystem/ext.py diff --git a/carbono/filesystem/generic.py b/src/carbono/filesystem/generic.py similarity index 100% rename from carbono/filesystem/generic.py rename to src/carbono/filesystem/generic.py diff --git a/carbono/filesystem/linux_swap.py b/src/carbono/filesystem/linux_swap.py similarity index 100% rename from carbono/filesystem/linux_swap.py rename to src/carbono/filesystem/linux_swap.py diff --git a/carbono/filesystem/ntfs.py b/src/carbono/filesystem/ntfs.py similarity index 100% rename from carbono/filesystem/ntfs.py rename to src/carbono/filesystem/ntfs.py diff --git a/carbono/image_creator.py b/src/carbono/image_creator.py similarity index 100% rename from carbono/image_creator.py rename to src/carbono/image_creator.py diff --git a/carbono/image_reader/__init__.py b/src/carbono/image_reader/__init__.py similarity index 100% rename from carbono/image_reader/__init__.py rename to src/carbono/image_reader/__init__.py diff --git a/carbono/image_reader/compressed.py b/src/carbono/image_reader/compressed.py similarity index 100% rename from carbono/image_reader/compressed.py rename to src/carbono/image_reader/compressed.py diff --git a/carbono/image_reader/generic.py b/src/carbono/image_reader/generic.py similarity index 100% rename from carbono/image_reader/generic.py rename to src/carbono/image_reader/generic.py diff --git a/carbono/image_restorer.py b/src/carbono/image_restorer.py similarity index 100% rename from carbono/image_restorer.py rename to src/carbono/image_restorer.py diff --git a/carbono/information.py b/src/carbono/information.py similarity index 100% rename from carbono/information.py rename to src/carbono/information.py diff --git a/carbono/iso_creator.py b/src/carbono/iso_creator.py similarity index 100% rename from carbono/iso_creator.py rename to src/carbono/iso_creator.py diff --git a/carbono/log.py b/src/carbono/log.py similarity index 100% rename from carbono/log.py rename to src/carbono/log.py diff --git a/carbono/mbr.py b/src/carbono/mbr.py similarity index 100% rename from carbono/mbr.py rename to src/carbono/mbr.py diff --git a/carbono/partition.py b/src/carbono/partition.py similarity index 100% rename from carbono/partition.py rename to src/carbono/partition.py diff --git a/carbono/partition_expander.py b/src/carbono/partition_expander.py similarity index 100% rename from carbono/partition_expander.py rename to src/carbono/partition_expander.py diff --git a/carbono/ui/__init__.py b/src/carbono/ui/__init__.py similarity index 100% rename from carbono/ui/__init__.py rename to src/carbono/ui/__init__.py diff --git a/carbono/ui/cli.py b/src/carbono/ui/cli.py similarity index 100% rename from carbono/ui/cli.py rename to src/carbono/ui/cli.py diff --git a/carbono/utils.py b/src/carbono/utils.py similarity index 100% rename from carbono/utils.py rename to src/carbono/utils.py diff --git a/scripts/carbono b/src/scripts/carbono similarity index 100% rename from scripts/carbono rename to src/scripts/carbono diff --git a/setup.py b/src/setup.py similarity index 100% rename from setup.py rename to src/setup.py diff --git a/test/gui/carbono.glade b/src/test/gui/carbono.glade similarity index 100% rename from test/gui/carbono.glade rename to src/test/gui/carbono.glade diff --git a/test/gui/gui.py b/src/test/gui/gui.py similarity index 100% rename from test/gui/gui.py rename to src/test/gui/gui.py diff --git a/test/gui/main_bg.png b/src/test/gui/main_bg.png similarity index 100% rename from test/gui/main_bg.png rename to src/test/gui/main_bg.png diff --git a/test/gui/refs b/src/test/gui/refs similarity index 100% rename from test/gui/refs rename to src/test/gui/refs diff --git a/test/gui/select_file.glade b/src/test/gui/select_file.glade similarity index 100% rename from test/gui/select_file.glade rename to src/test/gui/select_file.glade diff --git a/test/gui/treeview_disks.py b/src/test/gui/treeview_disks.py similarity index 100% rename from test/gui/treeview_disks.py rename to src/test/gui/treeview_disks.py diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 0000000..e69de29 From 9e472c23e6b1ee7e32932e3602a9636bf3b83517 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Mon, 13 Apr 2015 18:12:56 -0300 Subject: [PATCH 92/99] Update version number to 2.2.2 --- src/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.py b/src/setup.py index 2650cde..6749c42 100755 --- a/src/setup.py +++ b/src/setup.py @@ -5,7 +5,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() -VERSION_ID = "2.2.1" +VERSION_ID = "2.2.2" BUILD_ID = "{0}-SNAPSHOT".format(VERSION_ID) if 'BUILD_NUMBER' in os.environ: BUILD_ID = "{0}.{1}".format(VERSION_ID, os.environ['BUILD_NUMBER']) From 503f9228bb51f887667200e1d56a99ab0e553f62 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 14 Apr 2015 15:25:36 -0300 Subject: [PATCH 93/99] refs #7956 @1h - Tenta criar pasta de a pasta destino nao existir --- src/carbono/image_creator.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/carbono/image_creator.py b/src/carbono/image_creator.py index 68ce632..dfefd4b 100755 --- a/src/carbono/image_creator.py +++ b/src/carbono/image_creator.py @@ -66,10 +66,13 @@ def __init__(self, source_device, output_folder, {"not_root":"You dont't have permission"}) if not os.path.isdir(output_folder): - log.info("The folder is invalid") - self.notify_status("invalid_folder",\ - {"invalid_folder":output_folder}) - raise InvalidFolder("Invalid folder {0}".format(output_folder)) + try: + os.mkdir(output_folder) + except Exception as e: + log.info("The folder is invalid. {0}".format(e)) + self.notify_status("invalid_folder",\ + {"invalid_folder":output_folder}) + raise InvalidFolder("Invalid folder {0}".format(output_folder)) def notify_percent(self): #refresh the interface percentage From e7f7e8e7f86810dc379aa41946642a2808d6d373 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Fri, 17 Apr 2015 14:41:22 -0300 Subject: [PATCH 94/99] refs #7957 @2h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Problema: Se todas Hds com partition table, mas sem particoes, dava erro - Solucao: Nao tentar pegar informacoes das particoes quando nao há particoes (doh!) --- src/carbono/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/carbono/utils.py b/src/carbono/utils.py index 01bb063..77c248f 100644 --- a/src/carbono/utils.py +++ b/src/carbono/utils.py @@ -252,6 +252,10 @@ def formated_partitions(self, filter_disk=None): self.__collect_information_about_devices() device_info = {"size":None,"label":None,"partitions":None} + + if self.__PARTITION_DICT.keys() == []: + return formated_partitions_dict + for part in self.__PARTITION_DICT.keys(): part_dict = {} disk_part = DiskPartition(part) From d43d3b588b59bf3a1d346915f3c4526012c2f52d Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Fri, 17 Apr 2015 17:05:12 -0300 Subject: [PATCH 95/99] fix #7957 @2h - Problema: se hd sem tabela de particao = erro - Solucao - filtrar procura por detalhes de hds sem tabela de particao --- src/carbono/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/carbono/utils.py b/src/carbono/utils.py index 77c248f..663a877 100644 --- a/src/carbono/utils.py +++ b/src/carbono/utils.py @@ -162,9 +162,15 @@ def __get_devices(self): dev_path = device.path dev_model = device.model dev_size = device.getSize('b') + + disk_dict[dev_path] = {"model": dev_model, + "size": dev_size, + "partitions": {}} + try: disk = parted.Disk(device) - except: + except Exception as e: + print e continue part_dict = {} @@ -182,9 +188,8 @@ def __get_devices(self): part_dict[part_path] = {"size": part_size, "type": part_type} - disk_dict[dev_path] = {"model": dev_model, - "size": dev_size, - "partitions": part_dict} + disk_dict[dev_path]["partitions"] = part_dict + return disk_dict From 101ab188604aa5eb968ef5a17a3b12d3cf30e343 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Wed, 15 Jul 2015 10:25:35 -0300 Subject: [PATCH 96/99] solve gpt problem - Quando a hd tinha uma tabela de particao GTP ao inves de MBR a restauracao da imagem falhava - Solucao: sempre cria uma tabela MBR antes do inicio de uma aplicacao de disco --- src/carbono/image_restorer.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/carbono/image_restorer.py b/src/carbono/image_restorer.py index 8466dbe..816f483 100755 --- a/src/carbono/image_restorer.py +++ b/src/carbono/image_restorer.py @@ -36,6 +36,8 @@ from carbono.log import log from partition_expander import PartitionExpander +from _ped import disk_new_fresh + class ImageRestorer: def __init__(self, image_folder, target_device, @@ -122,6 +124,12 @@ def restore_image(self): raise ErrorRestoringImage("Unrecognized disk label") if information.get_image_is_disk(): + + d = disk_new_fresh(device.getPedDevice(), _ped.disk_type_get("msdos")) + d.commit_to_dev() + disk = Disk(device) + #meacha + #Get total disk target size disk_size = get_disk_size(self.target_device) if (total_bytes > disk_size): @@ -145,16 +153,16 @@ def restore_image(self): raise ErrorFileNotFound("File not Found {0}".format(image_path)) dlm = DiskLayoutManager(self.image_path) - try: - if self.expand != 2: - dlm.restore_from_file(disk, True) - else: - dlm.restore_from_file(disk, False) - except Exception as e: - log.error("Error to restore the disk.dl file") - image_path = self.image_path.split("/")[3] + "/disk.dl" - self.notify_status("file_not_found",{"file_not_found":image_path}) - raise ErrorFileNotFound("File not found {0}".format(image_path)) + #try: + if self.expand != 2: + dlm.restore_from_file(disk, True) + else: + dlm.restore_from_file(disk, False) + #except Exception as e: + # log.error("Error to restore the disk.dl file") + # image_path = self.image_path.split("/")[3] + "/disk.dl" + # self.notify_status("file_not_found",{"file_not_found":image_path}) + # raise ErrorFileNotFound("File not found {0}".format(image_path)) else: parent_path = get_parent_path(self.target_device) From d05910c7fc7d7d3b715cc33eb3e2eda561bd3041 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Wed, 15 Jul 2015 14:01:43 -0300 Subject: [PATCH 97/99] make a best solution o gtp problem - ao inves de sempre criar um tabela msdos antes de aplicar uma imagem de disco, verifica se o tipo de tabela eh diferente do tipo "msdos", e se sim, recria ela --- src/carbono/image_restorer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/carbono/image_restorer.py b/src/carbono/image_restorer.py index 816f483..8bc32ab 100755 --- a/src/carbono/image_restorer.py +++ b/src/carbono/image_restorer.py @@ -124,11 +124,11 @@ def restore_image(self): raise ErrorRestoringImage("Unrecognized disk label") if information.get_image_is_disk(): - - d = disk_new_fresh(device.getPedDevice(), _ped.disk_type_get("msdos")) - d.commit_to_dev() - disk = Disk(device) - #meacha + if ("msdos" not in disk.getPedDisk().type.name): + #se a tabela nao for msdos, recria ela como msdos para nao haver problemas + d = disk_new_fresh(device.getPedDevice(), _ped.disk_type_get("msdos")) + d.commit_to_dev() + disk = Disk(device) #Get total disk target size disk_size = get_disk_size(self.target_device) From 925c28526f6c8b5b9a21a7bd1dad60611f2bb3e2 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Tue, 26 Apr 2016 14:52:40 -0300 Subject: [PATCH 98/99] re-add readme.md --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..105ba3f --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +carbono +======= + +carbono is a hard disk imaging and recovery application. Carbono +optimize the resulting image saving and compacting only the used blocks in the +hard disk. Multiprocessing is used to deal with performance problems. + +Testing +------- + + WARNING: + DO NOT TEST CARBONO IN YOUR MAIN SYSTEM UNLESS YOU KNOW WHAT YOU ARE DOING. + ITS HIGHLY RECOMMENDED USING VIRTUAL MACHINES WHEN TESTING CARBONO! NEVER +FORGET IT! + +License +------- + +carbono is distributed under the terms of the GNU General Public License, +version 2. +See the [COPYING][4] file for more information. + +Contributor list +---------------- + +Lucas Alvares Gomes (aka umago) + +Contributing +------------ + +1. Fork it +2. Create a branch (`git checkout -b `) +3. Commit your changes (`git commit -am "Added ..."`) +4. Push to the branch (`git push origin `) +5. Create an [Issue][1] with a link to your branch + +Please take a look at [TODO][3] file to see bugs and not-implemented-yet +features. + +[1]: http://github.com/umago/carbono/issues +[2]: http://umago.info/carbono +[3]: https://github.com/umago/carbono/blob/master/TODO +[4]: https://github.com/umago/carbono/blob/master/COPYING +[5]: http://unetbootin.sourceforge.net/ + From e3b3bbbff7b4bc34b2fc4b86ad0d5870eacc47e5 Mon Sep 17 00:00:00 2001 From: Bruno Casella Date: Thu, 15 Feb 2018 16:28:26 -0200 Subject: [PATCH 99/99] fix typo in README --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 105ba3f..c46956c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -carbono +arbono ======= carbono is a hard disk imaging and recovery application. Carbono @@ -10,8 +10,7 @@ Testing WARNING: DO NOT TEST CARBONO IN YOUR MAIN SYSTEM UNLESS YOU KNOW WHAT YOU ARE DOING. - ITS HIGHLY RECOMMENDED USING VIRTUAL MACHINES WHEN TESTING CARBONO! NEVER -FORGET IT! + ITS HIGHLY RECOMMENDED USING VIRTUAL MACHINES WHEN TESTING CARBONO! NEVER FORGET IT! License -------