Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix pool duplicated #1996

Merged
merged 10 commits into from
Oct 24, 2023
Merged

Fix pool duplicated #1996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5564a28
wip: do not show volumes from duplicated pools
frankiejol Oct 6, 2023
62dbe0d
wip: ignore linked duplicated pools
frankiejol Oct 9, 2023
76f1628
wip: properly follow link
frankiejol Oct 9, 2023
50e7d83
test: test links at different levels
frankiejol Oct 9, 2023
59b4bc6
wip: another algorithm to search for unused
frankiejol Oct 11, 2023
6186c8c
doc: list unused volumes
frankiejol Oct 17, 2023
9d06a07
wip: properly create test
frankiejol Oct 18, 2023
00d405b
wip: remove storage pool
frankiejol Oct 19, 2023
f5c7f49
wip: properly show what file won't be deleted
frankiejol Oct 19, 2023
eb8ebe8
wip: properly cache link
frankiejol Oct 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 53 additions & 8 deletions lib/Ravada/VM.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,8 +146,6 @@ around 'ping' => \&_around_ping;
around 'connect' => \&_around_connect;
after 'disconnect' => \&_post_disconnect;

around '_list_used_volumes' => \&_around_list_used_volumes;

#############################################################
#
# method modifiers
Expand Down Expand Up @@ -2515,6 +2513,32 @@ sub dir_backup($self) {
return $dir_backup;
}


sub _follow_link($self, $file) {
my ($dir, $name) = $file =~ m{(.*)/(.*)};
my $file2 = $file;
if ($dir) {
my $dir2 = $self->_follow_link($dir);
$file2 = "$dir2/$name";
}

if (!defined $self->{_is_link}->{$file2} ) {
my ($out,$err) = $self->run_command("stat","-c",'"%N"', $file2);
chomp $out;
my ($link) = $out =~ m{ -> '(.+)'};
if ($link) {
if ($link !~ m{^/}) {
my ($path) = $file2 =~ m{(.*/)};
$path = "/" if !$path;
$link = "$path$link";
}
$self->{_is_link}->{$file2} = $link;
}
}
$self->{_is_link}->{$file2} = $file2 if !exists $self->{_is_link}->{$file2};
return $self->{_is_link}->{$file2};
}

sub _is_link_remote($self, $vol) {

my ($out,$err) = $self->run_command("stat",$vol);
Expand Down Expand Up @@ -2557,12 +2581,33 @@ sub _is_link($self,$vol) {
return $path_link if $path_link;
}

sub _around_list_used_volumes($orig, $self) {
my @vols = $self->$orig();
my @links;
for my $vol ( @vols ) {
my $link = $self->_is_link($vol);
push @links,($link) if $link;
=head2 list_unused_volumes

Returns a list of unused volume files

=cut

sub list_unused_volumes($self) {
my @all_vols = $self->list_volumes();

my @used = $self->list_used_volumes();
my %used;
for my $vol (@used) {
$used{$vol}++;
my $link = $self->_follow_link($vol);
$used{$link}++ if $link;
}

my @vols;
my %duplicated;
for my $vol ( @all_vols ) {
next if $used{$vol};

my $link = $self->_follow_link($vol);
next if $used{$link};
next if $duplicated{$link}++;

push @vols,($link);
}
return @vols;
}
Expand Down
53 changes: 22 additions & 31 deletions lib/Ravada/VM/KVM.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,8 @@ sub search_volume_re($self,$pattern,$refresh=0) {
my @vols;
for ( 1 .. 10) {
eval { @vols = $pool->list_all_volumes() };
last if $@ && ref($@) && $@->code == 55;
last if $@ && $@ =~ /code: (55|38),/;
last if !$@ || $@ =~ / no storage pool with matching uuid/;
warn "WARNING: on search volume_re: $@";
sleep 1;
Expand Down Expand Up @@ -383,15 +385,16 @@ sub remove_file($self,@files) {
sub _list_volumes($self) {
my @volumes;
for my $pool (_list_storage_pools($self->vm)) {
next if !$pool->is_active;
my @vols;
for ( 1 .. 10) {
my @vols;
for ( 1 .. 10) {
next if !$pool->is_active;
eval { @vols = $pool->list_all_volumes() };
last if $@ && ref($@) && $@->code == 55;
last if !$@ || $@ =~ / no storage pool with matching uuid/;
warn "WARNING: on search volume_re: $@";
sleep 1;
}
push @volumes,@vols;
}
push @volumes,@vols;
}
return @volumes;
}
Expand Down Expand Up @@ -449,7 +452,7 @@ sub _find_all_volumes($self, $xml) {
return @used;
}

sub _list_used_volumes($self) {
sub list_used_volumes($self) {
my @used =$self->_list_used_volumes_known();
for my $name ( $self->discover ) {
my $dom = $self->vm->get_domain_by_name($name);
Expand All @@ -458,20 +461,14 @@ sub _list_used_volumes($self) {
return @used;
}

sub list_unused_volumes($self) {
my %used = map { $_ => 1 } $self->_list_used_volumes();
my @unused;
sub list_volumes($self) {
my $file;
my @volumes;

my $n_found=0;
for my $vol ( $self->_list_volumes ) {

eval { ($file) = $vol->get_path };
confess $@ if $@ && $@ !~ /libvirt error code: 50,/;
next if $used{$file};

my $link = $self->_is_link($file);
next if $link && $used{$link};

my $info;
eval { $info = $vol->get_info() };
Expand All @@ -480,11 +477,10 @@ sub list_unused_volumes($self) {

next if !$info || $info->{type} eq 2;

# cluck Dumper([ $file, [sort grep /2023/,keys %used]]) if $file =~/2023/;
push @unused,($file);
push @volumes,($file);

}
return @unused;
return @volumes;
}

sub refresh_storage_pools($self) {
Expand Down Expand Up @@ -595,6 +591,7 @@ sub file_exists($self, $file) {
sub _file_exists_remote($self, $file) {
$file = $self->_follow_link($file) unless $file =~ /which$/;
for my $pool ($self->vm->list_all_storage_pools ) {
next if !$pool->is_active;
$self->_wait_storage( sub { $pool->refresh() } );
my @volumes = $self->_wait_storage( sub { $pool->list_all_volumes });
for my $vol ( @volumes ) {
Expand All @@ -617,20 +614,6 @@ sub _file_exists_remote($self, $file) {
return scalar(@ls);
}

sub _follow_link($self, $file) {
my ($dir, $name) = $file =~ m{(.*)/(.*)};
if (!defined $self->{_is_link}->{$dir} ) {
my ($out,$err) = $self->run_command("stat", $dir );
chomp $out;
$out =~ m{ -> (/.*)};
$self->{_is_link}->{$dir} = $1;
}
my $path = $self->{_is_link}->{$dir};
return $file if !$path;
return "$path/$name";

}

sub _wait_storage($self, $sub) {
my @ret;
for ( 1 .. 10 ) {
Expand Down Expand Up @@ -720,6 +703,14 @@ sub create_storage_pool($self, $name, $dir, $vm=$self->vm) {

}

sub remove_storage_pool($self, $name) {
my $sp = $self->vm->get_storage_pool_by_name($name);
return if !$sp;

$sp->destroy if $sp->is_active;
$sp->undefine();
}

sub _create_default_pool($self, $vm=$self->vm) {
my $dir = "/var/lib/libvirt/images";
mkdir $dir if ! -e $dir;
Expand Down
44 changes: 29 additions & 15 deletions lib/Ravada/VM/Void.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,7 @@ sub search_volume($self, $pattern) {
return;
}

sub _list_used_volumes($self) {
sub list_used_volumes($self) {
my @disk;
for my $domain ($self->list_domains) {
push @disk,($domain->list_disks());
Expand All @@ -363,32 +363,33 @@ sub _list_used_volumes($self) {
return @disk
}

sub _list_volumes($self) {
sub _list_volumes_sp($self, $sp) {
die "Error: TODO remote!" if !$self->is_local;

confess if !defined $sp;
my $dir = $sp->{path} or die "Error: unknown path ".Dumper($sp);
return if ! -e $dir;

my @vol;
opendir my $ls,$self->dir_img or die $!;
my $dir = $self->dir_img;

opendir my $ls,$dir or die "$! $dir";
while (my $file = readdir $ls) {
push @vol,("$dir/$file");
push @vol,("$dir/$file") if -f "$dir/$file";
}
closedir $ls;

return @vol;

}

sub list_unused_volumes($self) {
die "Error: TODO remote!" if !$self->is_local;
my %used = map { $_ => 1 } $self->_list_used_volumes();
my @unused;
for my $vol ( sort $self->_list_volumes ) {
next if ! -f $vol;
next if $vol =~ m{/\..*yml$};

push @unused,($vol) unless $used{$vol};
sub list_volumes($self) {
my @volumes;
for my $sp ($self->list_storage_pools(1)) {
for my $vol ( $self->_list_volumes_sp($sp) ) {
push @volumes,($vol);
}
}
return @unused;
return @volumes;
}

sub _search_volume_remote($self, $pattern) {
Expand Down Expand Up @@ -648,6 +649,19 @@ sub active_storage_pool($self, $name, $value) {
$self->write_file($file_sp, Dump( \@list));
}

sub remove_storage_pool($self, $name) {
die "TODO remote VM" unless $self->is_local;

my $file_sp = $self->dir_img."/.storage_pools.yml";
my $sp_list = LoadFile($file_sp);
my @sp2;
for my $sp (@$sp_list) {
push @sp2,($sp) if $sp->{name} ne $name;
}

$self->write_file($file_sp, Dump( \@sp2));
}

#########################################################################3

1;
27 changes: 18 additions & 9 deletions t/lib/Test/Ravada.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1023,7 +1023,12 @@ sub _activate_storage_pools($vm) {
next if $sp->is_active;
next unless $sp->get_name =~ /^tst_/;
diag("Activating sp ".$sp->get_name." on ".$vm->name);
$sp->build() unless $sp->is_active;
my $xml = XML::LibXML->load_xml(string => $sp->get_xml_description());
my ($path) = $xml->findnodes('/pool/target/path');
my $dir = $path->textContent();
mkdir $dir or die "$! $dir" if ! -e $dir;

$sp->build();
$sp->create() unless $sp->is_active;
}
}
Expand Down Expand Up @@ -1426,7 +1431,7 @@ sub _qemu_storage_pool {
}

sub remove_qemu_pools($vm=undef) {
return if !$VM_VALID{'KVM'} || $>;
return if !$vm && (!$VM_VALID{'KVM'} || $>);
return if defined $vm && $vm->type eq 'Void';
if (!defined $vm) {
eval { $vm = rvd_back->search_vm('KVM') };
Expand All @@ -1441,18 +1446,22 @@ sub remove_qemu_pools($vm=undef) {

my $base = base_pool_name();
$vm->connect();
for my $pool ( Ravada::VM::KVM::_list_storage_pools($vm->vm)) {
for my $pool ( $vm->vm->list_all_storage_pools) {
my $name = $pool->get_name;
next if $name !~ qr/^$base/;
diag($name);

eval {$pool->build(Sys::Virt::StoragePool::BUILD_NEW); $pool->create() };
warn $@ if $@ && $@ !~ /already active/;
next if $name !~ qr/^$base/;
diag("Removing ".$vm->name." storage_pool ".$pool->get_name);
for my $vol ( $pool->list_volumes ) {
diag("Removing ".$pool->get_name." vol ".$vol->get_name);
$vol->delete();
if ($pool->is_active) {
diag("Removing ".$vm->name." storage_pool ".$pool->get_name);
for my $vol ( $pool->list_volumes ) {
diag("Removing ".$pool->get_name." vol ".$vol->get_name);
$vol->delete();
}
}
_delete_qemu_pool($pool);
$pool->destroy();
$pool->destroy() if $pool->is_active;
eval { $pool->undefine() };
warn $@ if $@;
warn $@ if$@ && $@ !~ /libvirt error code: 49,/;
Expand Down
Loading
Loading