diff --git a/bin/ckan-webhooks b/bin/ckan-webhooks new file mode 100644 index 0000000..2970210 --- /dev/null +++ b/bin/ckan-webhooks @@ -0,0 +1,40 @@ +#!/usr/bin/env perl + +use v5.010; +use strict; +use warnings; +use Dancer2 appname => "xKanHooks"; +use File::Path qw(mkpath); + +# PODNAME: ckan-webhooks + +# ABSTRACT: ckan-webhooks - Small webserver for accepting webhooks + +# VERSION + +# TODO: Investigate config passing +$ENV{XKAN_GHSECRET} or die 'XKAN_GHSECRET needs to be set'; + +if ( -e "/tmp/xKan_netkan.lock" ) { + unlink "/tmp/xKan_netkan.lock"; +} + +# TODO: This could use a lot of improvement. +if (config->{environment} eq 'development') { + use lib 'lib/'; + set logger => "Console"; +} else { + if ( ! -d $ENV{HOME}."/CKAN-Webhooks" ) { + mkpath( $ENV{HOME}."/CKAN-Webhooks/" ); + } + + set appdir => $ENV{HOME}."/CKAN-Webhooks"; + set logger => "File"; + config->{logger}{log_level} = "info"; + config->{logger}{location} = config->{appdir}; +} + +set serializer => 'JSON'; + +use App::KSP_CKAN::WebHooks; +dance; diff --git a/bin/mirror-ckan b/bin/mirror-ckan new file mode 100644 index 0000000..2cf888c --- /dev/null +++ b/bin/mirror-ckan @@ -0,0 +1,94 @@ +#!/usr/bin/env perl + +use v5.010; +use strict; +use autodie qw(:all); +use App::KSP_CKAN::Tools::Config; +use App::KSP_CKAN::Mirror; +use Getopt::Long; +use File::Spec; + +# PODNAME: mirror-ckan + +# ABSTRACT: mirror-ckan - script for uploading ckan files to the mirror. + +# VERSION + +=head1 SYNOPSIS + +Usage: + + mirror-ckan --ckan /path/to/file.ckan : Takes a ckan file and mirrors it to the + : Internet Archive. + + Debugging commands: + + mirror-ckan --debug : Run with debugging enabled. + +=head1 Description + +This is a simple cli utility for uploading ckans to the Internet Archive. + +=head1 BUGS/Features Requests + +Please submit any bugs, feature requests to +L . + +Contributions are more than welcome! + +=head1 SEE ALSO + +L + +=cut + +my $PROGNAME = (File::Spec->splitpath($0))[2]; +$PROGNAME ||= 'mirror-ckan'; + +my $DEBUG = 0; +my $filename; + +# TODO: It'd be nice to specify a path/multiple files +my $getopts_rc = GetOptions( + "version" => \&version, + "debug!" => \$DEBUG, + "ckan=s" => \$filename, + + "help|?" => \&print_usage, +); + +# TODO: Allow config to be specified +my $config = App::KSP_CKAN::Tools::Config->new( + debugging => $DEBUG, +); +my $mirror = App::KSP_CKAN::Mirror->new( config => $config ); + +$mirror->upload_ckan($filename); + +exit 0; + +sub version { + $::VERSION ||= "Unreleased"; + say "$PROGNAME version : $::VERSION"; + exit 1; +} + +sub print_usage { + say q{ + Usage: + + mirror-ckan --ckan /path/to/file.ckan : Takes a ckan file and mirrors it to the + : Internet Archive. + + Debugging commands: + + mirror-ckan --debug : Run with debugging enabled. + mirror-ckan --version : Run with debugging enabled. + + For more documentation, use `perldoc mirror-ckan`. + }; + + exit 1; +} + +__END__ diff --git a/dist.ini b/dist.ini index 67634a4..ebbb222 100644 --- a/dist.ini +++ b/dist.ini @@ -15,6 +15,13 @@ repository.url = git://github.com/KSP-CKAN/NetKAN-bot repository.web = https://github.com/KSP-CKAN/NetKAN-bot repository.type = git +[Encoding] +encoding = bytes +match = zip + +[PruneFiles] +match = init/* + [@Basic] [Test::Perl::Critic] @@ -26,11 +33,29 @@ stopwords = netkan stopwords = api stopwords = KSP stopwords = BaconLabs +stopwords = CKANs +stopwords = CreateURLHash +stopwords = MirrorKAN +stopwords = NetFileCache +stopwords = ckans +stopwords = cli +stopwords = mediatype +stopwords = Webhook +stopwords = metapackage +stopwords = webhooks +stopwords = ckan +stopwords = iaS +stopwords = sha +stopwords = mimetype [AutoPrereqs] [Prereqs] +experimental = 0.013 IO::Socket::SSL = 0 +EV = 0 +Twiggy = 0 +LWP::Protocol::https = 0 [OurPkgVersion] [PodWeaver] diff --git a/init/ckan-webhooks b/init/ckan-webhooks new file mode 100755 index 0000000..695b037 --- /dev/null +++ b/init/ckan-webhooks @@ -0,0 +1,106 @@ +#! /bin/sh +# +### BEGIN INIT INFO +# Provides: ckan-webhooks +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: CKAN Webhooks +# Description: init script for CKAN Webhook Service. +### END INIT INFO + +USER=netkan +GROUP=netkan +PATH=/sbin:/bin:/usr/sbin:/usr/bin:/home/$USER/perl5/bin +NAME=ckan-webhooks +DESC="CKAN Webhooks" +PIDDIR=/var/run/$NAME +PIDFILE=$PIDDIR/$NAME.pid +WORKDIR=/home/$USER/CKAN-Webhooks +DAEOPTS="-E production -s Twiggy /home/$USER/perl5/bin/ckan-webhooks" +DAEMON=/home/$USER/perl5/bin/plackup + +# TODO: This could use improvement. +GH_SECRET=$WORKDIR/ghsecret +test -e $GH_SECRET || exit 0 +export XKAN_GHSECRET=`cat $GH_SECRET` + +test -x $DAEMON || exit 0 +export PERL5LIB=$PERL5LIB:/home/$USER/perl5/lib/perl5 + +case "$1" in + start) + echo -n "Starting $DESC ..." + [ -d $PIDDIR ] || install -o $USER -d $PIDDIR + start-stop-daemon --start --quiet \ + --pidfile $PIDFILE \ + --make-pidfile \ + --chuid $USER:$GROUP \ + --chdir $WORKDIR \ + --background \ + --exec $DAEMON -- $DAEOPTS + + case "$?" in + 0|1) echo "Started" ;; + 2) echo "Failed" ;; + esac + ;; + stop) + echo -n "Stopping $DESC ..." + start-stop-daemon --stop --quiet \ + --retry=TERM/30/KILL/5 \ + --pidfile $PIDFILE \ + --user $USER + case "$?" in + 0|1) rm -f $PIDFILE + echo "Stopped" + ;; + 2) echo "Failed" ;; + esac + ;; + status) + if start-stop-daemon --test --stop --quiet \ + --pidfile $PIDFILE \ + --user $USER + then + echo "$DESC is running." + exit 0 + else + echo "$DESC is not running" + exit 3 + fi + ;; + restart) + echo -n "Restarting $DESC ..." + start-stop-daemon --stop --quiet \ + --retry=TERM/30/KILL/5 \ + --pidfile $PIDFILE \ + --user $USER + case "$?" in + 0|1) + [ -d $PIDDIR ] || install -o $USER -d $PIDDIR + rm -f $PIDFILE + start-stop-daemon --start --quiet \ + --pidfile $PIDFILE \ + --make-pidfile \ + --chuid $USER:$GROUP \ + --chdir $WORKDIR \ + --background \ + --exec $DAEMON -- $DAEOPTS + case "$?" in + 0) echo "Restarted" ;; + *) echo "Start Failed" ;; + esac + ;; + *) + echo "Stop Failed" + ;; + esac + ;; + *) + N=/etc/init.d/$NAME + echo "Usage: $N {start|stop|restart|status}" >&2 + exit 3 + ;; +esac + +exit 0 diff --git a/lib/App/KSP_CKAN/Metadata/Ckan.pm b/lib/App/KSP_CKAN/Metadata/Ckan.pm new file mode 100644 index 0000000..052706d --- /dev/null +++ b/lib/App/KSP_CKAN/Metadata/Ckan.pm @@ -0,0 +1,304 @@ +package App::KSP_CKAN::Metadata::Ckan; + +use v5.010; +use strict; +use warnings; +use autodie; +use Method::Signatures 20140224; +use Config::JSON; # Saves us from file handling +use List::MoreUtils 'any'; +use Carp qw( croak ); +use Digest::SHA 'sha1_hex'; +use Scalar::Util 'reftype'; +use Moo; +use namespace::clean; + +# ABSTRACT: Metadata Wrapper for CKAN files + +# VERSION: Generated by DZP::OurPkg:Version + +=head1 SYNOPSIS + + use App::KSP_CKAN::Metadata::Ckan; + + my $ckan = App::KSP_CKAN::Metadata::Ckan->new( + file => "/path/to/file.ckan", + ); + +=head1 DESCRIPTION + +Provides a ckan metadata object for KSP-CKAN. Has the following +attributes available. + +=over + +=item identifier + +Returns the identifier for the loaded CKAN. + +=item abstract + +Returns the abstract for the loaded CKAN. + +=item author + +Returns the author field of the loaded CKAN. This could be a single +'author' or an array of 'authors'. + +=item authors + +Returns the author field of the loaded CKAN as an array regardless +of whether there is a single author or multiple. + +=item kind + +Returns the kind of CKAN. Default is 'package', but will return +'metapackage' for CKANs marked as such. + +=item download + +Returns the download url or 0 (in the case of a metapackage). + +=item download_sha1 + +Returns the download sha1 hash or 0 if not present. + +=item download_sha256 + +Returns the download sha256 hash or 0 if not present. + +=item download_content_type + +Returns the download content type or 0 if not present. + +=item homepage + +Returns the homepage url or 0. + +=item repository + +Returns the download url or 0. + +=back + +=cut + +has 'file' => ( is => 'ro', required => 1 ); # TODO: we should do some validation here. +has '_raw' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'identifier' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'authors' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'author' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'name' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'abstract' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'kind' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'download' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'download_sha1' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'download_sha256' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'download_content_type' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'homepage' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'repository' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'license' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'version' => ( is => 'ro', lazy => 1, builder => 1 ); + +# TODO: We're already using file slurper + JSON elsewhere. We should +# pick one method for consistency. +# TODO: This could also barf out on an invalid file, we'll need to +# Handle that somewhere. +method _build__raw { + return Config::JSON->new($self->file); +} + +method _build_identifier { + return $self->_raw->{config}{identifier}; +} + +method _build_kind { + return $self->_raw->{config}{kind} ? $self->_raw->{config}{kind} : 'package' ; +} + +method _build_version{ + return $self->_raw->{config}{version}; +} + +method _build_download { + return $self->_raw->{config}{download} ? $self->_raw->{config}{download} : 0; +} + +method _build_license { + return $self->_raw->{config}{license} ? $self->_raw->{config}{license} : "unknown"; +} + +method _build_download_sha1 { + return $self->_raw->{config}{download_hash}{sha1} ? $self->_raw->{config}{download_hash}{sha1} : 0; +} + +method _build_download_sha256 { + return $self->_raw->{config}{download_hash}{sha256} ? $self->_raw->{config}{download_hash}{sha256} : 0; +} + +method _build_download_content_type { + return $self->_raw->{config}{download_content_type} ? $self->_raw->{config}{download_content_type} : 0; +} + +method _build_authors { + my $authors = $self->_raw->{config}{author}; + my @authors = reftype \$authors ne "SCALAR" ? @{$authors} : $authors; + return \@authors; +} + +method _build_author { + return $self->_raw->{config}{author}; +} + +method _build_name { + return $self->_raw->{config}{name}; +} + +method _build_abstract { + return $self->_raw->{config}{abstract}; +} + +method _build_homepage { + return $self->_raw->{config}{resources}{homepage}; +} + +method _build_repository { + return $self->_raw->{config}{resources}{repository}; +} + +=method licenses + + $ckan->licenses(); + +Returns the license field as an array. Because unless there is +multiple values it won't be. + +=cut + +# Sometimes we always want an array. +method licenses { + my @licenses = reftype \$self->license ne "SCALAR" ? @{$self->license} : $self->license; + return \@licenses; +} + +=method redistributable + + $ckan->redistributable; + +Shortcut method so we can test that the CKAN is redistributable. Returns +'1' if it has a license which allows distribution, otherwise '0'. + +=cut + +method redistributable { + foreach my $license (@{$self->licenses}) { + if (any { $_ eq $license } @{$self->redistributable_licenses}) { + return 1; + } + } + return 0; +} + +=method is_metapackage + + $ckan->is_package; + +Shortcut method so we can test that the CKAN is a package. Returns +'1' if it is a package, otherwise '0'. + +=cut + +method is_package { + if ( $self->kind eq 'package' ) { + return 1; + } + return 0; +} + +=method can_mirror + + $ckan->can_mirror; + +Shortcut method for deciding if the license allows us to mirror it. +Returns '1' if allowed, '0' if not. + +=cut + +method can_mirror { + if ( ! $self->is_package ) { + return 0; + } elsif ( ! $self->redistributable ) { + return 0; + } elsif ( ! $self->extension($self->download_content_type) ) { + return 0; + } + return 1; +} + +=method url_hash + + $ckan->url hash; + +Produces a url hash in the same format as the 'NetFileCache.cs' +method 'CreateURLHash'. + +=cut + +method url_hash { + my $hash = sha1_hex($self->download); + $hash =~ s/-//g; + return uc(substr $hash, 0, 8); +} + +=method mirror_item + + $ckan->mirror_item; + +Produces an item name based of the 'identifier' and 'version'. + +=cut + +method mirror_item { + return $self->identifier."-".$self->version; +} + +=method mirror_filename + + $ckan->mirror_filename; + +Produces a filename based of the first 8 digits in sha1 hash, +the 'identifier' and the 'version' in the metadata if the +download_hash exists. Returns '0' if there is no download hash +or has an content type other than zip/gz/tar/tar.gz. + +=cut + +method mirror_filename { + if ( ! $self->download_sha1 ) { + return 0; + } elsif ( ! $self->extension($self->download_content_type) ) { + return 0; + } + return + substr($self->download_sha1,0,8)."-" + .$self->identifier."-" + .$self->version."." + .$self->extension($self->download_content_type); +} + +=method mirror_url + + $ckan->mirror_url' + +Produces a mirror url based of the 'identifier' and 'version'. + +=cut + +method mirror_url { + # TODO: Maybe not hardcode this. + return "https://archive.org/details/".$self->identifier."-".$self->version; +} + +with('App::KSP_CKAN::Roles::Licenses','App::KSP_CKAN::Roles::FileServices'); + +1; diff --git a/lib/App/KSP_CKAN/Mirror.pm b/lib/App/KSP_CKAN/Mirror.pm new file mode 100644 index 0000000..28f21ba --- /dev/null +++ b/lib/App/KSP_CKAN/Mirror.pm @@ -0,0 +1,162 @@ +package App::KSP_CKAN::Mirror; + +use v5.010; +use strict; +use warnings; +use autodie; +use Method::Signatures 20140224; +use Carp qw( croak ); +use List::MoreUtils 'first_index'; +use Digest::file qw(digest_file_hex); +use File::chdir; +use File::Basename qw(basename); +use File::Temp qw(tempdir); +use File::Path qw(remove_tree); +use File::Copy qw(copy); +use App::KSP_CKAN::Tools::IA; +use App::KSP_CKAN::Tools::Http; +use App::KSP_CKAN::Metadata::Ckan; +use Moo; +use namespace::clean; + +# ABSTRACT: MirrorKAN mirroring service + +# VERSION: Generated by DZP::OurPkg:Version + +=head1 SYNOPSIS + + use App::KSP_CKAN::Mirror; + + my $mirror = App::KSP_CKAN::Mirror->new( + config => $config, + ); + +=head1 DESCRIPTION + +Mirroring abstraction to be used by other libraries. + +=cut + +my $Ref = sub { + croak("auth isn't a 'App::KSP_CKAN::Tools::Config' object!") unless $_[0]->DOES("App::KSP_CKAN::Tools::Config"); +}; + +has 'config' => ( is => 'ro', required => 1, isa => $Ref ); +has '_ia' => ( is => 'ro', lazy => 1, builder => 1 ); +has '_http' => ( is => 'ro', lazy => 1, builder => 1 ); + +method _build__ia { + return App::KSP_CKAN::Tools::IA->new(config => $self->config); +} + +method _build__http { + return App::KSP_CKAN::Tools::Http->new(); +} + +method _tmp_dir { + return File::Temp::tempdir(); +} + +method _clean_tmp_dir($tmp) { + if ( -d $tmp ) { + remove_tree($tmp); + } + return; +} + +# TODO: What happens when we load an invalid file? +method _load_ckan($ckanfile) { + $self->logdie("Ckan file does not exist at '$ckanfile'") unless (-f $ckanfile); + return App::KSP_CKAN::Metadata::Ckan->new( file => $ckanfile ); +} + +# TODO: Could use some more logic here, maybe a while loop. +method _check_overload { + if ( $self->_ia->check_overload ) { + return 1; + } + return 0; +} + +method _check_cached($hash) { + my @files = glob($self->config->cache."/*"); + my $index = first_index { basename($_) =~ /^$hash/i } @files; + if ( $index != '-1' ) { + return $files[$index]; + } + return 0; +} + +method _check_file($file,$ckan) { + my $cached = $self->_check_cached($ckan->url_hash); + + if ( $cached eq 0 ) { + $self->info("Not found in cache, downloading"); + $self->_http->mirror( + url => $ckan->download, + path => $file, + ); + } else { + $self->info("Download found as $cached"); + copy($cached, $file) or $self->warn("Cache copy failed: $!"); + } + + if ( ! -f $file ) { + $self->warn("File was not able to be downloaded or copied"); + return 0; + } + + if ( $ckan->download_sha256 ne uc(digest_file_hex( $file, "SHA-256" )) ) { + $self->warn("SHA256 of file doesn't match metadata"); + return 0; + } + + return 1; +} + +# TODO: A lot of this logic should be moved into the IA class. We could +# very easily support multiple mirror backends. Potentially +# convert IA to a Role, or make this class exntendable with a +# required attribute of App::KSP_CKAN::Mirror::$Backend +method upload_ckan($ckanfile) { + my $ckan = $self->_load_ckan($ckanfile); + # TODO: Deaths are painful, we should avoid those. + $self->logdie("Ckan '".$ckan->mirror_item."' cannot be mirrored") unless $ckan->can_mirror; + my $tmp = $self->_tmp_dir; + my $file = $tmp."/".$ckan->mirror_filename; + + if ( $self->_check_overload ) { + # Let's clean ourselves up first. + $self->_clean_tmp_dir($tmp); + $self->warn("The Internet Archive is overloaded, try again later"); + return 0; + } + + if ( $self->_ia->ckan_mirrored( ckan => $ckan ) ) { + $self->info($ckan->mirror_item." Already has a file mirrrored with the SHA1 hash of ".$ckan->download_sha1); + return 1; + } + + my $result = 0; + + if ( $self->_check_file( $file, $ckan ) ) { + $result = $self->_ia->put_ckan( + ckan => $ckan, + file => $file, + ); + } + + if ( $result ) { + $self->info($ckan->mirror_item." mirrored to the Internet Archive successfully @ ".$ckan->mirror_url); + return 1; + } + + # TODO: This needs a test, we weren't cleaning our temp files. + $self->_clean_tmp_dir($tmp); + $self->warn($ckan->mirror_item." failed to mirror to the Internet Archive"); + return 0; +} + +with('App::KSP_CKAN::Roles::Logger'); + +1; diff --git a/lib/App/KSP_CKAN/NetKAN.pm b/lib/App/KSP_CKAN/NetKAN.pm index c77649c..70bd7c1 100644 --- a/lib/App/KSP_CKAN/NetKAN.pm +++ b/lib/App/KSP_CKAN/NetKAN.pm @@ -110,7 +110,7 @@ method _inflate_all(:$rescan = 1) { my $netkan = App::KSP_CKAN::Tools::NetKAN->new( config => $config, netkan => $config->working."/netkan.exe", - cache => $config->working."/cache", + cache => $config->cache, token => $config->GH_token, file => $file, ckan_meta => $self->_CKAN_meta, diff --git a/lib/App/KSP_CKAN/Roles/FileServices.pm b/lib/App/KSP_CKAN/Roles/FileServices.pm new file mode 100644 index 0000000..2bfaf3d --- /dev/null +++ b/lib/App/KSP_CKAN/Roles/FileServices.pm @@ -0,0 +1,45 @@ +package App::KSP_CKAN::Roles::FileServices; + +use v5.010; +use strict; +use warnings; +use autodie; +use Method::Signatures 20140224; +use Moo::Role; +use experimental 'switch'; + +# ABSTRACT: File services role for consuming + +# VERSION: Generated by DZP::OurPkg:Version + +=head1 SYNOPSIS + + with('App::KSP_CKAN::Roles::FileServices'); + +=head1 DESCRIPTION + +Provides various file operations. + +extensions. + +=cut + +=method extension + + $self->extension( "application/zip" ); + +Returns a file extension for a given valid upload mimetype. + +=cut + +method extension($mimetype) { + given ( $mimetype ) { + when ( "application/x-gzip" ) { return "gz"; } + when ( "application/x-tar" ) { return "tar"; } + when ( "application/x-compressed-tar" ) { return "tar.gz"; } + when ( "application/zip" ) { return "zip"; } + } + return 0; +} + +1; diff --git a/lib/App/KSP_CKAN/Roles/Licenses.pm b/lib/App/KSP_CKAN/Roles/Licenses.pm new file mode 100644 index 0000000..4d302ba --- /dev/null +++ b/lib/App/KSP_CKAN/Roles/Licenses.pm @@ -0,0 +1,167 @@ +package App::KSP_CKAN::Roles::Licenses; + +use v5.010; +use strict; +use warnings; +use autodie; +use Method::Signatures 20140224; +use Moo::Role; + +# ABSTRACT: At CKAN we care about licenses + +# VERSION: Generated by DZP::OurPkg:Version + +=head1 SYNOPSIS + + with('App::KSP_CKAN::Roles::Licenses'); + +=head1 DESCRIPTION + +We care about licenses deeply. It helps us make decisions about +what we can and can't do with hosted mods. Such as mirroring. + +=cut + +has '_license_urls' => ( is => 'ro', lazy => 1, builder => 1 ); + +# NOTE: Did look at a couple of libs to do this, but nothing quite fit +# without building a translation table anyway. +# TODO: Do we consider no version number for a version to be the first +# version of the relevant license or the latest? +method _build__license_urls { + return { + "Apache" => 'http://www.apache.org/licenses/LICENSE-1.0', + "Apache-1.0" => 'http://www.apache.org/licenses/LICENSE-1.0', + "Apache-2.0" => 'http://www.apache.org/licenses/LICENSE-2.0', + "Artistic" => 'http://www.gnu.org/licenses/license-list.en.html#ArtisticLicense', + "Artistic-1.0" => 'http://www.gnu.org/licenses/license-list.en.html#ArtisticLicense', + "Artistic-2.0" => 'http://www.perlfoundation.org/artistic_license_2_0', + "BSD-2-clause" => 'https://opensource.org/licenses/BSD-2-Clause', + "BSD-3-clause" => 'https://opensource.org/licenses/BSD-3-Clause', +# "BSD-4-clause" => '', # TODO: Clarify this + "ISC" => 'https://opensource.org/licenses/ISC', + "CC-BY" => 'https://creativecommons.org/licenses/by/1.0/', + "CC-BY-1.0" => 'https://creativecommons.org/licenses/by/1.0/', + "CC-BY-2.0" => 'https://creativecommons.org/licenses/by/2.0/', + "CC-BY-2.5" => 'https://creativecommons.org/licenses/by/2.5/', + "CC-BY-3.0" => 'https://creativecommons.org/licenses/by/3.0/', + "CC-BY-4.0" => 'https://creativecommons.org/licenses/by/4.0/', + "CC-BY-SA" => 'https://creativecommons.org/licenses/by-sa/1.0/', + "CC-BY-SA-1.0" => 'https://creativecommons.org/licenses/by-sa/1.0/', + "CC-BY-SA-2.0" => 'https://creativecommons.org/licenses/by-sa/2.0/', + "CC-BY-SA-2.5" => 'https://creativecommons.org/licenses/by-sa/2.5/', + "CC-BY-SA-3.0" => 'https://creativecommons.org/licenses/by-sa/3.0/', + "CC-BY-SA-4.0" => 'https://creativecommons.org/licenses/by-sa/4.0/', + "CC-BY-NC" => 'https://creativecommons.org/licenses/by-nc/1.0/', + "CC-BY-NC-1.0" => 'https://creativecommons.org/licenses/by-nc/1.0/', + "CC-BY-NC-2.0" => 'https://creativecommons.org/licenses/by-nc/2.0/', + "CC-BY-NC-2.5" => 'https://creativecommons.org/licenses/by-nc/2.5/', + "CC-BY-NC-3.0" => 'https://creativecommons.org/licenses/by-nc/3.0/', + "CC-BY-NC-4.0" => 'https://creativecommons.org/licenses/by-nc/4.0/', + "CC-BY-NC-SA" => 'http://creativecommons.org/licenses/by-nc-sa/1.0/', + "CC-BY-NC-SA-1.0" => 'http://creativecommons.org/licenses/by-nc-sa/1.0', + "CC-BY-NC-SA-2.0" => 'http://creativecommons.org/licenses/by-nc-sa/2.0', + "CC-BY-NC-SA-2.5" => 'http://creativecommons.org/licenses/by-nc-sa/2.5', + "CC-BY-NC-SA-3.0" => 'http://creativecommons.org/licenses/by-nc-sa/3.0', + "CC-BY-NC-SA-4.0" => 'http://creativecommons.org/licenses/by-nc-sa/4.0', + "CC-BY-NC-ND" => 'https://creativecommons.org/licenses/by-nd-nc/1.0/', + "CC-BY-NC-ND-1.0" => 'https://creativecommons.org/licenses/by-nd-nc/1.0/', + "CC-BY-NC-ND-2.0" => 'https://creativecommons.org/licenses/by-nd-nc/2.0/', + "CC-BY-NC-ND-2.5" => 'https://creativecommons.org/licenses/by-nd-nc/2.5/', + "CC-BY-NC-ND-3.0" => 'https://creativecommons.org/licenses/by-nd-nc/3.0/', + "CC-BY-NC-ND-4.0" => 'https://creativecommons.org/licenses/by-nd-nc/4.0/', + "CC0" => 'https://creativecommons.org/publicdomain/zero/1.0/', + "CDDL" => 'https://opensource.org/licenses/CDDL-1.0', + "CPL" => 'https://opensource.org/licenses/cpl1.0.php', + "EFL-1.0" => 'https://opensource.org/licenses/ver1_eiffel', + "EFL-2.0" => 'https://opensource.org/licenses/EFL-2.0', + "Expat" => 'https://opensource.org/licenses/MIT', # https://en.wikipedia.org/wiki/MIT_License + "MIT" => 'https://opensource.org/licenses/MIT', + "GPL-1.0" => 'http://www.gnu.org/licenses/old-licenses/gpl-1.0.en.html', + "GPL-2.0" => 'http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html', + "GPL-3.0" => 'http://www.gnu.org/licenses/gpl-3.0.en.html', + "LGPL-2.0" => 'https://www.gnu.org/licenses/old-licenses/lgpl-2.0.html', + "LGPL-2.1" => 'https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html', + "LGPL-3.0" => 'http://www.gnu.org/licenses/lgpl-3.0.en.html', +# "GFDL-1.0" => '', # TODO: Doesn't appear to exist. + "GFDL-1.1" => 'http://www.gnu.org/licenses/old-licenses/fdl-1.1.en.html', + "GFDL-1.2" => 'http://www.gnu.org/licenses/old-licenses/fdl-1.2.html', + "GFDL-1.3" => 'http://www.gnu.org/licenses/fdl-1.3.en.html', +# "GFDL-NIV-1.0" => '', # TODO: Can't seem to find links to NIV, aside +# "GFDL-NIV-1.1" => '', # from it referred to in the debian spec. +# "GFDL-NIV-1.2" => '', +# "GFDL-NIV-1.3" => '', + "LPPL-1.0" => 'https://latex-project.org/lppl/lppl-1-0.html', + "LPPL-1.1" => 'https://latex-project.org/lppl/lppl-1-1.html', + "LPPL-1.2" => 'https://latex-project.org/lppl/lppl-1-2.html', + "LPPL-1.3c" => 'https://latex-project.org/lppl/lppl-1-3c.html', + "MPL-1.1" => 'https://www.mozilla.org/en-US/MPL/1.1/', + "Perl" => 'http://dev.perl.org/licenses/', + "Python-2.0" => 'https://www.python.org/download/releases/2.0/license/', + "QPL-1.0" => 'https://opensource.org/licenses/QPL-1.0', + "W3C" => 'https://www.w3.org/Consortium/Legal/2015/copyright-software-and-document', + "Zlib" => 'http://www.zlib.net/zlib_license.html', + "Zope" => 'http://old.zope.org/Resources/License.1', + } +} + +=method redistributable_licenses + + my @licenses = $self->redistributable_licenses; + +Returns an array of open source licenses which allow redistribution. + +=cut + +# This is an array of explicit licenses which are allowed to be mirrored. +# TODO: Maybe we can consume this from somewhere externally. +method redistributable_licenses { + return [ + "public-domain", + "Apache", "Apache-1.0", "Apache-2.0", + "Artistic", "Artistic-1.0", "Artistic-2.0", + "BSD-2-clause", "BSD-3-clause", "BSD-4-clause", + "ISC", + "CC-BY", "CC-BY-1.0", "CC-BY-2.0", "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0", + "CC-BY-SA", "CC-BY-SA-1.0", "CC-BY-SA-2.0", "CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0", + "CC-BY-NC", "CC-BY-NC-1.0", "CC-BY-NC-2.0", "CC-BY-NC-2.5", "CC-BY-NC-3.0", "CC-BY-NC-4.0", + "CC-BY-NC-SA", "CC-BY-NC-SA-1.0", "CC-BY-NC-SA-2.0", "CC-BY-NC-SA-2.5", "CC-BY-NC-SA-3.0", "CC-BY-NC-SA-4.0", + "CC-BY-NC-ND", "CC-BY-NC-ND-1.0", "CC-BY-NC-ND-2.0", "CC-BY-NC-ND-2.5", "CC-BY-NC-ND-3.0", "CC-BY-NC-ND-4.0", + "CC0", + "CDDL", "CPL", + "EFL-1.0", "EFL-2.0", + "Expat", "MIT", + "GPL-1.0", "GPL-2.0", "GPL-3.0", + "LGPL-2.0", "LGPL-2.1", "LGPL-3.0", + "GFDL-1.0", "GFDL-1.1", "GFDL-1.2", "GFDL-1.3", + "GFDL-NIV-1.0", "GFDL-NIV-1.1", "GFDL-NIV-1.2", "GFDL-NIV-1.3", + "LPPL-1.0", "LPPL-1.1", "LPPL-1.2", "LPPL-1.3c", + "MPL-1.1", + "Perl", + "Python-2.0", + "QPL-1.0", + "W3C", + "Zlib", + "Zope", + "WTFPL", + "open-source", "unrestricted" ]; +} + +=metho has_license_url + + $self->license_url("GPL-2.0"); + +We may not have all the URLs for licenses initially. In the fullness +of time this will just prevent errors for new licenses being added +to the spec. Returns a url when it is is known and 0 for no license url. + +=cut + +method license_url($license) { + if ( defined $self->_license_urls->{$license} ) { + return $self->_license_urls->{$license}; + } + return 0; +} + +1; diff --git a/lib/App/KSP_CKAN/Tools/Config.pm b/lib/App/KSP_CKAN/Tools/Config.pm index 70a1ebb..51b980b 100644 --- a/lib/App/KSP_CKAN/Tools/Config.pm +++ b/lib/App/KSP_CKAN/Tools/Config.pm @@ -37,6 +37,10 @@ has 'netkan_exe' => ( is => 'ro', lazy => 1, builder => 1 ); has 'ckan_validate' => ( is => 'ro', lazy => 1, builder => 1 ); has 'ckan_schema' => ( is => 'ro', lazy => 1, builder => 1 ); has 'GH_token' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'IA_access' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'IA_secret' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'IA_collection' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'cache' => ( is => 'ro', lazy => 1, builder => 1 ); has 'working' => ( is => 'ro', lazy => 1, builder => 1 ); has 'debugging' => ( is => 'ro', default => sub { 0 } ); @@ -72,6 +76,20 @@ method _build_ckan_schema { return $self->_config->{_}{'ckan_schema'}; } +method _build_IA_access { + croak( "Missing 'IA_access' from config" ) if ! $self->_config->{_}{'IA_access'}; + return $self->_config->{_}{'IA_access'}; +} + +method _build_IA_secret { + croak( "Missing 'IA_secret' from config" ) if ! $self->_config->{_}{'IA_secret'}; + return $self->_config->{_}{'IA_secret'}; +} + +method _build_IA_collection { + return $self->_config->{_}{'IA_collection'} ? $self->_config->{_}{'IA_collection'} : "test_collection"; +} + method _build_GH_token { if ( ! $self->_config->{_}{'GH_token'} ) { return 0; @@ -80,6 +98,20 @@ method _build_GH_token { } } +method _build_cache { + my $cache; + if ( ! $self->_config->{_}{'cache'} ) { + $cache = $self->working."/cache"; + } else { + $cache = $self->_config->{_}{'cache'}; + } + + if ( ! -d $cache ) { + mkpath($cache); + } + return $cache; +} + method _build_working { my $working; if ( ! $self->_config->{_}{'working'} ) { diff --git a/lib/App/KSP_CKAN/Tools/Git.pm b/lib/App/KSP_CKAN/Tools/Git.pm index 17343fe..1922291 100644 --- a/lib/App/KSP_CKAN/Tools/Git.pm +++ b/lib/App/KSP_CKAN/Tools/Git.pm @@ -76,14 +76,14 @@ method _build__git { mkpath($self->local); } - if ($self->clean) { - $self->_clean; - } - if ( ! -d $self->local."/".$self->working ) { $self->_clone; } + if ($self->clean) { + $self->_clean; + } + return Git::Wrapper->new({ dir => $self->local."/".$self->working, }); @@ -108,11 +108,13 @@ method _clone { } method _clean { - local $CWD = $self->local; - if ( -d $self->working) { - remove_tree($self->working); - } - return; + # TODO: We could fail here too, we should return as such. + # NOTE: We've not instantiated a git object at this point, so + # we can't use it. + local $CWD = $self->local."/".$self->working; + capture { system("git", "reset", "--hard", "HEAD") }; + capture { system("git", "clean", "-df") }; + } method _build_branch { diff --git a/lib/App/KSP_CKAN/Tools/IA.pm b/lib/App/KSP_CKAN/Tools/IA.pm new file mode 100644 index 0000000..eced2f0 --- /dev/null +++ b/lib/App/KSP_CKAN/Tools/IA.pm @@ -0,0 +1,277 @@ +package App::KSP_CKAN::Tools::IA; + +use v5.010; +use strict; +use warnings; +use autodie; +use Method::Signatures 20140224; +use LWP::UserAgent; +use File::MimeInfo::Magic 'mimetype'; +use Scalar::Util 'reftype'; +use List::MoreUtils; +use HTTP::Request::StreamingUpload; +use Encode; +use JSON 'from_json'; +use Try::Tiny; +use List::MoreUtils 'any'; +use Moo; +use namespace::clean; + +# ABSTRACT: An abstraction to the Internet Archive S3 like interface + +# VERSION: Generated by DZP::OurPkg:Version + +=head1 SYNOPSIS + + use App::KSP_CKAN::Tools::IA; + + my $IA = App::KSP_CKAN::Tools::IA->new( config => $config ); + +=head1 DESCRIPTION + +Provides a light wrapper to the Internet Archives' S3 like interface. + +Takes the following named attributes: + +=over + +=item config + +Requires a 'App::KSP_CKAN::Tools::Config' object. + +=item collection + +Defaults to 'test_collection'. + +=item mediatype + +Defaults to 'software' + +=item iaS3uri + +Defaults to 'https://s3.us.archive.org'. + +=back + +=cut + +my $Ref = sub { + croak("auth isn't a 'App::KSP_CKAN::Tools::Config' object!") unless $_[0]->DOES("App::KSP_CKAN::Tools::Config"); +}; + +has 'config' => ( is => 'ro', required => 1, isa => $Ref ); +has 'mediatype' => ( is => 'ro', default => sub { 'software' } ); +has 'iaS3uri' => ( is => 'ro', default => sub { 'https://s3.us.archive.org' } ); +has 'iaDLuri' => ( is => 'ro', default => sub { 'https://www.archive.org/download' } ); +has 'iaMDuri' => ( is => 'ro', default => sub { 'https://www.archive.org/metadata' } ); +has 'collection' => ( is => 'ro', lazy => 1, builder => 1 ); +has '_ua' => ( is => 'rw', lazy => 1, builder => 1 ); +has '_ias3keys' => ( is => 'ro', lazy => 1, builder => 1 ); + +method _build__ua { + my $ua = LWP::UserAgent->new(); + $ua->agent('upload_via_KSP-CKAN/NetKAN-bot'); + $ua->timeout(60); + $ua->env_proxy; + $ua->default_headers->push_header( 'authorization' => "LOW ". $self->_ias3keys); + return $ua; +} + +method _build__ias3keys { + my $config = $self->config; + return $config->IA_access.":".$config->IA_secret; +} + +method _build_collection { + return $self->config->IA_collection; +} + +method _upload_uri($ckan) { + $self->logdie("\$ckan isn't a 'App::KSP_CKAN::Metadata::Ckan' object!") unless $ckan->DOES("App::KSP_CKAN::Metadata::Ckan"); + return $self->iaS3uri."/".$ckan->mirror_item."/".$ckan->mirror_filename; +} + +method _download_uri($ckan) { + $self->logdie("\$ckan isn't a 'App::KSP_CKAN::Metadata::Ckan' object!") unless $ckan->DOES("App::KSP_CKAN::Metadata::Ckan"); + return $self->iaDLuri."/".$ckan->mirror_item."/".$ckan->mirror_filename; +} + +method _metadata_uri($ckan) { + $self->logdie("\$ckan isn't a 'App::KSP_CKAN::Metadata::Ckan' object!") unless $ckan->DOES("App::KSP_CKAN::Metadata::Ckan"); + return $self->iaMDuri."/".$ckan->mirror_item; +} + +# TODO: Likely makes sense to be part of the Ckan lib. +method _description($ckan) { + my $description = $ckan->abstract; + $description .= "

Homepage: homepage."\">".$ckan->homepage."" if $ckan->homepage; + $description .= "
Repository: repository."\">".$ckan->repository."" if $ckan->repository; + $description .= "
License(s): @{$ckan->licenses}" if $ckan->license; + return $description; +} + +method _archive_header( $header, $value ) { + # Credit for logic to: https://github.com/kngenie/ias3upload + if (reftype \$value ne "SCALAR") { + if ($#{$value} == 0) { + return ('x-archive-meta-' . $header, encode('UTF-8', $value->[0])); + } else { + my $i = 1; + return map((sprintf('x-archive-meta%02d-%s', $i++, $header), encode('UTF-8',$_)), @{$value}); + } + } else { + return ('x-archive-meta-' . $header, encode('UTF-8',$value)); + } +} + +method _metadata_headers ( $file, $ckan ) { + $self->logdie("\$ckan isn't a 'App::KSP_CKAN::Metadata::Ckan' object!") unless $ckan->DOES("App::KSP_CKAN::Metadata::Ckan"); + my $mimetype = mimetype( $file ); + + $self->logdie("Content type doesn't match metadata") + unless ( $mimetype eq $ckan->download_content_type ); + + my $headers = HTTP::Headers->new( + 'Content-Type' => $mimetype, + 'Content-Length' => -s $file, + $self->_archive_header('collection', $self->collection), + $self->_archive_header('creator', \@{$ckan->authors}), + $self->_archive_header('subject', "ksp; kerbal space program; mod"), + $self->_archive_header('title', $ckan->name." - ".$ckan->version), + $self->_archive_header('description', $self->_description($ckan)), + $self->_archive_header('mediatype', $self->mediatype), + ); + + # TODO: oh gosh, this looks more complicated than it needs to be. + # Note: It's like this because we need to increment the headers with a + # number, we might have multipel licenses we may not have a + # licenses url for all of them. + my @urls; + foreach my $license (@{$ckan->licenses}) { + push(@urls, $self->license_url($license)) if $self->license_url($license); + } + my @url_headers; + @url_headers = $self->_archive_header('licenseurl', \@urls) if $urls[0]; + $headers->push_header(@url_headers) if $url_headers[0]; + + return $headers; +} + +# NOTE: We're using StreamingUpload here, because LWP likes to +# pull the entire file into memory when uploading. +method _put_request( :$headers, :$uri, :$file) { + return HTTP::Request::StreamingUpload->new( + PUT => $uri, + path => $file, + headers => $headers, + ); +} + +=method check_overload + + $ia->check_overload; + +Checks if the submission servers are overloaded or we've reached +an API limit. Returns '1' if overloaded, '0' if not. + +=cut + +method check_overload { + my $res = $self->_ua->get($self->iaS3uri."/?check_limit=1&accesskey=".$self->config->IA_access); + + if ( $res->{_rc} == '503' ) { + $self->warn("Internet archive overloaded"); + return 1; + } + return 0; +} + +=method put_ckan + + $ia->( + ckan => $ckan, + file => "/path/to/mod.zip", + ); + +Takes a ckan object and file, then puts it on the Internet Archive. +Returns '1' on success, '0' on failure. + +Requires the following named attributes: +=over + +=item ckan + +Requires a 'App::KSP_CKAN::Metadata::Ckan' object. + +=item file + +Path to the file being uploaded. + +=back + +=cut + +method put_ckan( :$ckan, :$file ) { + $self->logdie("\$ckan isn't a 'App::KSP_CKAN::Metadata::Ckan' object!") + unless $ckan->DOES("App::KSP_CKAN::Metadata::Ckan"); + + my $headers = $self->_metadata_headers( $file, $ckan ); + $headers->push_header('x-amz-auto-make-bucket', 1); + + my $request = $self->_put_request( + headers => $headers, + uri => $self->_upload_uri( $ckan ), + file => $file, + ); + + my $res = $self->_ua->request($request); + + if ($res->is_success) { + return 1; + } + + $self->warn("Put failed: ".$res->message) if $res->is_error; + + return 0; +} + +=method ckan_mirrored + + $ia->ckan_mirrored( ckan => $ckan ); + +Requires a 'App::KSP_CKAN::Metadata::Ckan' object within the 'ckan' +attribute. Returns '1' if mirrored, Otherwise '0' if the archive +can't be contacted or no file is found. + +=cut + +# TODO: Not 100% happy with our returns here. I've seen spotty +# responses from the IA at times. +method ckan_mirrored( :$ckan ) { + $self->logdie("\$ckan isn't a 'App::KSP_CKAN::Metadata::Ckan' object!") + unless $ckan->DOES("App::KSP_CKAN::Metadata::Ckan"); + + my $res = $self->_ua->get($self->_metadata_uri($ckan)); + + if ( $res->is_error ) { + $self->warn($res->message); + return 0; + } + + my $data; + try { + $data = from_json($res->decoded_content); + } catch { + $self->logdie("Metadata JSON failed to parse"); + }; + + if ( any { uc($_->{sha1}) eq $ckan->download_sha1 } @{$data->{files}} ) { + return 1; + } + + return 0; +} + +with('App::KSP_CKAN::Roles::Logger', 'App::KSP_CKAN::Roles::Licenses'); + +1; diff --git a/lib/App/KSP_CKAN/WebHooks.pm b/lib/App/KSP_CKAN/WebHooks.pm new file mode 100644 index 0000000..34b9604 --- /dev/null +++ b/lib/App/KSP_CKAN/WebHooks.pm @@ -0,0 +1,210 @@ +package App::KSP_CKAN::WebHooks; + +use Dancer2 appname => "xKanHooks"; +use App::KSP_CKAN::WebHooks::InflateNetKAN; +use App::KSP_CKAN::WebHooks::MirrorCKAN; +use Method::Signatures 20140224; +use Digest::SHA qw(hmac_sha1_hex); +use File::Basename 'basename'; +use List::MoreUtils 'none'; +use AnyEvent::Util; +use Try::Tiny; +use File::Touch; + +# ABSTRACT: Webhook Routes for ckan-webhooks + +# VERSION: Generated by DZP::OurPkg:Version + +############## +### Routes ### +############## + +post '/inflate' => sub { + my @identifiers; + + try { + @identifiers = @{from_json(request->body)->{identifiers}}; + }; + + if ($#identifiers == -1) { + info("No identifiers received"); + send_error "An array of identifiers is required", 400; + } + + inflate_netkans(identifiers => \@identifiers); + + status(204); + return; +}; + +post '/gh/:task' => sub { + my $signature = request->header('x-hub-signature'); + my $body = request->content; + my $task = params->{task}; + + if ( ! defined $signature ) { + send_error("Post header 'x-hub-signature required'", 400); + } + + if ( ! $body ) { + send_error("post content required", 400); + } + + if ( $signature ne calc_gh_signature( body => $body) ) { + send_error("Signature mismatch", 400); + } + + my @commits; + try { + @commits = @{from_json(request->body)->{commits}}; + }; + + if ($#commits == -1) { + info("No commits received"); + return { "message" => "No add/remove commits received" }; + } + + if ( $task eq "inflate" ) { + inflate_github(commits => \@commits); + } elsif ( $task eq "mirror" ) { + mirror_github(commits => \@commits); + } else { + send_error "Unknown task '".$task."', accepted tasks are 'inflate' and 'mirror'", 400; + } + + status(204); + return; +}; + +###################### +### Mirror Methods ### +###################### + +method mirror_github($commits) { + my @files; + foreach my $commit (@{$commits}) { + push(@files, (@{$commit->{added}},@{$commit->{modified}})); + } + + if ($#files == -1) { + info("Nothing add/modified"); + return; + } + + my @ckans; + foreach my $file (@files) { + # Lets only try to only mirror actual ckans. + # Also only do each one once + if ($file =~ /\.ckan$/ && (none { $_ eq $file } @ckans)) { + push(@ckans, $file); + } + } + + if ($#ckans == -1) { + info("No ckans found in file list"); + return; + } + + mirror_ckans(identifiers => \@ckans); +} + +method mirror_ckans($ckans) { + fork_call { + my $mirror = App::KSP_CKAN::WebHooks::MirrorCKAN->new(); + + while (-e "/tmp/xKan_mirror.lock" ) { + debug("Waiting for lock release"); + sleep 5; + } + + # TODO: Do something better, this doesn't handle stale + # locks at all. Also if following requests come in + # at exactly 5 seconds apart we could still fork + # twice simultaneously. + debug("Locking environment"); + touch("/tmp/xKan_mirror.lock"); + + info("Mirroring: ".join(", ", @{$ckans})); + $mirror->mirror(\@{$ckans}); + info("Completed: ".join(", ", @{$ckans})); + + return; + } sub { + debug("Unlocking environment"); + unlink("/tmp/xKan_mirror.lock"); + return; + }; + return; +} + +####################### +### Inflate Methods ### +####################### + +method inflate_github($commits) { + my @files; + foreach my $commit (@{$commits}) { + push(@files, (@{$commit->{added}},@{$commit->{modified}})); + } + + if ($#files == -1) { + info("Nothing add/modified"); + return; + } + + my @netkans; + foreach my $file (@files) { + # Lets only try to send NetKAN actual netkans. + # Also only do each one once + my $netkan = basename($file,".netkan"); + if ($file =~ /\.netkan$/ && (none { $_ eq $netkan } @netkans)) { + push(@netkans, basename($file,".netkan")); + } + } + + if ($#netkans == -1) { + info("No netkans found in file list"); + return; + } + + inflate_netkans(identifiers => \@netkans); +} + +method inflate_netkans($identifiers) { + fork_call { + my $inflater = App::KSP_CKAN::WebHooks::InflateNetKAN->new(); + + while (-e "/tmp/xKan_netkan.lock" ) { + debug("Waiting for lock release"); + sleep 5; + } + + # TODO: Do something better, this doesn't handle stale + # locks at all. Also if following requests come in + # at exactly 5 seconds apart we could still fork + # twice simultaneously. + debug("Locking environment"); + touch("/tmp/xKan_netkan.lock"); + + info("Inflating: ".join(", ", @{$identifiers})); + $inflater->inflate(\@{$identifiers}); + info("Completed: ".join(", ", @{$identifiers})); + + return; + } sub { + debug("Unlocking environment"); + unlink("/tmp/xKan_netkan.lock"); + return; + }; + return; +} + +######################## +### Shortcut Methods ### +######################## + +method calc_gh_signature($body) { + return 'sha1='.hmac_sha1_hex($body, $ENV{XKAN_GHSECRET}); +} + +1; diff --git a/lib/App/KSP_CKAN/WebHooks/InflateNetKAN.pm b/lib/App/KSP_CKAN/WebHooks/InflateNetKAN.pm new file mode 100644 index 0000000..e5f77ae --- /dev/null +++ b/lib/App/KSP_CKAN/WebHooks/InflateNetKAN.pm @@ -0,0 +1,90 @@ +package App::KSP_CKAN::WebHooks::InflateNetKAN; + +use v5.010; +use strict; +use warnings; +use autodie; +use Method::Signatures 20140224; +use File::chdir; +use File::Path qw( mkpath ); +use Scalar::Util 'reftype'; +use App::KSP_CKAN::Tools::Config; +use Moo; +use namespace::clean; + +extends 'App::KSP_CKAN::NetKAN'; + +# ABSTRACT: NetKAN Index on demand + +# VERSION: Generated by DZP::OurPkg:Version + +=head1 SYNOPSIS + + use App::KSP_CKAN::WebHooks::InflateNetKAN 'inflate'; + + inflate("Netkan.netkan"); + +=head1 DESCRIPTION + +Webhook wrapper for NetKAN inflation on demand. + +=cut + +has 'config' => ( is => 'ro', lazy => 1, builder => 1 ); + +# TODO: This is a hack, the application should be multi +# function aware. +method _build_config { + my $working = $ENV{HOME}."/CKAN-Webhooks/inflator"; + if ( ! -d $working ) { + mkpath($working); + } + return App::KSP_CKAN::Tools::Config->new( + working => $working, + ); +} + +method inflate($identifiers) { + # Lets take an array as well! + my @identifiers = reftype \$identifiers ne "SCALAR" ? @{$identifiers} : $identifiers; + + # Prepare Enironment + $self->_mirror_files; + $self->_CKAN_meta->pull; + $self->_NetKAN->pull; + local $CWD = $self->config->working."/".$self->_NetKAN->working; + + foreach my $identifier (@identifiers) { + my $file = "NetKAN/".$identifier.".netkan"; + + if (! -e $file) { + $self->warn("The identifier '".$identifier."' doesn't appear to exist"); + next; + } + + # TODO: We're already passing in the config, is it really + # necessary to pass in each of the other things as attributes? + my $config = $self->config; + my $netkan = App::KSP_CKAN::Tools::NetKAN->new( + config => $config, + netkan => $config->working."/netkan.exe", + cache => $config->cache, + token => $config->GH_token, + file => $file, + ckan_meta => $self->_CKAN_meta, + status => $self->_status, + rescan => 1, + ); + $netkan->inflate; + } + + $self->_push; + + # TODO: Status won't be set for inflate on demand because currently + # if the a NetKAN is not present in the status object it'll + # drop from the JSON output. So for individual inflation we'll + # end up with with 1 item in the status. + return 1; +} + +1; diff --git a/lib/App/KSP_CKAN/WebHooks/MirrorCKAN.pm b/lib/App/KSP_CKAN/WebHooks/MirrorCKAN.pm new file mode 100644 index 0000000..cf3ee21 --- /dev/null +++ b/lib/App/KSP_CKAN/WebHooks/MirrorCKAN.pm @@ -0,0 +1,82 @@ +package App::KSP_CKAN::WebHooks::MirrorCKAN; + +use v5.010; +use strict; +use warnings; +use autodie; +use Method::Signatures 20140224; +use File::chdir; +use File::Path qw( mkpath ); +use Scalar::Util 'reftype'; +use Try::Tiny; +use App::KSP_CKAN::Tools::Config; +use Moo; +use namespace::clean; + +extends 'App::KSP_CKAN::Mirror'; + +# ABSTRACT: CKAN Mirror on demand + +# VERSION: Generated by DZP::OurPkg:Version + +=head1 SYNOPSIS + + use App::KSP_CKAN::WebHooks::MirrorCKAN; + + my $mirror = App::KSP_CKAN::WebHooks::MirrorCKAN->new(); + $mirror->mirror("/path/to/ckan"); + +=head1 DESCRIPTION + +Webhook wrapper for Mirror CKAN on demand. + +=cut + +has 'config' => ( is => 'ro', lazy => 1, builder => 1 ); +has '_CKAN_meta' => ( is => 'ro', lazy => 1, builder => 1 ); + +# TODO: This is a hack, the application should be multi +# function aware. +method _build_config { + my $working = $ENV{HOME}."/CKAN-Webhooks/mirror"; + if ( ! -d $working ) { + mkpath($working); + } + return App::KSP_CKAN::Tools::Config->new( + working => $working, + ); +} + +method _build__CKAN_meta { + return App::KSP_CKAN::Tools::Git->new( + remote => $self->config->CKAN_meta, + local => $self->config->working, + clean => 1, + ); +} + +method mirror($files) { + # Lets take an array as well! + my @files = reftype \$files ne "SCALAR" ? @{$files} : $files; + + # Prepare Enironment + $self->_CKAN_meta->pull; + local $CWD = $self->config->working."/".$self->_CKAN_meta->working; + + foreach my $file (@files) { + # Lets not try mirroring non existent files + if (! -e $file) { + $self->warn("The ckan '".$file."' doesn't appear to exist"); + next; + } + + # Attempt Mirror + try { + $self->upload_ckan($file); + }; + } + + return 1; +} + +1; diff --git a/t/App/KSP_CKAN/Metadata/Ckan.t b/t/App/KSP_CKAN/Metadata/Ckan.t new file mode 100644 index 0000000..b694253 --- /dev/null +++ b/t/App/KSP_CKAN/Metadata/Ckan.t @@ -0,0 +1,96 @@ +#!/usr/bin/env perl -w + +use lib 't/lib/'; + +use strict; +use warnings; +use Test::Most; +use Test::Warnings; +use App::KSP_CKAN::Test; + +# Setup our environment +my $test = App::KSP_CKAN::Test->new(); + +use_ok("App::KSP_CKAN::Metadata::Ckan"); +$test->create_ckan( file => $test->tmp."/package.ckan", random => 0 ); +$test->create_ckan( file => $test->tmp."/metapackage.ckan", kind => "metapackage" ); +$test->create_ckan( file => $test->tmp."/nohash.ckan", kind => "nohash" ); +$test->create_ckan( file => $test->tmp."/no_mirror.ckan", license => '"restricted"', download => ""); +$test->create_ckan( file => $test->tmp."/hash.ckan", download => "https://github.com/pjf/DogeCoinFlag/releases/download/v1.02/DogeCoinFlag-1.02.zip", license => '[ "restricted", "GPL-2.0" ]' ); + +my $package = App::KSP_CKAN::Metadata::Ckan->new( file => $test->tmp."/package.ckan"); +subtest 'package' => sub { + is($package->identifier, 'ExampleKAN', "Package identifier successfully retrieved"); + is($package->kind, 'package', "Kind successfully retrieved"); + is($package->download, 'https://example.com/example.zip', "Download url successfully retrieved"); + is($package->is_package, 1, "This CKAN is a package"); +}; + +subtest 'fields' => sub { + is($package->homepage, 'https://example.com/homepage', "Homepage url successfully retrieved"); + is($package->repository, 'https://example.com/repository', "Repository url successfully retrieved"); + is($package->abstract, "It's a random example!", "Abstract successfully retrieved"); + is($package->name, "Example KAN", "Name successfully retrieved"); + is($package->license, "CC-BY-NC-SA", "License successfully retrieved"); + is($package->version, "1.0.0.1", "Version successfully retrieved"); + is($package->download_sha1, '1A2B3C4D5E', "Download sha1 successfully retrieved"); + is($package->download_sha256, '1A2B3C4D5E1A2B3C4D5E', "Download sha256 successfully retrieved"); + is($package->download_content_type, 'application/zip', "Download content type successfully retrieved"); +}; + +my $metapackage = App::KSP_CKAN::Metadata::Ckan->new( file => $test->tmp."/metapackage.ckan"); +subtest 'metapackage' => sub { + is($metapackage->identifier, 'ExampleKAN', "Package identifier successfully retrieved"); + is($metapackage->kind, 'metapackage', "Kind successfully retrieved"); + is($metapackage->download, '0', "No download url"); + is($metapackage->is_package, '0', "This CKAN is not a package"); +}; + +my $no_mirror = App::KSP_CKAN::Metadata::Ckan->new( file => $test->tmp."/no_mirror.ckan" ); +my $hash = App::KSP_CKAN::Metadata::Ckan->new( file => $test->tmp."/hash.ckan" ); +my $no_hash = App::KSP_CKAN::Metadata::Ckan->new( file => $test->tmp."/nohash.ckan"); +subtest 'mirror' => sub { + # Can mirror + is($package->can_mirror, 1, "Package can be mirrored"); + is($metapackage->can_mirror, 0, "Meta package can't be mirrored"); + is($hash->can_mirror, 1, "If multi license ckan has a license which can be mirrored"); + is($no_mirror->can_mirror, 0, "License not explicitly listed for mirroring"); + is($no_hash->can_mirror, 0, "No hash in metadata, unable to mirror"); + is($no_mirror->download, 0, "0 returned for blank download link"); + + # Hashes + is($hash->url_hash, "6F8BEBCB", "Hash '".$hash->url_hash."' calculated correctly"); + + # Filenames + is( + $package->mirror_filename, + "1A2B3C4D-ExampleKAN-1.0.0.1.zip", + "Filename '".$package->mirror_filename."' produced correctly" + ); + is( + $hash->mirror_filename, + "1A2B3C4D-ExampleKAN-1.0.0.1.zip", + "Filename '".$hash->mirror_filename."' produced correctly" + ); + is($no_hash->mirror_filename, 0, "Content type not applicable for producing a filename"); + + # Item names + is( + $package->mirror_item, + "ExampleKAN-1.0.0.1", + "Item name '".$package->mirror_item."' produced correctly" + ); + + # Mirror URL + is( + $package->mirror_url, + "https://archive.org/details/ExampleKAN-1.0.0.1", + "URL '".$package->mirror_item."' produced correctly" + ); +}; + +# Cleanup after ourselves +$test->cleanup; + +done_testing(); +__END__ diff --git a/t/App/KSP_CKAN/Mirror.t b/t/App/KSP_CKAN/Mirror.t new file mode 100644 index 0000000..6ac147a --- /dev/null +++ b/t/App/KSP_CKAN/Mirror.t @@ -0,0 +1,139 @@ +#!/usr/bin/env perl -w + +use lib 't/lib/'; + +use v5.010; +use strict; +use warnings; +use Test::Most; +use Test::Warnings; +use File::Copy qw(copy); +use App::KSP_CKAN::Test; +use App::KSP_CKAN::Cached; +use App::KSP_CKAN::Metadata::Ckan; +use App::KSP_CKAN::Tools::Config; + +# Setup our environment +my $test = App::KSP_CKAN::Test->new(); +$test->create_config; + +use_ok("App::KSP_CKAN::Mirror"); + +my $config = App::KSP_CKAN::Tools::Config->new( + file => $test->tmp."/.ksp-ckan", + ); + +$test->create_ckan( + file => $test->tmp."/upload.ckan", + random => 0, + license => '[ "CC-BY-NC-SA", "GPL-2.0" ]', + download => 'http://localhost:3001/test.zip', + sha256 => '4C2BC8312BF1DDE1275A65E09C5D64482C069C7DB2150A4457FA16E06B827C4F', +); + +$test->create_ckan( + file => $test->tmp."/fail.ckan", + random => 0, + license => '[ "CC-BY-NC-SA", "GPL-2.0" ]', + download => 'http://localhost:3001/fail-test.zip', +); + +$test->create_ckan( + file => $test->tmp."/upload2.ckan", + random => 0, + identifier => "ExampleTest", + license => '[ "CC-BY-NC-SA", "GPL-2.0" ]', + download => 'http://localhost:3001/fail-test.zip', + sha256 => '4C2BC8312BF1DDE1275A65E09C5D64482C069C7DB2150A4457FA16E06B827C4F', +); + +my $tester = App::KSP_CKAN::Cached->new( + test_config => $config, + tmp => $test->tmp, + package => 'Mirror', +); + +$tester->test_live(\&iaS3_testing, 1); +$tester->test_cached(\&iaS3_testing, 1); + +sub iaS3_testing { + my ($mirror,$tmp,$message) = @_; + + isa($mirror, "App::KSP_CKAN::Mirror"); + dies_ok + { $mirror->_load_ckan($tmp."doesnt.exist") } + "'_load_ckan' requires a real file"; + + subtest 'Check Cached' => sub { + my $cache = $mirror->config->cache; + is( + $mirror->_check_cached(123456), 0, + "'_check_cached' returns '0' when file is not found in cache", + ); + copy($tmp."/data/test.zip", $cache."/123456-test.zip"); + is( + $mirror->_check_cached(123456), $cache."/123456-test.zip", + "'_check_cached' returns filename and path when file found in cache", + ); + }; + + subtest 'Check Cached' => sub { + my $ckan = App::KSP_CKAN::Metadata::Ckan->new( file => $tmp."/upload.ckan"); + my $fail = App::KSP_CKAN::Metadata::Ckan->new( file => $tmp."/fail.ckan"); + my $cache = $mirror->config->cache; + + is( + $mirror->_check_file($tmp."/".$ckan->mirror_filename, $ckan), 1, + "'_check_file' returns '1' when file is downloaded and sha256 matches", + ); + is( + $mirror->_check_file($tmp."/".$fail->mirror_filename, $fail), 0, + "'_check_file' returns '0' when file fails to download", + ); + copy($tmp."/data/CKAN-meta/README.md", $cache."/".$ckan->url_hash."-test.zip"); + is( + $mirror->_check_file($tmp."/".$ckan->mirror_filename, $ckan), 0, + "'_check_file' returns '0' when file exists but sha256 fails to match", + ); + + }; + + subtest 'Upload CKAN' => sub { + my $ckan = App::KSP_CKAN::Metadata::Ckan->new( file => $tmp."/upload2.ckan"); + my $cache = $mirror->config->cache; + # TODO: If we implement live testing, these will fail. + # we can add a header to our request to simulate + # a failure. + # + # Something like this: + # $ia->_ua->default_headers->push_header( 'x-archive-simulate-error' => 'SlowDown' ); + is( + $mirror->upload_ckan( $tmp."/upload.ckan" ), + 1, "File already mirrored" + ); + is( + $mirror->upload_ckan( $tmp."/upload.ckan" ), + 0, "Archive overloaded while attempting to uploaded" + ); + copy($tmp."/data/test.zip", $cache."/".$ckan->url_hash."-test.zip"); + is( + $mirror->upload_ckan( $tmp."/upload2.ckan" ), + 1, "Uploaded to archive successfully" + ); + is( + $mirror->_check_overload, + 1, "Archive is overloaded" + ); + is( + $mirror->_check_overload, + 0, "Archive not overloaded" + ); + }; +} + + +# Cleanup after ourselves +$test->cleanup; + +done_testing(); +__END__ diff --git a/t/App/KSP_CKAN/NetKAN.t b/t/App/KSP_CKAN/NetKAN.t index 28a8d59..e594d54 100644 --- a/t/App/KSP_CKAN/NetKAN.t +++ b/t/App/KSP_CKAN/NetKAN.t @@ -48,7 +48,7 @@ $netkan->full_index; } -ok( -d $config->working."/cache", "NetKAN path set correctly" ); +ok( -d $config->cache, "NetKAN cache path set correctly" ); $test->cleanup; diff --git a/t/App/KSP_CKAN/NetKAN_debug.t b/t/App/KSP_CKAN/NetKAN_debug.t index 179e3ae..46ccf11 100644 --- a/t/App/KSP_CKAN/NetKAN_debug.t +++ b/t/App/KSP_CKAN/NetKAN_debug.t @@ -47,7 +47,7 @@ $netkan->full_index; ok(! -d "CKAN-meta/DogeCoinFlag", "No metadata commited"); } -#$test->cleanup; +$test->cleanup; done_testing(); __END__ diff --git a/t/App/KSP_CKAN/Roles/FileServices.t b/t/App/KSP_CKAN/Roles/FileServices.t new file mode 100644 index 0000000..e71bbbd --- /dev/null +++ b/t/App/KSP_CKAN/Roles/FileServices.t @@ -0,0 +1,51 @@ +#!/usr/bin/env perl -w + +use lib 't/lib/'; + +use strict; +use warnings; +use Test::Most; +use Test::Warnings; +use App::KSP_CKAN::Test; +use App::KSP_CKAN::Test::FileServices; + +my $fileservices = App::KSP_CKAN::Test::FileServices->new(); + +is( + $fileservices->extension("application/x-gzip"), + "gz", + "'gz' returned for 'application/x-gzip'", +); + +is( + $fileservices->extension("application/x-tar"), + "tar", + "'tar' returned for 'application/x-tar'", +); + +is( + $fileservices->extension("application/x-compressed-tar"), + "tar.gz", + "tar.gz'' returned for 'application/x-compressed-tar'", +); + +is( + $fileservices->extension("application/zip"), + "zip", + "'zip' returned for 'application/zip'", +); + +is( + $fileservices->extension("application/octect-stream"), + 0, + "'0' returned for 'application/octect-stream'", +); + +is( + $fileservices->extension("plain/text"), + 0, + "'0' returned for 'plain/text'", +); + +done_testing(); +__END__ diff --git a/t/App/KSP_CKAN/Roles/Licenses.t b/t/App/KSP_CKAN/Roles/Licenses.t new file mode 100644 index 0000000..1096618 --- /dev/null +++ b/t/App/KSP_CKAN/Roles/Licenses.t @@ -0,0 +1,65 @@ +#!/usr/bin/env perl -w + +use lib 't/lib/'; + +use strict; +use warnings; +use Test::Most; +use Test::Warnings; +use App::KSP_CKAN::Test; +use App::KSP_CKAN::Test::Licenses; + +my $licenses = App::KSP_CKAN::Test::Licenses->new(); + +subtest 'licenses' => sub { + my @licenses = [ + "public-domain", + "Apache", "Apache-1.0", "Apache-2.0", + "Artistic", "Artistic-1.0", "Artistic-2.0", + "BSD-2-clause", "BSD-3-clause", "BSD-4-clause", + "ISC", + "CC-BY", "CC-BY-1.0", "CC-BY-2.0", "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0", + "CC-BY-SA", "CC-BY-SA-1.0", "CC-BY-SA-2.0", "CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0", + "CC-BY-NC", "CC-BY-NC-1.0", "CC-BY-NC-2.0", "CC-BY-NC-2.5", "CC-BY-NC-3.0", "CC-BY-NC-4.0", + "CC-BY-NC-SA", "CC-BY-NC-SA-1.0", "CC-BY-NC-SA-2.0", "CC-BY-NC-SA-2.5", "CC-BY-NC-SA-3.0", "CC-BY-NC-SA-4.0", + "CC-BY-NC-ND", "CC-BY-NC-ND-1.0", "CC-BY-NC-ND-2.0", "CC-BY-NC-ND-2.5", "CC-BY-NC-ND-3.0", "CC-BY-NC-ND-4.0", + "CC0", + "CDDL", "CPL", + "EFL-1.0", "EFL-2.0", + "Expat", "MIT", + "GPL-1.0", "GPL-2.0", "GPL-3.0", + "LGPL-2.0", "LGPL-2.1", "LGPL-3.0", + "GFDL-1.0", "GFDL-1.1", "GFDL-1.2", "GFDL-1.3", + "GFDL-NIV-1.0", "GFDL-NIV-1.1", "GFDL-NIV-1.2", "GFDL-NIV-1.3", + "LPPL-1.0", "LPPL-1.1", "LPPL-1.2", "LPPL-1.3c", + "MPL-1.1", + "Perl", + "Python-2.0", + "QPL-1.0", + "W3C", + "Zlib", + "Zope", + "WTFPL", + "open-source", "unrestricted" ]; + + my @test = $licenses->redistributable_licenses; + + is_deeply( + @test, + @licenses, + "Redistributable licenses returned correctly" + ); +}; + +subtest 'license urls' => sub { + is( + $licenses->license_url("GPL-2.0"), + "http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html", + "GPL-2.0 license url returned successfully", + ); +}; + +1; + +done_testing(); +__END__ diff --git a/t/App/KSP_CKAN/Roles/Validate.t b/t/App/KSP_CKAN/Roles/Validate.t index ca88ad1..f4a2d20 100644 --- a/t/App/KSP_CKAN/Roles/Validate.t +++ b/t/App/KSP_CKAN/Roles/Validate.t @@ -48,7 +48,7 @@ isnt( "DogeCoinFlag-v1.02.ckan Invalid", ); -#$test->cleanup; +$test->cleanup; done_testing(); __END__ diff --git a/t/App/KSP_CKAN/Tools/Config.t b/t/App/KSP_CKAN/Tools/Config.t index 62aad6e..76de44b 100644 --- a/t/App/KSP_CKAN/Tools/Config.t +++ b/t/App/KSP_CKAN/Tools/Config.t @@ -23,9 +23,14 @@ is($config->NetKAN, $test->_tmp."/data/NetKAN", "NetKAN loaded from config"); is($config->GH_token, "123456789", "GH_token loaded from config"); is($config->working, $test->_tmp."/working", "working loaded from config"); is(-d $config->working, 1, "working was automatically created"); +is($config->cache, $test->_tmp."/cache", "cache loaded from config"); +is(-d $config->cache, 1, "cache was automatically created"); is($config->netkan_exe, "https://ckan-travis.s3.amazonaws.com/netkan.exe", "netkan_exe loaded from config"); is($config->ckan_validate, "https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/bin/ckan-validate.py", "ckan_validate loaded from config"); is($config->ckan_schema, "https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/CKAN.schema", "ckan_schema loaded from config"); +is($config->IA_access, "12345678", "IA_access loaded from config"); +is($config->IA_secret, "87654321", "IA_secret loaded from config"); +is($config->IA_collection, "collection", "IA_collection loaded from config"); is($config->debugging, 0, "debug disabled"); $test->create_config( optional => 0 ); @@ -35,6 +40,8 @@ $config = App::KSP_CKAN::Tools::Config->new( is($config->GH_token, 0, "GH_token returns false"); is($config->working, $ENV{HOME}."/CKAN-working", "working default generated"); +is($config->cache, $ENV{HOME}."/CKAN-working/cache", "cache default generated"); +is($config->IA_collection, "test_collection", "IA_collection default generated"); $test->cleanup; diff --git a/t/App/KSP_CKAN/Tools/Git.t b/t/App/KSP_CKAN/Tools/Git.t index 74657bf..e0d0aa0 100644 --- a/t/App/KSP_CKAN/Tools/Git.t +++ b/t/App/KSP_CKAN/Tools/Git.t @@ -42,11 +42,6 @@ my $git = App::KSP_CKAN::Tools::Git->new( ); isa_ok($git, "App::KSP_CKAN::Tools::Git"); -# Test Cleanup -mkpath($test->tmp."/CKAN-meta"); -$git->_clean; -isnt($test->tmp."/CKAN-meta", 1, "Clean was successful"); - # Test our clone # Git gives benign 'warning: --depth is ignored in local clones; use file:// instead.' # Local pulls don't honor depth, but we're only testing that we can clone. @@ -54,7 +49,7 @@ isa_ok($git->_git, "Git::Wrapper"); is(-e $test->tmp."/CKAN-meta/README.md", 1, "Cloned successfully"); # Test adding -$test->create_ckan( $test->tmp."/CKAN-meta/test_file.ckan" ); +$test->create_ckan( file => $test->tmp."/CKAN-meta/test_file.ckan" ); is($git->changed, 0, "No file was added"); $git->add($test->tmp."/CKAN-meta/test_file.ckan"); is($git->changed, 1, "File was added"); @@ -70,7 +65,7 @@ subtest 'Committing' => sub { # Test committing all files for my $filename (qw(test_file2.ckan test_file3.ckan)) { - $test->create_ckan( $test->tmp."/CKAN-meta/".$filename ); + $test->create_ckan( file => $test->tmp."/CKAN-meta/".$filename ); } $git->add; is($git->changed, 2, "Files were added"); @@ -80,7 +75,7 @@ subtest 'Committing' => sub { is($git->changed, 0, "Commit pushed"); # Test reseting - $test->create_ckan( $test->tmp."/CKAN-meta/test_file2.ckan" ); + $test->create_ckan( file => $test->tmp."/CKAN-meta/test_file2.ckan" ); is($git->changed, 1, "test_file2.ckan was changed"); @files = $git->changed; $git->reset( file => $files[0] ); @@ -97,7 +92,7 @@ my $pull = App::KSP_CKAN::Tools::Git->new( clean => 1, ); $pull->pull; -$test->create_ckan( $test->tmp."/CKAN-meta-pull/test_pull.ckan" ); +$test->create_ckan( file => $test->tmp."/CKAN-meta-pull/test_pull.ckan" ); $pull->add; $pull->commit(all => 1); $pull->push; @@ -109,6 +104,11 @@ unlink($test->tmp."/CKAN-meta/test_file.ckan"); $git->add; is($git->changed, 2, "File delete not commited"); +# Test cleanup +$test->create_ckan( file => $test->tmp."/CKAN-meta/cleaned_file.ckan" ); +$git->_clean; +isnt(-e $test->tmp."/CKAN-meta/cleaned_file.ckan", 1, "Cleanup Successful"); + subtest 'Git Errors' => sub { my $remote_error = App::KSP_CKAN::Tools::Git->new( remote => $test->tmp."/data/CKAN-meta", @@ -117,7 +117,7 @@ subtest 'Git Errors' => sub { clean => 1, ); $remote_error->pull; - $test->create_ckan( $test->tmp."/remote-error/test_push.ckan" ); + $test->create_ckan( file => $test->tmp."/remote-error/test_push.ckan" ); $remote_error->add; $remote_error->commit(all => 1); { diff --git a/t/App/KSP_CKAN/Tools/IA.t b/t/App/KSP_CKAN/Tools/IA.t new file mode 100644 index 0000000..72b1d67 --- /dev/null +++ b/t/App/KSP_CKAN/Tools/IA.t @@ -0,0 +1,176 @@ +#!/usr/bin/env perl -w + +use lib 't/lib/'; + +use v5.010; +use strict; +use warnings; +use Test::Most; +use Test::Warnings; +use App::KSP_CKAN::Test; +use App::KSP_CKAN::Cached; +use App::KSP_CKAN::Tools::Config; +use App::KSP_CKAN::Metadata::Ckan; + +# Setup our environment +my $test = App::KSP_CKAN::Test->new(); +$test->create_config; + +use_ok("App::KSP_CKAN::Tools::IA"); + +my $config = App::KSP_CKAN::Tools::Config->new( + file => $test->tmp."/.ksp-ckan", + ); + +my $ia = App::KSP_CKAN::Tools::IA->new( + config => $config, + collection => "test_collection", +); + +isa_ok($ia->_ua, "LWP::UserAgent"); +is( + $ia->_ua->{def_headers}{authorization}, + 'LOW 12345678:87654321', + "Authorization header set", +); + +subtest 'single value' => sub { + my ($header, $value) = $ia->_archive_header( "collection", "test" ); + is( $header, "x-archive-meta-collection", + "Single header name generated corerctly", + ); + is( $value, "test", + "Single header value generated corerctly", + ); +}; + +subtest 'single value as array' => sub { + my @array = "test"; + my ($header, $value) = $ia->_archive_header( "collection", \@array ); + is( $header, "x-archive-meta-collection", + "Single header name generated corerctly", + ); + is( $value, "test", + "Single header value generated corerctly", + ); +}; + +subtest 'multi value as array' => sub { + my @values = qw( test1 test2 test3 ); + my @expected = qw( + x-archive-meta01-creator + test1 + x-archive-meta02-creator + test2 + x-archive-meta03-creator + test3 + ); + my @result = $ia->_archive_header( "creator", \@values ); + is_deeply( + \@result, \@expected, "Multi value header returned correctly", + ); +}; + +$test->create_ckan( + file => $test->tmp."/upload.ckan", + random => 0, + license => '[ "CC-BY-NC-SA", "GPL-2.0" ]', +); +my $ckan = App::KSP_CKAN::Metadata::Ckan->new( file => $test->tmp."/upload.ckan" ); + +subtest '_metadata_headers' => sub { + my $metadata_headers = $ia->_metadata_headers( $test->tmp."/test.zip", $ckan ); + + isa_ok($metadata_headers, 'HTTP::Headers'); + is($metadata_headers->{'x-archive-meta-title'}, 'Example KAN - 1.0.0.1', "Title header generated"); + is($metadata_headers->{'x-archive-meta-creator'}, 'Techman83', "Creator header generated"); + is($metadata_headers->{'x-archive-meta-mediatype'}, 'software', "Mediatype header generated"); + is($metadata_headers->{'content-type'}, 'application/zip', "Content type header generated"); + is($metadata_headers->{'x-archive-meta-collection'}, 'test_collection', "Collection header generated"); + is( + $metadata_headers->{'x-archive-meta-description'}, + 'It\'s a random example!

Homepage: https://example.com/homepage
Repository: https://example.com/repository
License(s): CC-BY-NC-SA GPL-2.0', + "Description header generated", + ); + is($metadata_headers->{'x-archive-meta-subject'}, 'ksp; kerbal space program; mod', "Subject header generated"); + is( + $metadata_headers->{'x-archive-meta01-licenseurl'}, + 'http://creativecommons.org/licenses/by-nc-sa/1.0/', + "CC-BY-NC-SA License header added", + ); + is( + $metadata_headers->{'x-archive-meta02-licenseurl'}, + 'http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html', + "GPL-2.0 License header added", + ); +}; + +my $tester = App::KSP_CKAN::Cached->new( test_config => $config, tmp => $test->tmp ); + +$tester->test_live(\&iaS3_testing, 1); +$tester->test_cached(\&iaS3_testing, 1); + +sub iaS3_testing { + my ($ia,$tmp,$message) = @_; + my $ckan = App::KSP_CKAN::Metadata::Ckan->new( file => $tmp."/upload.ckan" ); + + subtest 'files' => sub { + my $file = $tmp."/data/test.zip"; + is( + $ia->_upload_uri($ckan), + $ia->iaS3uri."/ExampleKAN-1.0.0.1/1A2B3C4D-ExampleKAN-1.0.0.1.zip", + "URI produced correctly", + ); + + my $put = $ia->_put_request( + file => $file, + headers => $ia->_metadata_headers( $file, $ckan ), + uri => $ia->_upload_uri($ckan) + ); + isa_ok($put, "HTTP::Request", "\$put is a 'HTTP::Request' object"); + is($put->{_method}, "PUT", "Method 'PUT' set correctly"); + is( + $put->{_uri}, + $ia->iaS3uri.'/ExampleKAN-1.0.0.1/1A2B3C4D-ExampleKAN-1.0.0.1.zip', + "Uri in put correct", + ); + isa_ok($put->{_headers}, "HTTP::Headers", "Headers are an 'HTTP::Headers' object"); + is( + $ia->put_ckan( ckan => $ckan, file => $file), + 1, "File uploaded successfully" + ); + is( + $ia->ckan_mirrored( ckan => $ckan ), + 1, "Ckan is mirrored" + ); + is( + $ia->check_overload, + 0, "IA is not overloaded" + ); + + # TODO: If we implement live testing, these will fail. + # we can add a header to our request to simulate + # a failure. + # + # Something like this: + # $ia->_ua->default_headers->push_header( 'x-archive-simulate-error' => 'SlowDown' ); + is( + $ia->put_ckan( ckan => $ckan, file => $file), + 0, "File upload returned failure correctly" + ); + is( + $ia->ckan_mirrored( ckan => $ckan ), + 0, "Ckan is not mirrored" + ); + is( + $ia->check_overload, + 1, "IA is overloaded" + ); + }; +} + +# Cleanup after ourselves +$test->cleanup; + +done_testing(); +__END__ diff --git a/t/App/KSP_CKAN/Tools/NetKAN.t b/t/App/KSP_CKAN/Tools/NetKAN.t index 3f3df75..2e0cd1d 100644 --- a/t/App/KSP_CKAN/Tools/NetKAN.t +++ b/t/App/KSP_CKAN/Tools/NetKAN.t @@ -95,12 +95,12 @@ TODO: { # Test file validation subtest 'File Validation' => sub { - $test->create_ckan( $config->working."/CKAN-meta/test_file.ckan" ); + $test->create_ckan( file => $config->working."/CKAN-meta/test_file.ckan" ); $netkan->_commit( $config->working."/CKAN-meta/test_file.ckan" ); is($netkan->ckan_meta->changed(origin => 0), 0, "Commit validated file successful"); $netkan->ckan_meta->push; is($netkan->ckan_meta->changed, 0, "Changes pushed repository" ); - $test->create_ckan( $config->working."/CKAN-meta/test_file2.ckan", 0 ); + $test->create_ckan( file => $config->working."/CKAN-meta/test_file2.ckan", valid => 0 ); $netkan->_commit( $config->working."/CKAN-meta/test_file2.ckan" ); is( $netkan->ckan_meta->changed, 0, "broken metadata was not committed" ); $netkan->ckan_meta->add; diff --git a/t/App/KSP_CKAN/WebHooks/InflateNetKAN.t b/t/App/KSP_CKAN/WebHooks/InflateNetKAN.t new file mode 100644 index 0000000..a35555e --- /dev/null +++ b/t/App/KSP_CKAN/WebHooks/InflateNetKAN.t @@ -0,0 +1,85 @@ +#!/usr/bin/env perl -w + +use lib 't/lib/'; + +use strict; +use warnings; +use Test::Most; +use Test::Warnings; +use File::chdir; +use App::KSP_CKAN::Test; +use App::KSP_CKAN::Tools::Config; + + +use_ok("App::KSP_CKAN::WebHooks::InflateNetKAN"); + +subtest 'Scalar Identifier' => sub { + ## Setup our environment + my $test = App::KSP_CKAN::Test->new(); + $test->create_repo("CKAN-meta"); + $test->create_repo("NetKAN"); + + # Config + $test->create_config(nogh => 1); + my $config = App::KSP_CKAN::Tools::Config->new( + file => $test->tmp."/.ksp-ckan", + ); + my $inflate = App::KSP_CKAN::WebHooks::InflateNetKAN->new( + config => $config, + ); + + $inflate->inflate("DogeCoinFlag"); + + { + local $CWD = $config->working; + my @files = glob( "./CKAN-meta/DogeCoinFlag/*.ckan" ); + foreach my $file (@files) { + ok($file =~ /DogeCoinFlag-v\d.\d\d.ckan/, "NetKAN Inflated"); + } + + TODO: { + local $TODO = "This test is broken on travis for some reason." if ($ENV{TRAVIS}); + is($#files, 0, "A file was found"); + } + ok( -d $config->cache, "NetKAN cache path set correctly" ); + } + + $test->cleanup; +}; + +subtest 'Array of Identifiers' => sub { + ## Setup our environment + my $test = App::KSP_CKAN::Test->new(); + $test->create_repo("CKAN-meta"); + $test->create_repo("NetKAN"); + + # Config + $test->create_config(nogh => 1); + my $config = App::KSP_CKAN::Tools::Config->new( + file => $test->tmp."/.ksp-ckan", + ); + my $inflate = App::KSP_CKAN::WebHooks::InflateNetKAN->new( + config => $config, + ); + + # TODO: We should expand for multiple unique identifiers in testing + my @identifiers = qw(DogeCoinFlag DogeCoinFlag); + $inflate->inflate(\@identifiers); + { + local $CWD = $config->working; + my @files = glob( "./CKAN-meta/DogeCoinFlag/*.ckan" ); + foreach my $file (@files) { + ok($file =~ /DogeCoinFlag-v\d.\d\d.ckan/, "NetKAN Inflated"); + } + + TODO: { + local $TODO = "This test is broken on travis for some reason." if ($ENV{TRAVIS}); + is($#files, 0, "A file was found"); + } + ok( -d $config->cache, "NetKAN cache path set correctly" ); + } + $test->cleanup; +}; + +done_testing(); +__END__ diff --git a/t/bin/cached_api.pl b/t/bin/cached_api.pl new file mode 100755 index 0000000..6b4d62d --- /dev/null +++ b/t/bin/cached_api.pl @@ -0,0 +1,21 @@ +#!/usr/bin/env perl +use lib './t/lib/'; + +use strict; +use Dancer2; +use App::KSP_CKAN::Test::IA; + +my $DEBUG = $ENV{KSP_CKAN_DEBUG} || 0; + +# Debug/Dev +if ($DEBUG) { + set logger => 'console'; + set log => 'core'; +} + +# Setting it in the config file didn't appear to work. +# Not sure why, normally would use plackup, but unnecessary +# here. +set port => 3001; + +dance; diff --git a/t/data/test.zip b/t/data/test.zip new file mode 100644 index 0000000..4e2d51e Binary files /dev/null and b/t/data/test.zip differ diff --git a/t/lib/App/KSP_CKAN/Cached.pm b/t/lib/App/KSP_CKAN/Cached.pm new file mode 100644 index 0000000..1198d39 --- /dev/null +++ b/t/lib/App/KSP_CKAN/Cached.pm @@ -0,0 +1,93 @@ +package App::KSP_CKAN::Cached; + +use strict; +use warnings; +use App::KSP_CKAN::Tools::Config; +use Method::Signatures 20140224; +use Test::Most; +use Moo; +use namespace::clean; + +has 'package' => ( is => 'ro', default => sub { 'IA' } ); +has 'config' => ( is => 'ro', lazy => 1, builder => 1 ); +has 'test_config' => ( is => 'ro' ); +has 'tmp' => ( is => 'ro' ); + +method _build_config() { + my $config = App::KSP_CKAN::Tools::Config->new( file => "$ENV{HOME}/.ksp_ckan-test" ); + return $config; +} + +method _live_ia { + return App::KSP_CKAN::Tools::IA->new( + config => $self->config, + collection => "test_collection", + ); +} + +method _cached_ia { + return App::KSP_CKAN::Tools::IA->new( + config => $self->test_config, + collection => "test_collection", + iaS3uri => "http://localhost:3001", + iaDLuri => "http://localhost:3001/download", + iaMDuri => "http://localhost:3001/metadata", + ); +} + +method _live_environment { + if ( $self->package eq 'Mirror' ) { + return App::KSP_CKAN::Mirror->new( + config => $self->config, + tmp => $self->tmp, + '_ia' => $self->_live_ia, + ); + } + return $self->_cached_ia; +} + +method _cached_environment { + if ( $self->package eq 'Mirror' ) { + return App::KSP_CKAN::Mirror->new( + config => $self->test_config, + tmp => $self->tmp, + '_ia' => $self->_cached_ia, + ); + } + return $self->_cached_ia; +} + +method test_live($test, $number_tests) { + SKIP: { + # skip "No auth credentials found.", $number_tests unless ( -e "$ENV{HOME}/.ksp_ckan-test" ); + skip "Live tests are not yet implemented.", $number_tests; + + $test->($self->_live_environment, $self->tmp, "Testing Live Internet Archive API"); + } +} + +method test_cached($test, $number_tests) { + SKIP: { + eval { + require Dancer2; + }; + + skip 'These tests are for cached testing and require Dancer2', $number_tests if ($@); + + my $pid = fork(); + + if (!$pid) { + exec("t/bin/cached_api.pl"); + } + + # Allow some time for the instance to spawn. TODO: Make this smarter + sleep 5; + + $test->($self->_cached_environment, $self->tmp,"Testing Cached API"); + + # Kill Dancer + kill 9, $pid; + } +} + +1; diff --git a/t/lib/App/KSP_CKAN/Test.pm b/t/lib/App/KSP_CKAN/Test.pm index 49c95e1..daf33ce 100644 --- a/t/lib/App/KSP_CKAN/Test.pm +++ b/t/lib/App/KSP_CKAN/Test.pm @@ -10,6 +10,7 @@ use File::Temp qw(tempdir); use File::Path qw(remove_tree mkpath); use File::chdir; use File::Copy::Recursive qw(dircopy dirmove); +use File::Copy qw(copy); use Capture::Tiny qw(capture); use Moo; use namespace::clean; @@ -81,7 +82,7 @@ method create_repo($repo) { =method create_ckan - $test->create_ckan("Path to file"); + $test->create_ckan(file => "/path/to/file"); Creates an example ckan that would pass validation at the specified path. @@ -89,22 +90,67 @@ path. Takes an optional extra argument, that if set to false will create a ckan that won't pass schema validation. - $test->create_ckan("Path to file", 0); + $test->create_ckan( file => "/path/to/file", valid => 0); + +=over + +=item file + +Path and file we are creating. + +=item valid + +Defaults to true. False value will create a CKAN that will fail +validation against the schema. + +=item kind + +Allows us to specify a different kind of package. 'metadata' is the +only accepted one at the moment. + +=item license + +Allows us to specify a different license. + +=back =cut -method create_ckan($file, $valid = 1) { - my $identifier = $valid ? "identifier" : "invalid_schema"; +method create_ckan( + :$file, + :$valid = 1, + :$random = 1, + :$identifier = "ExampleKAN", + :$kind = "package", + :$license = '"CC-BY-NC-SA"', + :$download = "https://example.com/example.zip", + :$sha256 = "1A2B3C4D5E1A2B3C4D5E", +) { + my $attribute = $valid ? "identifier" : "invalid_schema"; + + # Allows us against a metapackage. TODO: make into valid metapackage + my $package; + if ( $kind eq "metapackage" ) { + $package = '"kind": "metapackage"'; + } elsif ( $kind eq "nohash" ) { + $package = qq|"download": "$download","download_content_type": "text/plain"|; + } else { + $package = qq|"download": "$download","download_hash": { "sha1": "1A2B3C4D5E","sha256": "$sha256" }, "download_content_type": "application/zip"|; + } # Lets us generate CKANs that are different. # http://www.perlmonks.org/?node_id=233023 my @chars = ("A".."Z", "a".."z"); my $rand; - $rand .= $chars[rand @chars] for 1..8; + if ( $random ) { + $rand .= $chars[rand @chars] for 1..8; + } else { + $rand = "random"; + } # Create the CKAN open my $in, '>', $file; - print $in qq|{"spec_version": 1, "$identifier": "ExampleKAN", "license": "CC-BY-NC-SA", "ksp_version": "0.90", "name": "Example KAN", "abstract": "It's a $rand example!", "author": "Techman83", "version": "1.0.0.1", "download": "https://example.com/example.zip"}|; + print $in qq|{"spec_version": 1, "$attribute": "$identifier", "license": $license, "ksp_version": "0.90", "name": "Example KAN", "abstract": "It's a $rand example!", "author": "Techman83", "version": "1.0.0.1", $package, "resources": { "homepage": "https://example.com/homepage", "repository": "https://example.com/repository" }}|; close $in; return; } @@ -134,11 +180,15 @@ method create_config(:$optional = 1, :$nogh = 0) { print $in "netkan_exe=https://ckan-travis.s3.amazonaws.com/netkan.exe\n"; print $in "ckan_validate=https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/bin/ckan-validate.py\n"; print $in "ckan_schema=https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/CKAN.schema\n"; + print $in "IA_access=12345678\n"; + print $in "IA_secret=87654321\n"; # TODO: This is a little ugly. if ($optional) { print $in "GH_token=123456789\n" if ! $nogh; print $in "working=".$self->_tmp."/working\n"; + print $in "cache=".$self->_tmp."/cache\n"; + print $in "IA_collection=collection\n"; } close $in; diff --git a/t/lib/App/KSP_CKAN/Test/FileServices.pm b/t/lib/App/KSP_CKAN/Test/FileServices.pm new file mode 100644 index 0000000..22d4e52 --- /dev/null +++ b/t/lib/App/KSP_CKAN/Test/FileServices.pm @@ -0,0 +1,10 @@ +package App::KSP_CKAN::Test::FileServices; + +use Method::Signatures 20140224; +use App::KSP_CKAN::Tools::Http; +use Moo; +use namespace::clean; + +with('App::KSP_CKAN::Roles::FileServices'); + +1; diff --git a/t/lib/App/KSP_CKAN/Test/IA.pm b/t/lib/App/KSP_CKAN/Test/IA.pm new file mode 100644 index 0000000..d817f44 --- /dev/null +++ b/t/lib/App/KSP_CKAN/Test/IA.pm @@ -0,0 +1,53 @@ +package App::KSP_CKAN::Test::IA; + +use Dancer2; + +# A little ugly, but works for this purpose. +my $state; + +# TODO: These could stand to be more thorough, but is enough +# to test the code pathways. + +get '/' => sub { + if ( ! defined $state->{overload} ) { + $state->{overload} = 1; + return ''; + } else { + $state->{overload} = undef; + status(503); + return ''; + } +}; + +put '/:item/:file' => sub { + if ( ! defined $state->{upload} ) { + $state->{upload} = 1; + return ''; + } else { + $state->{upload} = undef; + status(400); + return ''; + } +}; + +get '/download/:item/:file' => sub { + if ( ! defined $state->{check} ) { + $state->{check} = 1; + return ''; + } else { + $state->{check} = undef; + status(400); + return ''; + } +}; + +get '/metadata/:item' => sub { + if ( ! defined $state->{mirrored} ) { + $state->{mirrored} = 1; + return '{ "files": [ { "name": "74770739-ExampleKAN-1.0.0.1.zip", "sha1": "1a2B3c4D5e"} ] }'; + } else { + return '{ }'; + } +}; + +1; diff --git a/t/lib/App/KSP_CKAN/Test/Licenses.pm b/t/lib/App/KSP_CKAN/Test/Licenses.pm new file mode 100644 index 0000000..da9a356 --- /dev/null +++ b/t/lib/App/KSP_CKAN/Test/Licenses.pm @@ -0,0 +1,9 @@ +package App::KSP_CKAN::Test::Licenses; +use Method::Signatures 20140224; +use App::KSP_CKAN::Tools::Http; +use Moo; +use namespace::clean; + +with('App::KSP_CKAN::Roles::Licenses'); + +1; diff --git a/t/public/test.zip b/t/public/test.zip new file mode 100644 index 0000000..4e2d51e Binary files /dev/null and b/t/public/test.zip differ