Merge pull request #93 from HebaruSan/remove-validation

Retire Perl/Python schema validation

.travis.yml

@@ -15,9 +15,7 @@ before_install:
   # Prevent "Please tell me who you are" errors for certain DZIL configs
   - git config --global user.name "TravisCI"
 install:
-  # Deal with all of the DZIL dependancies, quickly and quietly
-  - pip install --user jsonschema
-  - export PYTHONPATH=~/.local/lib/python2.7/site-packages/
+  # Deal with all of the DZIL dependencies, quickly and quietly
   - cpanm --quiet --notest --skip-satisfied Dist::Zilla
   - cpanm --quiet --notest --skip-satisfied Test::Perl::Critic
   - dzil authordeps | grep -vP '[^\w:]' | xargs -n 5 -P 10 cpanm --quiet --notest --skip-satisfied

README.md

@@ -3,7 +3,7 @@ App::KSP-CKAN    [![Build Status](https://travis-ci.org/KSP-CKAN/NetKAN-bot.svg?
 Non Perl Dependencies
 =====================
 ```bash
-apt-get install liblocal-lib-perl cpanminus build-essential mono-complete libcurl4-openssl-dev python-jsonschema libdist-zilla-perl
+apt-get install liblocal-lib-perl cpanminus build-essential mono-complete libcurl4-openssl-dev libdist-zilla-perl
 ```
 
 NetKAN will need certs for mono
@@ -42,8 +42,6 @@ An ini file with the following contents will need to created at ~/.ksp-ckan
 [email protected]:KSP-CKAN/CKAN-meta.git
 [email protected]:KSP-CKAN/NetKAN-bot.git
 netkan_exe=https://ckan-travis.s3.amazonaws.com/netkan.exe
-ckan_validate=https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/bin/ckan-validate.py
-ckan_schema=https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/CKAN.schema
 working=/home/NetKAN/NetKAN
 ```
 
@@ -71,7 +69,7 @@ Enable it in cron with (crontab -e as the netkan user):
 00 */3 * * * PERL5LIB=/home/netkan/perl5/lib/perl5/ netkan-indexer
 ```
 
-There is a 'lite' cli option is not implemented. It's a future concept to allow 'lite' 
+There is a 'lite' cli option is not implemented. It's a future concept to allow 'lite'
 skimming of metadata API endpoints without performing a full metadata inflation.
 
 License

dist.ini

@@ -2,7 +2,7 @@ name    = App-KSP_CKAN
 author  = Leon Wright <[email protected]>
 license = MIT
 copyright_holder = Leon Wright
-copyright_year   = 2015
+copyright_year   = 2019
 
 [Git::NextVersion]
 fist_version = 0.01
@@ -27,7 +27,7 @@ match     = init/*
 [Test::Perl::Critic]
 [Test::PodSpelling]
 stopwords = exe
-stopwords = inflater 
+stopwords = inflater
 stopwords = CKAN
 stopwords = netkan
 stopwords = api

lib/App/KSP_CKAN/NetKAN.pm

@@ -83,19 +83,6 @@ method _mirror_files {
     exe   => 1,
   );
 
-  # ckan-validate.py
-  $self->_http->mirror(
-    url   => $config->ckan_validate,
-    path  => $config->working."/ckan-validate.py",
-    exe   => 1,
-  );
-
-  # CKAN.schema
-  $self->_http->mirror(
-    url   => $config->ckan_schema,
-    path  => $config->working."/CKAN.schema",
-  );
-
   return;
 }
 

lib/App/KSP_CKAN/Tools/Config.pm

@@ -34,8 +34,6 @@ has '_config'       => ( is => 'ro', lazy => 1, builder => 1 );
 has 'CKAN_meta'     => ( is => 'ro', lazy => 1, builder => 1 );
 has 'NetKAN'        => ( is => 'ro', lazy => 1, builder => 1 );
 has 'netkan_exe'    => ( is => 'ro', lazy => 1, builder => 1 );
-has 'ckan_validate' => ( is => 'ro', lazy => 1, builder => 1 );
-has 'ckan_schema'   => ( is => 'ro', lazy => 1, builder => 1 );
 has 'GH_token'      => ( is => 'ro', lazy => 1, builder => 1 );
 has 'GH_user'       => ( is => 'ro', lazy => 1, builder => 1 );
 has 'GH_repo'       => ( is => 'ro', lazy => 1, builder => 1 );
@@ -68,16 +66,6 @@ method _build_netkan_exe {
   return $self->_config->{_}{'netkan_exe'};
 }
 
-method _build_ckan_validate {
-  croak( "Missing 'ckan_validate' from config" ) if ! $self->_config->{_}{'ckan_validate'};
-  return $self->_config->{_}{'ckan_validate'};
-}
-
-method _build_ckan_schema {
-  croak( "Missing 'ckan_schema' from config" ) if ! $self->_config->{_}{'ckan_schema'};
-  return $self->_config->{_}{'ckan_schema'};
-}
-
 method _build_IA_access {
   croak( "Missing 'IA_access' from config" ) if ! $self->_config->{_}{'IA_access'};
   return $self->_config->{_}{'IA_access'};

lib/App/KSP_CKAN/Tools/NetKAN.pm

@@ -166,14 +166,6 @@ method _commit($file) {
   $self->ckan_meta->add($file);
   my $changed = basename($file,  ".ckan");
 
-  if ( $self->validate($file) ) {
-    $self->warn("Failed to Parse $changed");
-    $self->ckan_meta->reset(file => $file);
-    $self->ckan_meta->clean_untracked;
-    $self->_status->failure("Schema validation failed");
-    return 1;
-  }
-
   if ($self->is_debug()) {
     $self->debug("$changed would have been committed");
     $self->ckan_meta->reset(file => $file);
@@ -256,6 +248,6 @@ method inflate {
   return 0;
 }
 
-with('App::KSP_CKAN::Roles::Logger','App::KSP_CKAN::Roles::Validate');
+with('App::KSP_CKAN::Roles::Logger');
 
 1;

t/App/KSP_CKAN/NetKAN.t

@@ -24,7 +24,7 @@ my $config = App::KSP_CKAN::Tools::Config->new(
 );
 
 use_ok("App::KSP_CKAN::NetKAN");
-my $netkan = App::KSP_CKAN::NetKAN->new( 
+my $netkan = App::KSP_CKAN::NetKAN->new(
   config => $config,
 );
 
@@ -41,34 +41,31 @@ $netkan->full_index;
   my $why = "These tests pass locally, but not within Travis";
   TODO: {
     local $TODO = $why if $ENV{TRAVIS};
-    ok($#files != -1, "We commited files to master");
+    ok($#files != -1, "We committed files to master");
   }
 
   my $git = App::KSP_CKAN::Tools::Git->new(
     remote => $config->CKAN_meta,
     local => $config->working,
     clean => 1,
   );
-  
+
   my $identifier = "DogeCoinFlagStaged";
   is($git->current_branch, "master", "We started on the master branch");
-  ok(! -d "CKAN-meta/$identifier", "Staged netkan not commited to master");
+  ok(! -d "CKAN-meta/$identifier", "Staged netkan not committed to master");
 
   $git->checkout_branch($identifier);
   is($git->current_branch, $identifier, "We are on the $identifier branch");
   my @id_branch = glob( "./CKAN-meta/$identifier/*.ckan" );
 
   TODO: {
     local $TODO = $why if $ENV{TRAVIS};
-    ok($#id_branch != -1, "We commited files to $identifier");
+    ok($#id_branch != -1, "We committed files to $identifier");
   }
 
   foreach my $file (@id_branch) {
-    ok($file =~ /$identifier-v\d.\d\d.ckan/, "Commited to $identifier");
+    ok($file =~ /$identifier-v\d.\d\d.ckan/, "Committed to $identifier");
   }
-
-  ok(! -d  "CKAN-meta/DogeCoinFlag-broken", "No broken metadata committed");
-  ok(! -d  "CKAN-meta/DogeCoinFlag-invalid", "No invalid metadata committed");
 }
 
 ok( -d $config->cache, "NetKAN cache path set correctly" );

t/App/KSP_CKAN/Tools/Config.t

@@ -27,8 +27,6 @@ is(-d $config->working, 1, "working was automatically created");
 is($config->cache, $test->_tmp."/cache", "cache loaded from config");
 is(-d $config->cache, 1, "cache was automatically created");
 is($config->netkan_exe, "https://ckan-travis.s3.amazonaws.com/netkan.exe", "netkan_exe loaded from config");
-is($config->ckan_validate, "https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/bin/ckan-validate.py", "ckan_validate loaded from config");
-is($config->ckan_schema, "https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/CKAN.schema", "ckan_schema loaded from config");
 is($config->IA_access, "12345678", "IA_access loaded from config");
 is($config->IA_secret, "87654321", "IA_secret loaded from config");
 is($config->IA_collection, "collection", "IA_collection loaded from config");

t/App/KSP_CKAN/Tools/Http.t

@@ -20,8 +20,6 @@ subtest 'mirror' => sub {
   $http->mirror( url => "https://ckan-travis.s3.amazonaws.com/netkan.exe", path => $test->tmp."/netkan.exe");
   is(-e $test->tmp."/netkan.exe", 1, "Mirrored successfully");
   isnt(-X $test->tmp."/netkan.exe", 1, "File not executable");
-  $http->mirror( url => "https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/bin/ckan-validate.py", path => $test->tmp."/ckan-validate.py", exe => 1);
-  is(-X $test->tmp."/ckan-validate.py", 1, "File executable");
 };
 
 $test->cleanup;

t/App/KSP_CKAN/Tools/NetKAN.t

@@ -48,8 +48,6 @@ $netkan_git->pull;
 # netkan.exe
 my $http = App::KSP_CKAN::Tools::Http->new();
 $http->mirror( url => $config->netkan_exe, path => $test->tmp."/netkan.exe", exe => 1 );
-$http->mirror( url => $config->ckan_validate, path => $config->working."/ckan-validate.py", exe => 1 );
-$http->mirror( url => $config->ckan_schema, path => $config->working."/CKAN.schema" );
 
 use_ok("App::KSP_CKAN::Tools::NetKAN");
 my $netkan = App::KSP_CKAN::Tools::NetKAN->new(
@@ -87,20 +85,6 @@ TODO: {
   ok( -d $test->tmp."/cache", "NetKAN Cache path set correctly");
 }
 
-# Test file validation
-subtest 'File Validation' => sub {
-  $test->create_ckan( file => $config->working."/CKAN-meta/test_file.ckan" );
-  $netkan->_commit( $config->working."/CKAN-meta/test_file.ckan" );
-  is($netkan->ckan_meta->changed(origin => 0), 0, "Commit validated file successful");
-  $netkan->ckan_meta->push;
-  is($netkan->ckan_meta->changed, 0, "Changes pushed repository" );
-  $test->create_ckan( file => $config->working."/CKAN-meta/test_file2.ckan", valid => 0 );
-  $netkan->_commit( $config->working."/CKAN-meta/test_file2.ckan" );
-  is( $netkan->ckan_meta->changed, 0, "broken metadata was not committed" );
-  $netkan->ckan_meta->add;
-  is( $netkan->ckan_meta->changed, 0, "broken metadata gets removed" );
-};
-
 # Test staged commits
 subtest 'Staged Commits' => sub {
   # Setup