From 880757ea73cd599b20ef4d1f126886e3679c43dc Mon Sep 17 00:00:00 2001
From: tiagoassuncao
Date: Tue, 7 Jun 2016 19:36:09 -0300
Subject: [PATCH] Fixing Bug #786664
Adding ci status on pet
Signed-off-by: Tiago
Signed-off-by: Victor
---
.gitignore | 3 +
pet/ci.py | 117 +++++++++++
pet/classifier.py | 6 +
pet/sql.py | 3 +-
pet/web/__init__.py | 2 +
pet/web/templates/overview.pt | 378 ++++++++++++++++++++--------------
pet/web/views.py | 2 +-
test/test_ci_debian.py | 31 +++
update-archive | 4 +
9 files changed, 393 insertions(+), 153 deletions(-)
create mode 100644 pet/ci.py
create mode 100644 test/test_ci_debian.py
diff --git a/.gitignore b/.gitignore
index e0a48fa..ae5d6d3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -67,3 +67,6 @@ docs/_build/
.vagrant/
nodes/vagrant*
+
+# Json to store packages of ci_debian info
+packages.json
diff --git a/pet/ci.py b/pet/ci.py
new file mode 100644
index 0000000..bb4a48b
--- /dev/null
+++ b/pet/ci.py
@@ -0,0 +1,117 @@
+from os import path, makedirs
+import json
+import pet
+import pet.models
+import pet.update
+import requests
+import subprocess
+
+
+def update_ci_status():
+ """The main method.
+ Creates an instance of the database and change the status of each package.
+ It also returns the amount of packages that were updated."""
+ engine = pet.engine(True)
+ connection_instance = engine.connect()
+ result = connection_instance.execute("select * from named_tree")
+ connection_instance.close()
+
+ count_changed_packages = 0
+ for row in result:
+ if row.source is not None:
+ status = found_on_json(row.source)
+ if status is not False:
+ count_changed_packages += 1
+ change_status(row, status)
+ return count_changed_packages
+
+
+def change_status(row, status):
+ """Changes the status column of the package (of given row)."""
+ engine = pet.engine(True)
+ connection_instance = engine.connect()
+
+ script_sql = "update named_tree set status='%s' where id=%s" \
+ % (status, row.id)
+ status = connection_instance.execute(script_sql)
+
+ connection_instance.close()
+
+
+def found_on_json(source):
+ """Searches the packages that are on the database on the json.
+ It returns the status of each found package."""
+ data = get_ci_debian_json()
+ for package in data:
+ package_name = package["package"]
+ if package_name == source:
+ return package["status"]
+ return False
+
+
+def get_packages_from_url(url):
+ """Downloads the json file."""
+ try:
+ print "Downloading json file with ci debian status..."
+ request_of_ci_debian = requests.get(url)
+ file_ci_debian = request_of_ci_debian.content
+ file_status = open("pet/.debian_ci/packages.json", 'w')
+ file_status.write(file_ci_debian)
+ file_status.close()
+ except IOError:
+ print "File package.json coundn't open"
+ except:
+ print "Coundn't acess json of packages on url: ", url
+
+
+def create_dir():
+ """This method creates the directory where the .debian_ci will be stored."""
+ dir_path = path.abspath(path.dirname(__file__))
+ dir_path += "/.debian_ci"
+ if not path.exists(dir_path):
+ makedirs(dir_path)
+ return dir_path
+
+
+def get_ETag(url):
+ """Verifies if the content of the json was modified and need to be
+ downloaded again."""
+ bashCommand = "HEAD %s | grep ETag" % url
+ output = subprocess.check_output(['bash', '-c', bashCommand])
+ ETAG = slice(7, -2, 1)
+ ETag = output[ETAG]
+
+ method_response = False
+ try:
+ file_version = open("pet/.debian_ci/etag_version", "r")
+ file_read = file_version.read()
+ if ETag != file_read:
+ method_response = True
+ except IOError:
+ file_version = open("pet/.debian_ci/etag_version", "w+")
+ file_version.write(ETag)
+ method_response = True
+
+ file_version.close()
+ return method_response
+
+
+def get_ci_debian_json():
+ """Verifies if the file exists and if not, downloads and parses the json."""
+ data = None
+ url = "https://ci.debian.net/data/status/unstable/amd64/packages.json"
+ create_dir()
+
+ if not get_ETag(url):
+ try:
+ packages_status = open('pet/.debian_ci/packages.json')
+ data = json.load(packages_status)
+ packages_status.close()
+ except IOError, e:
+ print e
+ else:
+ print("Outdated version of ci")
+ get_packages_from_url(url)
+ data = get_ci_debian_json()
+
+ return data
diff --git a/pet/classifier.py b/pet/classifier.py
index c9619f7..01a0915 100644
--- a/pet/classifier.py
+++ b/pet/classifier.py
@@ -40,6 +40,7 @@ def __init__(self, named_tree, bugs, suite_packages, tags):
last_changed_by = property(lambda self: self.named_tree.last_changed_by)
last_changed = property(lambda self: self.named_tree.last_changed)
todo = property(lambda self: self.named_tree.todo)
+ status = property(lambda self: self.named_tree.status)
@property
def has_rc_bugs(self):
@@ -198,6 +199,11 @@ def classify(self):
else:
cls = 'other'
+ if p.named_tree.status == "fail".decode('unicode-escape'):
+ p.named_tree.status = True
+ else:
+ p.named_tree.status = False
+
package_popcon = popcon.package(p.name)
if package_popcon:
ordered_packages.put((-package_popcon[p.name], cls, p))
diff --git a/pet/sql.py b/pet/sql.py
index 2e49420..b7e4532 100644
--- a/pet/sql.py
+++ b/pet/sql.py
@@ -386,6 +386,7 @@ def run(self, engine, create_database=False):
DBUpdater().add(15, statements=[
"""
ALTER TABLE named_tree
- ADD COLUMN todo BOOLEAN NOT NULL DEFAULT 'f'
+ ADD COLUMN todo BOOLEAN NOT NULL DEFAULT 'f',
+ ADD COLUMN status TEXT
""",
])
diff --git a/pet/web/__init__.py b/pet/web/__init__.py
index fcf2839..762ebae 100644
--- a/pet/web/__init__.py
+++ b/pet/web/__init__.py
@@ -15,6 +15,7 @@
from pyramid.config import Configurator
from pyramid.events import subscriber, NewRequest
+import pet.ci
from pet.models import Session
@@ -31,6 +32,7 @@ def callback(request):
def main(global_config=None, **settings):
# settings.update({
# })
+ pet.ci.update_ci_status()
config = Configurator(settings=settings)
config.include('pyramid_chameleon')
config.add_static_view('static', 'pet.web:static')
diff --git a/pet/web/templates/overview.pt b/pet/web/templates/overview.pt
index ea785d1..2c2ab0a 100644
--- a/pet/web/templates/overview.pt
+++ b/pet/web/templates/overview.pt
@@ -1,158 +1,234 @@
-
-
- Package Entropy Tracker
-
-
-
-
-
-
-
- Skip Navigation
-
-
-
-
-
-
-
-
+ Skip navigation for text only browsers
+ -->
+ Skip Navigation
+
+
+
+
+
+
+
+
+ CI Packages
+
+ Package Repository Archive Bugs Upstream
+
+
+
diff --git a/pet/web/views.py b/pet/web/views.py
index 8cc1d37..5f74f8b 100644
--- a/pet/web/views.py
+++ b/pet/web/views.py
@@ -51,7 +51,7 @@ def __call__(self):
return {
"classified": classifier.classify(),
- "classes": classifier.classes()
+ "classes": classifier.classes(),
}
diff --git a/test/test_ci_debian.py b/test/test_ci_debian.py
new file mode 100644
index 0000000..4862812
--- /dev/null
+++ b/test/test_ci_debian.py
@@ -0,0 +1,31 @@
+import unittest
+import pet.ci as ci
+
+
+class TestPetCi(unittest.TestCase):
+
+ def test_if_file_is_registered(self):
+ url = "https://ci.debian.net/data/status/unstable/amd64/packages.json"
+ ci.get_packages_from_url(url)
+
+ try:
+ file = open("pet/.debian_ci/packages.json", "r")
+ except IOError:
+ file = None
+
+ self.assertNotEqual(None, file)
+
+ def test_if_data_is_a_list(self):
+ data = ci.get_ci_debian_json()
+
+ self.assertEquals(isinstance(data, list), True)
+
+ def test_if_element_is_dict(self):
+ data = ci.get_ci_debian_json()
+ package = data[0]
+
+ self.assertEquals(isinstance(package, dict), True)
+
+ def test_file_not_found(self):
+ response = ci.found_on_json("without_this_package")
+ self.assertEquals(response, False)
diff --git a/update-archive b/update-archive
index 4dd63ce..5bef0c2 100755
--- a/update-archive
+++ b/update-archive
@@ -17,11 +17,15 @@
import pet.models
import pet.update
+import pet.ci
import sys
def main(argv):
+ number_of_updates = pet.ci.update_ci_status()
+ print "%s number(s) of status of packages was updated" % number_of_updates
+
session = pet.models.Session()
for archive_name in argv[1:]:
archives = session.query(pet.models.Archive).filter_by(