diff --git a/.gitignore b/.gitignore index e0a48fa..ae5d6d3 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,6 @@ docs/_build/ .vagrant/ nodes/vagrant* + +# Json to store packages of ci_debian info +packages.json diff --git a/pet/ci.py b/pet/ci.py new file mode 100644 index 0000000..bb4a48b --- /dev/null +++ b/pet/ci.py @@ -0,0 +1,117 @@ +from os import path, makedirs +import json +import pet +import pet.models +import pet.update +import requests +import subprocess + + +def update_ci_status(): + """The main method. + Creates an instance of the database and change the status of each package. + It also returns the amount of packages that were updated.""" + engine = pet.engine(True) + connection_instance = engine.connect() + result = connection_instance.execute("select * from named_tree") + connection_instance.close() + + count_changed_packages = 0 + for row in result: + if row.source is not None: + status = found_on_json(row.source) + if status is not False: + count_changed_packages += 1 + change_status(row, status) + return count_changed_packages + + +def change_status(row, status): + """Changes the status column of the package (of given row).""" + engine = pet.engine(True) + connection_instance = engine.connect() + + script_sql = "update named_tree set status='%s' where id=%s" \ + % (status, row.id) + status = connection_instance.execute(script_sql) + + connection_instance.close() + + +def found_on_json(source): + """Searches the packages that are on the database on the json. + It returns the status of each found package.""" + data = get_ci_debian_json() + for package in data: + package_name = package["package"] + if package_name == source: + return package["status"] + return False + + +def get_packages_from_url(url): + """Downloads the json file.""" + try: + print "Downloading json file with ci debian status..." + request_of_ci_debian = requests.get(url) + file_ci_debian = request_of_ci_debian.content + file_status = open("pet/.debian_ci/packages.json", 'w') + file_status.write(file_ci_debian) + file_status.close() + except IOError: + print "File package.json coundn't open" + except: + print "Coundn't acess json of packages on url: ", url + + +def create_dir(): + """This method creates the directory where the .debian_ci will be stored.""" + dir_path = path.abspath(path.dirname(__file__)) + dir_path += "/.debian_ci" + if not path.exists(dir_path): + makedirs(dir_path) + return dir_path + + +def get_ETag(url): + """Verifies if the content of the json was modified and need to be + downloaded again.""" + bashCommand = "HEAD %s | grep ETag" % url + output = subprocess.check_output(['bash', '-c', bashCommand]) + ETAG = slice(7, -2, 1) + ETag = output[ETAG] + + method_response = False + try: + file_version = open("pet/.debian_ci/etag_version", "r") + file_read = file_version.read() + if ETag != file_read: + method_response = True + except IOError: + file_version = open("pet/.debian_ci/etag_version", "w+") + file_version.write(ETag) + method_response = True + + file_version.close() + return method_response + + +def get_ci_debian_json(): + """Verifies if the file exists and if not, downloads and parses the json.""" + data = None + url = "https://ci.debian.net/data/status/unstable/amd64/packages.json" + create_dir() + + if not get_ETag(url): + try: + packages_status = open('pet/.debian_ci/packages.json') + data = json.load(packages_status) + packages_status.close() + except IOError, e: + print e + else: + print("Outdated version of ci") + get_packages_from_url(url) + data = get_ci_debian_json() + + return data diff --git a/pet/classifier.py b/pet/classifier.py index c9619f7..01a0915 100644 --- a/pet/classifier.py +++ b/pet/classifier.py @@ -40,6 +40,7 @@ def __init__(self, named_tree, bugs, suite_packages, tags): last_changed_by = property(lambda self: self.named_tree.last_changed_by) last_changed = property(lambda self: self.named_tree.last_changed) todo = property(lambda self: self.named_tree.todo) + status = property(lambda self: self.named_tree.status) @property def has_rc_bugs(self): @@ -198,6 +199,11 @@ def classify(self): else: cls = 'other' + if p.named_tree.status == "fail".decode('unicode-escape'): + p.named_tree.status = True + else: + p.named_tree.status = False + package_popcon = popcon.package(p.name) if package_popcon: ordered_packages.put((-package_popcon[p.name], cls, p)) diff --git a/pet/sql.py b/pet/sql.py index 2e49420..b7e4532 100644 --- a/pet/sql.py +++ b/pet/sql.py @@ -386,6 +386,7 @@ def run(self, engine, create_database=False): DBUpdater().add(15, statements=[ """ ALTER TABLE named_tree - ADD COLUMN todo BOOLEAN NOT NULL DEFAULT 'f' + ADD COLUMN todo BOOLEAN NOT NULL DEFAULT 'f', + ADD COLUMN status TEXT """, ]) diff --git a/pet/web/__init__.py b/pet/web/__init__.py index fcf2839..762ebae 100644 --- a/pet/web/__init__.py +++ b/pet/web/__init__.py @@ -15,6 +15,7 @@ from pyramid.config import Configurator from pyramid.events import subscriber, NewRequest +import pet.ci from pet.models import Session @@ -31,6 +32,7 @@ def callback(request): def main(global_config=None, **settings): # settings.update({ # }) + pet.ci.update_ci_status() config = Configurator(settings=settings) config.include('pyramid_chameleon') config.add_static_view('static', 'pet.web:static') diff --git a/pet/web/templates/overview.pt b/pet/web/templates/overview.pt index ea785d1..2c2ab0a 100644 --- a/pet/web/templates/overview.pt +++ b/pet/web/templates/overview.pt @@ -1,158 +1,234 @@ - - - Package Entropy Tracker - - - - - -

- - Skip Navigation

- - - -
- - -
-
- - - - - - - - - - - - - -
${cl.name} (${len(classified[cl.key])})
PackageRepositoryArchiveBugsUpstream
a-package - - - 0.1-1 - - ${p.highest_tag.last_changed_by} — ${p.highest_tag.last_changed} - - - - - - (${p.version}) - - This package has a lower version on the repository than in the archive. - - - - - (${p.version}) - - - ${p.last_changed_by} — ${p.last_changed} - - - - - ${p.highest_archive.version} - - - -
- 7 - - - - - -
- #${b.bug_number} -
- forwarded -
-
- - tag, - -
- 		It doesn't work!
-
- 		- -
- ${p.watch.upstream_version} - - (Browse) -
-
- - + Skip navigation for text only browsers + --> + Skip Navigation

+ + + +
+ + +
+
+ + + + + + + + + + + + + +
${cl.name} (${len(classified[cl.key])})
PackageRepositoryArchiveBugsUpstream
a-package + + + 0.1-1 + + ${p.highest_tag.last_changed_by} — ${p.highest_tag.last_changed} + + + + + + (${p.version}) + + This package has a lower version on the repository than in the archive. + + + + + (${p.version}) + + + ${p.last_changed_by} — ${p.last_changed} + + + + + ${p.highest_archive.version} + + + +
+ 7 + + + + + +
+ #${b.bug_number} +
+ forwarded +
+
+ + tag, + +
+ 		It doesn't work!
+
+ 		+ +
+ ${p.watch.upstream_version} + + (Browse) +
+
+ + + + + +
+
+ +
+
+ + + + + + + + +
CI Packages
PackageRepositoryArchiveBugsUpstream
a-package + + + 0.1-1 + + ${p.highest_tag.last_changed_by} — ${p.highest_tag.last_changed} + + + + + + (${p.version}) + + This package has a lower version on the repository than in the archive. + + + + + (${p.version}) + + + ${p.last_changed_by} — ${p.last_changed} + + + + + ${p.highest_archive.version} + + + +
+ 7 + + + + + +
+ #${b.bug_number} +
+ forwarded +
+
+ + tag, + +
+ 		It doesn't work!
+
+ 		+ +
+ ${p.watch.upstream_version} + + (Browse) +
+
+ diff --git a/pet/web/views.py b/pet/web/views.py index 8cc1d37..5f74f8b 100644 --- a/pet/web/views.py +++ b/pet/web/views.py @@ -51,7 +51,7 @@ def __call__(self): return { "classified": classifier.classify(), - "classes": classifier.classes() + "classes": classifier.classes(), } diff --git a/test/test_ci_debian.py b/test/test_ci_debian.py new file mode 100644 index 0000000..4862812 --- /dev/null +++ b/test/test_ci_debian.py @@ -0,0 +1,31 @@ +import unittest +import pet.ci as ci + + +class TestPetCi(unittest.TestCase): + + def test_if_file_is_registered(self): + url = "https://ci.debian.net/data/status/unstable/amd64/packages.json" + ci.get_packages_from_url(url) + + try: + file = open("pet/.debian_ci/packages.json", "r") + except IOError: + file = None + + self.assertNotEqual(None, file) + + def test_if_data_is_a_list(self): + data = ci.get_ci_debian_json() + + self.assertEquals(isinstance(data, list), True) + + def test_if_element_is_dict(self): + data = ci.get_ci_debian_json() + package = data[0] + + self.assertEquals(isinstance(package, dict), True) + + def test_file_not_found(self): + response = ci.found_on_json("without_this_package") + self.assertEquals(response, False) diff --git a/update-archive b/update-archive index 4dd63ce..5bef0c2 100755 --- a/update-archive +++ b/update-archive @@ -17,11 +17,15 @@ import pet.models import pet.update +import pet.ci import sys def main(argv): + number_of_updates = pet.ci.update_ci_status() + print "%s number(s) of status of packages was updated" % number_of_updates + session = pet.models.Session() for archive_name in argv[1:]: archives = session.query(pet.models.Archive).filter_by(