From ece9223fb613e7846e7cc70e4a8a4b81b29b6727 Mon Sep 17 00:00:00 2001 From: Chris Orlando Date: Thu, 9 Apr 2020 01:47:37 -0400 Subject: [PATCH 1/6] update for werkzeug > 1.0 I came across this while working on https://github.com/overviewer/Overviewer-Web/pull/42 but it seemed like its own problem, worthy of its own PR. 9 days ago, [werkzeug was updated](https://github.com/pallets/werkzeug/releases/tag/1.0.0) The changelog contains this: > Remove most top-level attributes provided by the werkzeug module in favor of direct imports. For example, instead of import werkzeug; werkzeug.url_quote, do from werkzeug.urls import url_quote. Install version 0.16 first to see deprecation warnings while upgrading. #2, #1640 So I updated the components that import from werkzeug to versions that have the imports fixed. --- flask/overviewer/models.py | 2 +- flask/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flask/overviewer/models.py b/flask/overviewer/models.py index 278177d..c7787a3 100644 --- a/flask/overviewer/models.py +++ b/flask/overviewer/models.py @@ -1,6 +1,6 @@ from flask import url_for, current_app, session from flask_sqlalchemy import SQLAlchemy -from werkzeug import secure_filename +from werkzeug.utils import secure_filename import os.path import os import hashlib diff --git a/flask/requirements.txt b/flask/requirements.txt index 0d53f68..992cbfa 100644 --- a/flask/requirements.txt +++ b/flask/requirements.txt @@ -3,13 +3,13 @@ gunicorn==19.9.0 Flask-SQLAlchemy==2.3.2 Flask-Migrate==2.4.0 GitHub-Flask==3.2.0 -Flask-WTF==0.14.2 +Flask-WTF==0.14.3 unidecode==0.04.1 Flask-Markdown==0.3 pytoml==0.1.20 Pillow==6.2.0 requests==2.21.0 -Flask-Caching==1.0.0 +Flask-Caching==1.8.0 psycopg2==2.7.7 Flask-DebugToolbar==0.10.1 Flask-Script==2.0.6 From c83c793262b78be25322855348e5506b76d47612 Mon Sep 17 00:00:00 2001 From: Chris Orlando Date: Sat, 7 Mar 2020 22:26:24 -0500 Subject: [PATCH 2/6] Add centos8 builder --- .../docker-builders/centos8-64/Dockerfile | 17 +++++++++ .../centos8-64/worker-start.sh | 18 +++++++++ buildbot/master/master.cfg | 15 +++++--- .../control/centos8/Minecraft-Overviewer.spec | 38 +++++++++++++++++++ buildbot/repos/rpm/repo/Makefile | 2 +- docker-compose.yml | 9 +++++ 6 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 buildbot/docker-builders/centos8-64/Dockerfile create mode 100755 buildbot/docker-builders/centos8-64/worker-start.sh create mode 100644 buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec diff --git a/buildbot/docker-builders/centos8-64/Dockerfile b/buildbot/docker-builders/centos8-64/Dockerfile new file mode 100644 index 0000000..019a177 --- /dev/null +++ b/buildbot/docker-builders/centos8-64/Dockerfile @@ -0,0 +1,17 @@ +FROM centos:8 +RUN yum install -y epel-release +RUN yum --exclude=iputils update -y && yum install -y \ + git curl wget mock rpm-build \ + python36 python3-libs python36-devel python3-pip python3-numpy python3-pillow \ + redhat-rpm-config gcc libffi-devel openssl-devel \ + && yum groupinstall -y "Development Tools" +RUN ln -sf /usr/bin/python3.6 /usr/bin/python3 \ + && ln -sf /usr/bin/pip3.6 /usr/bin/pip3 + +RUN python3 -m pip install "buildbot[bundle]" sphinx + +ENV WORKER_NAME diax-centos8-64 +ENV WORKER_ADMIN Andrew Chin +ENV WORKER_INFO CentOS 8 64-bit +COPY worker-start.sh /root/ +CMD /root/worker-start.sh diff --git a/buildbot/docker-builders/centos8-64/worker-start.sh b/buildbot/docker-builders/centos8-64/worker-start.sh new file mode 100755 index 0000000..ebc26bd --- /dev/null +++ b/buildbot/docker-builders/centos8-64/worker-start.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +cd /root/ +if [ ! -d "worker" ]; then + echo "creating worker..." + buildbot-worker create-worker --umask=0o22 --no-logrotate worker bbmaster:9989 $WORKER_NAME $(python3 -c "import hmac, hashlib; print(hmac.new(bytes('$BUILDBOT_SECRET', 'utf-8'), bytes('$WORKER_NAME', 'utf-8'), hashlib.sha512).hexdigest())") + echo $WORKER_ADMIN > worker/info/admin + echo $WORKER_INFO > worker/info/host + uname -a >> worker/info/host +fi + +rm -f worker/twistd.pid worker/twistd.log + +# make sure /sys is rw (I HAVE NO IDEA but it helps rpm builders) +mount -o remount,rw /sys + +# run in a clean env, don't leak secrets +exec env -i -- /bin/bash -c "source /etc/profile; export HOME=`pwd`; exec buildbot-worker start --nodaemon worker" diff --git a/buildbot/master/master.cfg b/buildbot/master/master.cfg index 00c7814..a262f0d 100644 --- a/buildbot/master/master.cfg +++ b/buildbot/master/master.cfg @@ -52,9 +52,9 @@ class BaseConfig: DEFAULT_CLIENT_JAR_VER = "1.15" ALL_BUILDERS = ['src', 'render', 'win32', - 'win64', 'deb32', 'deb64', 'centos7-64'] + 'win64', 'deb32', 'deb64', 'centos7-64', 'centos8-64'] ENABLED_WORKERS = ['diax-stretch-64', - 'diax-stretch-32', 'ec2-windows', 'diax-centos7-64'] + 'diax-stretch-32', 'ec2-windows', 'diax-centos7-64', 'diax-centos8-64'] # high-level worker list # workers[name] = [... list of builders ...] @@ -62,6 +62,7 @@ class BaseConfig: 'diax-stretch-64': ['src', 'render', 'deb64'], 'diax-stretch-32': ['src', 'render', 'deb32'], 'diax-centos7-64': ['centos7-64'], + 'diax-centos8-64': ['centos8-64'], 'ec2-windows': ['win32', 'win64'], } @@ -79,8 +80,8 @@ class DevConfig(BaseConfig): BASE_URL = "http://localhost:8020" ENABLE_GITHUB_AUTH = os.environ.get("DISABLE_GITHUB_AUTH") is None - ALL_BUILDERS = ['src', 'render', 'deb32', 'deb64', 'centos7-64'] - ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32', 'diax-centos7-64'] + ALL_BUILDERS = ['src', 'render', 'deb32', 'deb64', 'centos7-64', 'centos8-64'] + ENABLED_WORKERS = ['diax-stretch-64', 'diax-stretch-32', 'diax-centos7-64', 'diax-centos8-64'] ENV = os.environ.get("ENV", "production") @@ -563,7 +564,7 @@ def rpm(rpmbase, mockbase, mockconfig, rpmarch, mockarch): yield MasterShellCommand(command=["/root/rpmsign.sh", upload_dest(".rpm")], name="sign package", description="signing", descriptionDone="signed") # link the new rpm into the repo - system_map = {'centos6': '6', 'centos7': '7'} + system_map = {'centos6': '6', 'centos7': '7', 'centos8': '8'} rpm_package_area = RPM_REPO / system_map[rpmbase] / rpmarch / "packages" yield MasterShellCommand(command=["ln", "-f", upload_dest(".rpm"), str(rpm_package_area)], name="link into repo", doStepIf=is_release_build, description="linking into repo", descriptionDone="linked into repo") @@ -584,6 +585,10 @@ def centos6_64(): def centos7_64(): yield rpm('centos7', 'el7', 'epel-7', 'x86_64', 'x86_64') +@builder(locks=[rpm_build_lock.access('exclusive')]) +def centos8_64(): + yield rpm('centos8', 'el8', 'epel-8', 'x86_64', 'x86_64') + class RenderDirUpload(DirectoryUpload): def __init__(self, *args, **kwargs): diff --git a/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec new file mode 100644 index 0000000..c20ad2f --- /dev/null +++ b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec @@ -0,0 +1,38 @@ +Summary: Generates large resolution images of a Minecraft map. +Name: Minecraft-Overviewer +Version: {VERSION} +Release: 1%{?dist} +Source0: %{name}-%{version}.tar.gz +License: GNU General Public License v3 +Group: Development/Libraries +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot +Vendor: Andrew Brown +Url: http://overviewer.org/ +Requires: epel-release python3-pillow python3-numpy +BuildRequires: epel-release git python36-devel python3-numpy + +%description +The Minecraft Overviewer is a command-line tool for rendering high-resolution +maps of Minecraft worlds. It generates a set of static html and image files and +uses the Google Maps API to display a nice interactive map. + +%prep +git clone https://github.com/python-pillow/Pillow.git %{_tmppath}/Pillow +%setup -n %{name} + +%build +env CFLAGS="$RPM_OPT_FLAGS" PIL_INCLUDE_DIR=%{_tmppath}/Pillow/src/libImaging %{__python3} setup.py build + +%install +%{__python3} setup.py install -O1 --root=%{buildroot} +rm -rf %{buildroot}%{_defaultdocdir}/minecraft-overviewer + +%clean +rm -rf %{buildroot} + +%files +%defattr(-,root,root) +%{python3_sitearch}/Minecraft_Overviewer-*-*.egg-info +%{python3_sitearch}/overviewer_core +%{_bindir}/overviewer.py +%doc README.rst COPYING.txt sample_config.py diff --git a/buildbot/repos/rpm/repo/Makefile b/buildbot/repos/rpm/repo/Makefile index 7d6acb0..3554dca 100644 --- a/buildbot/repos/rpm/repo/Makefile +++ b/buildbot/repos/rpm/repo/Makefile @@ -1,7 +1,7 @@ .PHONY : all -all : 6/i386.repo 6/x86_64.repo 7/i386.repo 7/x86_64.repo +all : 6/i386.repo 6/x86_64.repo 7/i386.repo 7/x86_64.repo 8/x86_64.repo %.repo : createrepo -s sha $* diff --git a/docker-compose.yml b/docker-compose.yml index 3a65443..1ddc9f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,6 +61,15 @@ services: - apparmor:unconfined links: - bbmaster + bbcentos864: + build: buildbot/docker-builders/centos8-64 + env_file: confidential.env + cap_add: + - SYS_ADMIN + security_opt: + - apparmor:unconfined + links: + - bbmaster bbstretch64: build: buildbot/docker-builders/stretch-64 env_file: confidential.env From 31d49a0c83fba6dabb8405356bc8a097a5b2febf Mon Sep 17 00:00:00 2001 From: Chris Orlando Date: Mon, 6 Apr 2020 19:42:54 -0400 Subject: [PATCH 3/6] http -> https --- buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec index c20ad2f..43ffc4b 100644 --- a/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec +++ b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec @@ -7,7 +7,7 @@ License: GNU General Public License v3 Group: Development/Libraries BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot Vendor: Andrew Brown -Url: http://overviewer.org/ +Url: https://overviewer.org/ Requires: epel-release python3-pillow python3-numpy BuildRequires: epel-release git python36-devel python3-numpy From bc5eea5d3d5703836acf6e07f4fe10dc38327a4f Mon Sep 17 00:00:00 2001 From: Chris Orlando Date: Mon, 6 Apr 2020 19:44:24 -0400 Subject: [PATCH 4/6] Google Maps -> Leaflet --- buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec index 43ffc4b..e3c84cb 100644 --- a/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec +++ b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec @@ -14,7 +14,7 @@ BuildRequires: epel-release git python36-devel python3-numpy %description The Minecraft Overviewer is a command-line tool for rendering high-resolution maps of Minecraft worlds. It generates a set of static html and image files and -uses the Google Maps API to display a nice interactive map. +uses the Leaflet javascript library to display a nice interactive map. %prep git clone https://github.com/python-pillow/Pillow.git %{_tmppath}/Pillow From 455676db5dbd67b7a4195d10c6019c673a9a8d21 Mon Sep 17 00:00:00 2001 From: Chris Orlando Date: Mon, 13 Apr 2020 20:58:10 -0400 Subject: [PATCH 5/6] remove epel-release from el8 rpm requirements --- buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec index e3c84cb..93ee900 100644 --- a/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec +++ b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec @@ -8,8 +8,8 @@ Group: Development/Libraries BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot Vendor: Andrew Brown Url: https://overviewer.org/ -Requires: epel-release python3-pillow python3-numpy -BuildRequires: epel-release git python36-devel python3-numpy +Requires: python3-pillow python3-numpy +BuildRequires: git python36-devel python3-numpy %description The Minecraft Overviewer is a command-line tool for rendering high-resolution From 61aaa5b21066b305a77ea84707cfa9ce397da2bc Mon Sep 17 00:00:00 2001 From: Chris Orlando Date: Mon, 13 Apr 2020 21:21:07 -0400 Subject: [PATCH 6/6] use specific version of Pillow instead of master --- .../repos/rpm/control/centos8/Minecraft-Overviewer.spec | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec index 93ee900..fc9a755 100644 --- a/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec +++ b/buildbot/repos/rpm/control/centos8/Minecraft-Overviewer.spec @@ -9,7 +9,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot Vendor: Andrew Brown Url: https://overviewer.org/ Requires: python3-pillow python3-numpy -BuildRequires: git python36-devel python3-numpy +BuildRequires: wget python36-devel python3-numpy %description The Minecraft Overviewer is a command-line tool for rendering high-resolution @@ -17,11 +17,12 @@ maps of Minecraft worlds. It generates a set of static html and image files and uses the Leaflet javascript library to display a nice interactive map. %prep -git clone https://github.com/python-pillow/Pillow.git %{_tmppath}/Pillow +wget -P %{_tmppath} https://github.com/python-pillow/Pillow/archive/5.1.0.tar.gz +tar -xf %{_tmppath}/5.1.0.tar.gz --directory %{_tmppath} %setup -n %{name} %build -env CFLAGS="$RPM_OPT_FLAGS" PIL_INCLUDE_DIR=%{_tmppath}/Pillow/src/libImaging %{__python3} setup.py build +env CFLAGS="$RPM_OPT_FLAGS" PIL_INCLUDE_DIR=%{_tmppath}/Pillow-5.1.0/src/libImaging %{__python3} setup.py build %install %{__python3} setup.py install -O1 --root=%{buildroot}