diff --git a/13.0.Dockerfile b/13.0.Dockerfile index 9a89ec1b..2ff74aae 100644 --- a/13.0.Dockerfile +++ b/13.0.Dockerfile @@ -13,6 +13,10 @@ ENV DB_FILTER=.* \ GEOIP_ACCOUNT_ID="" \ GEOIP_LICENSE_KEY="" \ GIT_AUTHOR_NAME=docker-odoo \ + GIT_AUTHOSHARE=0 \ + GIT_AUTOSHARE_CACHE_DIR="/home/odoo/.cache/git-autoshare/" \ + GIT_AUTOSHARE_CONFIG_DIR="/home/odoo/.config/git-autoshare/" \ + GIT_AUTOSHARE_ORGS_TO_CACHE="OCA,odoo,Tecnativa" \ INITIAL_LANG="" \ LC_ALL=C.UTF-8 \ LIST_DB=false \ @@ -126,21 +130,23 @@ RUN build_deps=" \ -r https://raw.githubusercontent.com/$ODOO_SOURCE/$ODOO_VERSION/requirements.txt \ 'websocket-client~=0.56' \ astor \ - git-aggregator \ + git+https://github.com/Tecnativa/git-aggregator@use-git-autoshare \ + git+https://github.com/Tecnativa/git-autoshare@bundle-git-script \ # Install fix from https://github.com/acsone/click-odoo-contrib/pull/93 git+https://github.com/Tecnativa/click-odoo-contrib.git@fix-active-modules-hashing \ "pg_activity<2.0.0" \ + debugpy \ + geoip2 \ + inotify \ phonenumbers \ plumbum \ ptvsd \ - debugpy \ - pydevd-odoo \ pudb \ + pydevd-odoo \ python-magic \ + pyyaml \ watchdog \ wdb \ - geoip2 \ - inotify \ && (python3 -m compileall -q /usr/local/lib/python3.6/ || true) \ && apt-get purge -yqq $build_deps \ && apt-get autopurge -yqq \ diff --git a/14.0.Dockerfile b/14.0.Dockerfile index cd03d583..a8adfa8d 100644 --- a/14.0.Dockerfile +++ b/14.0.Dockerfile @@ -12,6 +12,10 @@ ENV DB_FILTER=.* \ GEOIP_ACCOUNT_ID="" \ GEOIP_LICENSE_KEY="" \ GIT_AUTHOR_NAME=docker-odoo \ + GIT_AUTHOSHARE=0 \ + GIT_AUTOSHARE_CACHE_DIR="/home/odoo/.cache/git-autoshare/" \ + GIT_AUTOSHARE_CONFIG_DIR="/home/odoo/.config/git-autoshare/" \ + GIT_AUTOSHARE_ORGS_TO_CACHE="OCA,odoo,Tecnativa" \ INITIAL_LANG="" \ LC_ALL=C.UTF-8 \ LIST_DB=false \ @@ -127,7 +131,8 @@ RUN build_deps=" \ git+https://github.com/Tecnativa/click-odoo-contrib.git@fix-active-modules-hashing \ debugpy \ geoip2 \ - git-aggregator \ + git+https://github.com/Tecnativa/git-aggregator@use-git-autoshare \ + git+https://github.com/Tecnativa/git-autoshare@bundle-git-script \ inotify \ pdfminer.six \ pg_activity \ @@ -135,6 +140,7 @@ RUN build_deps=" \ plumbum \ pudb \ python-magic \ + pyyaml \ watchdog \ wdb \ && (python3 -m compileall -q /usr/local/lib/python3.8/ || true) \ diff --git a/15.0.Dockerfile b/15.0.Dockerfile index 4852e7a0..0096ac99 100644 --- a/15.0.Dockerfile +++ b/15.0.Dockerfile @@ -6,12 +6,18 @@ ARG GEOIP_UPDATER_VERSION=4.3.0 ARG WKHTMLTOPDF_VERSION=0.12.5 ARG WKHTMLTOPDF_CHECKSUM='dfab5506104447eef2530d1adb9840ee3a67f30caaad5e9bcb8743ef2f9421bd' ENV DB_FILTER=.* \ + DEBUGPY_ARGS="--listen 0.0.0.0:6899 --wait-for-client" \ + DEBUGPY_ENABLE=0 \ DEPTH_DEFAULT=1 \ DEPTH_MERGE=100 \ EMAIL=https://hub.docker.com/r/tecnativa/odoo \ GEOIP_ACCOUNT_ID="" \ GEOIP_LICENSE_KEY="" \ GIT_AUTHOR_NAME=docker-odoo \ + GIT_AUTHOSHARE=0 \ + GIT_AUTOSHARE_CACHE_DIR="/home/odoo/.cache/git-autoshare/" \ + GIT_AUTOSHARE_CONFIG_DIR="/home/odoo/.config/git-autoshare/" \ + GIT_AUTOSHARE_ORGS_TO_CACHE="OCA,odoo,Tecnativa" \ INITIAL_LANG="" \ LC_ALL=C.UTF-8 \ LIST_DB=false \ @@ -19,8 +25,6 @@ ENV DB_FILTER=.* \ OPENERP_SERVER=/opt/odoo/auto/odoo.conf \ PATH="/home/odoo/.local/bin:$PATH" \ PIP_NO_CACHE_DIR=0 \ - DEBUGPY_ARGS="--listen 0.0.0.0:6899 --wait-for-client" \ - DEBUGPY_ENABLE=0 \ PUDB_RDB_HOST=0.0.0.0 \ PUDB_RDB_PORT=6899 \ PYTHONOPTIMIZE=1 \ @@ -125,7 +129,8 @@ RUN build_deps=" \ git+https://github.com/Tecnativa/click-odoo-contrib.git@fix-active-modules-hashing \ debugpy \ geoip2 \ - git-aggregator \ + git+https://github.com/Tecnativa/git-aggregator@use-git-autoshare \ + git+https://github.com/Tecnativa/git-autoshare@bundle-git-script \ inotify \ pdfminer.six \ pg_activity \ @@ -133,6 +138,7 @@ RUN build_deps=" \ plumbum \ pudb \ python-magic \ + pyyaml \ watchdog \ wdb \ && (python3 -m compileall -q /usr/local/lib/python3.8/ || true) \ diff --git a/README.md b/README.md index 4a4f4fdd..705d5fe9 100644 --- a/README.md +++ b/README.md @@ -594,6 +594,50 @@ not expose the debugging port to attackers**. Usage: We found this one to be the most useful tool for downlading code, merging it and placing it somewhere. +It can also take advantage of a shared repository cache in your local machine, making +use of [git-autoshare](https://github.com/acsone/git-autoshare). + +To do so, you must map a local folder on the host machine to the container in the +aggregation process and set the `GIT_AUTOSHARE=1` in the container. + +The folder that will store the cache in the container will be +`/home/odoo/.cache/git-autoshare/`, but you can change it by providing a different value +through the `GIT_AUTOSHARE_CACHE_DIR` environment variable. Then, to map the local cache +folder to the container, you can add the following line to the `volumes` section of the +`odoo` service in the `docker-compose.yaml` file: + +```yaml +- ~/.cache/git-autoshare/github.com/:/home/odoo/.cache/git-autoshare/ +``` + +This will allow you to re-use the same cache for other git operations in the host +machine, if you have `git-autoshare` configured. + +`git-autoshare` will also need a configuration file, which should be placed in +`/home/odoo/.config/git-autoshare/repos.yml` (or another directory by changing the +`GIT_AUTOSHARE_CONFIG_DIR` environment variable). You can check the documentation for +more details, but you can either map a local file to the container containing this +configuration, or pass a comma-separated list of Github Organizations through the +`GIT_AUTOSHARE_ORGS_TO_CACHE` environment variable, which will make Doodba generate that +file automatically with the following structure: + +``` +GIT_AUTOSHARE_ORGS_TO_CACHE=Tecnativa,odoo,OCA +``` + +`/home/odoo/.config/git-autoshare/repos.yml`: + +``` +github.com: + "*": + orgs: + - odoo + - OCA + - Tecnativa +``` + +This will generate a cache for all repos in those 3 Github Organizations. + ### `autoaggregate` This little script wraps `git-aggregator` to make it work fine and automatically with diff --git a/build.d/100-repos-aggregate b/build.d/100-repos-aggregate index 67508d2a..310a3ac0 100755 --- a/build.d/100-repos-aggregate +++ b/build.d/100-repos-aggregate @@ -4,6 +4,8 @@ set -e # make sure odoo has a user.name configured, as merges would not succeed otherwise # (even if GIT_AUTHOR_NAME and EMAIL are set and should be used, it seems gitaggregate is not passing them to git) su --shell="$SHELL" odoo -c 'git config user.name 1>/dev/null || git config --global user.name "'"$GIT_AUTHOR_NAME"'"' +# set "master" as the default branch name to supress warnings +su --shell="$SHELL" odoo -c 'git config init.defaultBranch 1>/dev/null || git config --global init.defaultBranch master' # copy ssh directory to odoo user as well (gitaggregate may also be run as odoo user) if [[ ! -e ~odoo/.ssh ]] ; then diff --git a/build.d/50-configure-git-autoshare b/build.d/50-configure-git-autoshare new file mode 100755 index 00000000..2d07b345 --- /dev/null +++ b/build.d/50-configure-git-autoshare @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""Generates the appropiate configuration file for git-autoshare according to +user-defined settings through environment variables +""" + +import os +from pathlib import Path + +import yaml + +GIT_AUTOSHARE_ORGS_TO_CACHE = os.environ.get( + "GIT_AUTOSHARE_ORGS_TO_CACHE", "OCA,odoo,Tecnativa" +) +GIT_AUTOSHARE_CONFIG_DIR = os.environ.get( + "GIT_AUTOSHARE_CONFIG_DIR", "/home/odoo/.config/git-autoshare/" +) +GIT_AUTOSHARE_CACHE_DIR = os.environ.get( + "GIT_AUTOSHARE_CACHE_DIR", "/home/odoo/.cache/git-autoshare/" +) +UID = int(os.environ.get("UID", "1000")) +GID = int(os.environ.get("GID", "1000")) + +orgs_to_cache = GIT_AUTOSHARE_ORGS_TO_CACHE.split(",") +git_autoshare_config_path = "%srepos.yml" % GIT_AUTOSHARE_CONFIG_DIR + +if Path(git_autoshare_config_path).is_file(): + exit(0) +else: + Path(GIT_AUTOSHARE_CONFIG_DIR).mkdir(parents=True) + os.chown(GIT_AUTOSHARE_CONFIG_DIR, UID, GID) + +git_autoshare_config = {"github.com": {"*": {"orgs": orgs_to_cache,}}} + +with open(git_autoshare_config_path, "w") as f: + f.write(yaml.dump(git_autoshare_config)) +os.chown(git_autoshare_config_path, UID, GID) + +# Ensure cache dir exists and has correct permissions +Path(GIT_AUTOSHARE_CACHE_DIR).mkdir(parents=True, exist_ok=True) +os.chown(GIT_AUTOSHARE_CACHE_DIR, UID, GID) diff --git a/tests/__init__.py b/tests/__init__.py index a74ae88e..2c2f01b4 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -711,6 +711,36 @@ def test_aggregate_permissions(self): ("autoaggregate",), ) + def test_aggregate_autoshare(self): + symlink_dir = join(SCAFFOLDINGS_DIR, "repo_merge") + for sub_env in matrix(odoo_skip={"11.0", "12.0"}): + self.compose_test( + symlink_dir, + dict( + sub_env, UID=str(os.getuid()), GID=str(os.getgid()), GIT_AUTOSHARE=1 + ), + # Ensure no cache exists prior to 1st aggregation + ("rm", "-rf", "/home/odoo/.cache/git-autoshare/github.com"), + ("test", "!", "-d", "/home/odoo/.cache/git-autoshare/github.com/ocb"), + # The git-autoshare config file should have been correctly generated + ("test", "-f", "/home/odoo/.config/git-autoshare/repos.yml"), + ("autoaggregate",), + # Cache should have been generated + ("test", "-d", "/home/odoo/.cache/git-autoshare/github.com/ocb"), + # Cloned repo should reference to cache + ( + "test", + "-f", + "/opt/odoo/custom/src/odoo/.git/objects/info/alternates", + ), + ( + "test", + "(cat /opt/odoo/custom/src/odoo/.git/objects/info/alternates)", + "=", + "/home/odoo/.cache/git-autoshare/github.com/ocb/objects", + ), + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/scaffoldings/repo_merge/docker-compose.yaml b/tests/scaffoldings/repo_merge/docker-compose.yaml index de4ff6b2..b1b59675 100644 --- a/tests/scaffoldings/repo_merge/docker-compose.yaml +++ b/tests/scaffoldings/repo_merge/docker-compose.yaml @@ -10,6 +10,7 @@ services: environment: PYTHONOPTIMIZE: "" UNACCENT: "false" + GIT_AUTOSHARE: "${GIT_AUTOSHARE:-0}" entrypoint: "" # we are not using odoo but only checking the repo state here, this skips starting the db volumes: - filestore:/var/lib/odoo:z diff --git a/tests/scaffoldings/repo_merge/setup-devel.yaml b/tests/scaffoldings/repo_merge/setup-devel.yaml index 50620cc1..53e53112 100644 --- a/tests/scaffoldings/repo_merge/setup-devel.yaml +++ b/tests/scaffoldings/repo_merge/setup-devel.yaml @@ -15,6 +15,7 @@ services: UID: "${UID:-1000}" GID: "${GID:-1000}" UMASK: "$UMASK" + GIT_AUTOSHARE: "${GIT_AUTOSHARE:-0}" user: root entrypoint: "" # we are only calling autoaggregate inside the tests volumes: