diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index dd022fc786..406bc61b2c 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1339,7 +1339,14 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals default_values.update({key: value[0] for key, value in self.extra_options.items()}) self.generate_template_values() - templ_const = {quote_py_str(value): name for name, (value, _) in TEMPLATE_CONSTANTS.items()} + + # take into account that some values may be lists + templ_const = {} + for name, (value, _) in TEMPLATE_CONSTANTS.items(): + if isinstance(value, list): + templ_const.update({quote_py_str(v): name for v in value}) + else: + templ_const.update({quote_py_str(value): name}) # create reverse map of templates, to inject template values where possible # longer template values are considered first, shorter template keys get preference over longer ones diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 46fe4c5970..9f5b0ab974 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -115,15 +115,15 @@ TEMPLATE_CONSTANTS = { # source url constants 'APACHE_SOURCE': ('https://archive.apache.org/dist/%(namelower)s', - 'apache.org source url'), + 'apache.org source URL'), 'BITBUCKET_SOURCE': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get', - 'bitbucket.org source url ' + 'bitbucket.org source URL ' '(namelower is used if bitbucket_account easyconfig parameter is not specified)'), 'BITBUCKET_DOWNLOADS': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads', - 'bitbucket.org downloads url ' + 'bitbucket.org downloads URL ' '(namelower is used if bitbucket_account easyconfig parameter is not specified)'), 'CRAN_SOURCE': ('https://cran.r-project.org/src/contrib', - 'CRAN (contrib) source url'), + 'CRAN (contrib) source URL'), 'FTPGNOME_SOURCE': ('https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s', 'http download for gnome ftp server'), 'GITHUB_SOURCE': ('https://github.com/%(github_account)s/%(name)s/archive', @@ -139,33 +139,36 @@ 'GitHub release URL with lowercase name (if github_account easyconfig ' 'parameter is not specified, namelower is used in its place)'), 'GNU_SAVANNAH_SOURCE': ('https://download-mirror.savannah.gnu.org/releases/%(namelower)s', - 'download.savannah.gnu.org source url'), - 'GNU_SOURCE': ('https://ftpmirror.gnu.org/gnu/%(namelower)s', - 'gnu.org source url (ftp mirror)'), - 'GNU_FTP_SOURCE': ('https://ftp.gnu.org/gnu/%(namelower)s', - 'gnu.org source url (main ftp)'), + 'download.savannah.gnu.org source URL'), + 'GNU_FTP_URLS': (['https://ftpmirror.gnu.org/gnu/%(namelower)s', 'https://ftp.gnu.org/gnu/%(namelower)s'], + 'gnu.org source URLs (ftp mirror and main)'), 'GOOGLECODE_SOURCE': ('http://%(namelower)s.googlecode.com/files', - 'googlecode.com source url'), + 'googlecode.com source URL'), 'LAUNCHPAD_SOURCE': ('https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/', - 'launchpad.net source url'), + 'launchpad.net source URL'), 'PYPI_SOURCE': ('https://pypi.python.org/packages/source/%(nameletter)s/%(name)s', - 'pypi source url'), # e.g., Cython, Sphinx + 'pypi source URL'), # e.g., Cython, Sphinx 'PYPI_LOWER_SOURCE': ('https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s', - 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ + 'pypi source URL (lowercase name)'), # e.g., Greenlet, PyZMQ 'R_SOURCE': ('https://cran.r-project.org/src/base/R-%(version_major)s', - 'cran.r-project.org (base) source url'), + 'cran.r-project.org (base) source URL'), 'SOURCEFORGE_SOURCE': ('https://download.sourceforge.net/%(namelower)s', - 'sourceforge.net source url'), - 'XORG_DATA_SOURCE': ('https://xorg.freedesktop.org/archive/individual/data/', - 'xorg data source url'), - 'XORG_LIB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/lib/', - 'xorg lib source url'), - 'XORG_PROTO_SOURCE': ('https://xorg.freedesktop.org/archive/individual/proto/', - 'xorg proto source url'), - 'XORG_UTIL_SOURCE': ('https://xorg.freedesktop.org/archive/individual/util/', - 'xorg util source url'), - 'XORG_XCB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/xcb/', - 'xorg xcb source url'), + 'sourceforge.net source URL'), + 'XORG_DATA_URLS': (['https://www.x.org/archive/individual/data/', + 'https://xorg.freedesktop.org/archive/individual/data/'], + 'xorg data source URLs'), + 'XORG_LIB_URLS': (['https://www.x.org/archive/individual/lib/', + 'https://xorg.freedesktop.org/archive/individual/lib/'], + 'xorg lib source URLs'), + 'XORG_PROTO_URLS': (['https://www.x.org/archive/individual/proto/', + 'https://xorg.freedesktop.org/archive/individual/proto/'], + 'xorg proto source URLs'), + 'XORG_UTIL_URLS': (['https://www.x.org/archive/individual/util/', + 'https://xorg.freedesktop.org/archive/individual/util/'], + 'xorg util source URLs'), + 'XORG_XCB_URLS': (['https://www.x.org/archive/individual/xcb/', + 'https://xorg.freedesktop.org/archive/individual/xcb/'], + 'xorg xcb source URLs'), # TODO, not urgent, yet nice to have: # CPAN_SOURCE GNOME KDE_I18N XCONTRIB DEBIAN KDE GENTOO TEX_CTAN MOZILLA_ALL @@ -224,19 +227,12 @@ 'GITHUB_RELEASE_URL': 'GITHUB_RELEASE', 'GITHUB_RELEASE_URL_LOWER': 'GITHUB_LOWER_RELEASE', 'GNU_SAVANNAH_URL': 'GNU_SAVANNAH_SOURCE', - 'GNU_FTP_URL': 'GNU_FTP_SOURCE', - 'GNU_URL': 'GNU_SOURCE', 'GOOGLECODE_URL': 'GOOGLECODE_SOURCE', 'LAUNCHPAD_URL': 'LAUNCHPAD_SOURCE', 'PYPI_URL': 'PYPI_SOURCE', 'PYPI_URL_LOWER': 'PYPI_LOWER_SOURCE', 'R_URL': 'R_SOURCE', 'SOURCEFORGE_URL': 'SOURCEFORGE_SOURCE', - 'XORG_DATA_URL': 'XORG_DATA_SOURCE', - 'XORG_LIB_URL': 'XORG_LIB_SOURCE', - 'XORG_PROTO_URL': 'XORG_PROTO_SOURCE', - 'XORG_UTIL_URL': 'XORG_UTIL_SOURCE', - 'XORG_XCB_URL': 'XORG_XCB_SOURCE', 'SOURCE_LOWER_TAR_GZ': 'SOURCELOWER_TAR_GZ', 'SOURCE_LOWER_TAR_XZ': 'SOURCELOWER_TAR_XZ', 'SOURCE_LOWER_TAR_BZ2': 'SOURCELOWER_TAR_BZ2', @@ -257,6 +253,13 @@ # deprecated template constants, and their replacements DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS = { # : (, ), + 'GNU_FTP_SOURCE': ('GNU_FTP_URLS', '6.0'), + 'GNU_SOURCE': ('GNU_FTP_URLS', '6.0'), + 'XORG_DATA_SOURCE': ('XORG_DATA_URLS', '6.0'), + 'XORG_LIB_SOURCE': ('XORG_LIB_URLS', '6.0'), + 'XORG_PROTO_SOURCE': ('XORG_PROTO_URLS', '6.0'), + 'XORG_UTIL_SOURCE': ('XORG_PROTO_URLS', '6.0'), + 'XORG_XCB_SOURCE': ('XORG_XCB_SOURCE', '6.0'), } EXTENSIONS = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z'] diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 803868a51d..c8c498e07b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1305,10 +1305,10 @@ def test_templating_constants(self): def test_template_constant_import(self): """Test importing template constants works""" - from easybuild.framework.easyconfig.templates import GITHUB_SOURCE, GNU_SOURCE, SHLIB_EXT + from easybuild.framework.easyconfig.templates import GITHUB_SOURCE, GNU_FTP_URLS, SHLIB_EXT from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS self.assertEqual(GITHUB_SOURCE, TEMPLATE_CONSTANTS['GITHUB_SOURCE'][0]) - self.assertEqual(GNU_SOURCE, TEMPLATE_CONSTANTS['GNU_SOURCE'][0]) + self.assertEqual(GNU_FTP_URLS, TEMPLATE_CONSTANTS['GNU_FTP_URLS'][0]) self.assertEqual(SHLIB_EXT, get_shared_lib_ext()) def test_ec_method_resolve_template(self): diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index 4fbe6676a0..3f9a5b6140 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -201,7 +201,7 @@ def test_easyconfig_constants(self): val = constants[constant_name] fail_msg = "The constant %s should have an acceptable type, found %s (%s)" % (constant_name, type(val), str(val)) - self.assertIsInstance(val, (str, dict, tuple), fail_msg) + self.assertIsInstance(val, (str, dict, list, tuple), fail_msg) # check a couple of randomly picked constant values self.assertEqual(constants['SOURCE_TAR_GZ'], '%(name)s-%(version)s.tar.gz') diff --git a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb index 6969ac4b08..f53e576370 100644 --- a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb +++ b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb @@ -10,7 +10,7 @@ description = "binutils: GNU binary utilities" toolchain = {'name': 'GCCcore', 'version': '4.9.3'} sources = [SOURCE_TAR_GZ] -source_urls = [GNU_SOURCE] +source_urls = GNU_FTP_URLS # Testing purposes only so remove deps #builddependencies = [ diff --git a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb index c9cb4abfff..0a4051f629 100644 --- a/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb +++ b/test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb @@ -10,7 +10,7 @@ description = "binutils: GNU binary utilities" toolchain = {'name': 'GCCcore', 'version': '4.9.3'} sources = [SOURCE_TAR_GZ] -source_urls = [GNU_SOURCE] +source_urls = GNU_FTP_URLS # Testing purposes only so remove deps #builddependencies = [ diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb index 54c18d79e0..b8356ab34f 100644 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb @@ -24,7 +24,7 @@ toolchain = {'name': 'GCC', 'version': '4.6.3'} sources = ['%(name)s-%(version)s.tar.gz'] # download location for source files -source_urls = [GNU_SOURCE] +source_urls = GNU_FTP_URLS hiddendependencies = [('toy', '0.0', '-deps', SYSTEM)] dependencies = hiddendependencies # hidden deps must be included in list of deps diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb index 406f4e3342..331159203e 100644 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb @@ -24,7 +24,7 @@ toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'} sources = ['%(name)s-%(version)s.tar.gz'] # download location for source files -source_urls = [GNU_SOURCE] +source_urls = GNU_FTP_URLS # make sure the gzip and gunzip binaries are available after installation sanity_check_paths = { diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4.eb index ec646acb08..56021c7479 100644 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4.eb +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4.eb @@ -24,7 +24,8 @@ toolchain = SYSTEM sources = [SOURCE_TAR_GZ] # download location for source files -source_urls = ['https://ftpmirror.gnu.org/gzip'] +source_urls = ['https://ftpmirror.gnu.org/gzip', + 'https://ftp.gnu.org/gnu/gzip'] # make sure the gzip and gunzip binaries are available after installation sanity_check_paths = { diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-foss-2018a.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-foss-2018a.eb index af56176fa1..5492917388 100644 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-foss-2018a.eb +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-foss-2018a.eb @@ -20,7 +20,8 @@ description = "gzip (GNU zip) is a popular data compression program as a replace toolchain = {'name': 'foss', 'version': '2018a'} # eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz -source_urls = ['http://ftpmirror.gnu.org/gzip'] +source_urls = ['http://ftpmirror.gnu.org/gzip', + 'https://ftp.gnu.org/gnu/gzip'] sources = [SOURCE_TAR_GZ] # make sure the gzip, gunzip and compress binaries are available after installation diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-intel-2018a.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-intel-2018a.eb index bc583ca28a..7080ece636 100644 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-intel-2018a.eb +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-intel-2018a.eb @@ -20,7 +20,8 @@ description = "gzip (GNU zip) is a popular data compression program as a replace toolchain = {'name': 'intel', 'version': '2018a'} # eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz -source_urls = ['http://ftpmirror.gnu.org/gzip'] +source_urls = ['http://ftpmirror.gnu.org/gzip', + 'https://ftp.gnu.org/gnu/gzip'] sources = [SOURCE_TAR_GZ] # make sure the gzip, gunzip and compress binaries are available after installation diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-GCC-4.9.2.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-GCC-4.9.2.eb index 50c1c95c95..4f98a53fa1 100644 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-GCC-4.9.2.eb +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-GCC-4.9.2.eb @@ -21,7 +21,7 @@ description = "gzip (GNU zip) is a popular data compression program as a replace toolchain = {'name': 'GCC', 'version': '4.9.2'} # http://ftp.gnu.org/gnu/gzip/gzip-1.6.tar.gz -source_urls = [GNU_SOURCE] +source_urls = GNU_FTP_URLS sources = ['%(name)s-%(version)s.tar.gz'] # make sure the gzip, gunzip and compress binaries are available after installation diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb index 974fbad0c0..b9c74efbc9 100644 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb @@ -24,7 +24,7 @@ toolchain = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} sources = ['%(name)s-%(version)s.tar.gz'] # download location for source files -source_urls = [GNU_SOURCE] +source_urls = GNU_FTP_URLS # make sure the gzip and gunzip binaries are available after installation sanity_check_paths = { diff --git a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4-GCC-4.6.3.eb b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4-GCC-4.6.3.eb index a5b500d09d..a1e7014232 100644 --- a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4-GCC-4.6.3.eb +++ b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4-GCC-4.6.3.eb @@ -24,7 +24,7 @@ toolchain = {'name': 'GCC','version': '4.6.3'} sources = ['%s-%s.tar.gz'%(name,version)] # download location for source files -source_urls = [GNU_SOURCE] +source_urls = GNU_FTP_URLS # make sure the gzip and gunzip binaries are available after installation sanity_check_paths = { diff --git a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4.eb b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4.eb index 5126eff488..c310683574 100644 --- a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4.eb +++ b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4.eb @@ -24,7 +24,8 @@ toolchain = SYSTEM sources = [SOURCE_TAR_GZ] # download location for source files -source_urls = ['https://ftpmirror.gnu.org/gnu/gzip'] +source_urls = ['https://ftpmirror.gnu.org/gnu/gzip', + 'https://ftp.gnu.org/gnu/gzip'] # make sure the gzip and gunzip binaries are available after installation sanity_check_paths = { diff --git a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-foss-2018a.eb b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-foss-2018a.eb index 196df6cc55..dbf569467b 100644 --- a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-foss-2018a.eb +++ b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-foss-2018a.eb @@ -20,7 +20,8 @@ description = "gzip (GNU zip) is a popular data compression program as a replace toolchain = {'name': 'foss', 'version': '2018a'} # eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz -source_urls = ['https://ftpmirror.gnu.org/gnu/gzip'] +source_urls = ['https://ftpmirror.gnu.org/gnu/gzip', + 'https://ftp.gnu.org/gnu/gzip'] sources = [SOURCE_TAR_GZ] # make sure the gzip, gunzip and compress binaries are available after installation diff --git a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-intel-2018a.eb b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-intel-2018a.eb index 3a9a68aa16..e27e7f391a 100644 --- a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-intel-2018a.eb +++ b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-intel-2018a.eb @@ -20,7 +20,8 @@ description = "%(namelower)s (GNU zip) is a popular data compression program as toolchain = {'name': 'intel', 'version': '2018a'} # eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz -source_urls = ['https://ftpmirror.gnu.org/gnu/gzip'] +source_urls = ['https://ftpmirror.gnu.org/gnu/gzip', + 'https://ftp.gnu.org/gnu/gzip'] sources = [SOURCE_TAR_GZ] # make sure the gzip, gunzip and compress binaries are available after installation diff --git a/test/framework/easyconfigs/v2.0/doesnotexist.eb b/test/framework/easyconfigs/v2.0/doesnotexist.eb index 31d9aa1ae0..3396b3a3b6 100644 --- a/test/framework/easyconfigs/v2.0/doesnotexist.eb +++ b/test/framework/easyconfigs/v2.0/doesnotexist.eb @@ -20,7 +20,8 @@ software_license_urls = ['http://localhost/my_license'] sources = ['%(name)s-%(version)s.tar.gz'] # download location for source files -source_urls = ['http://ftpmirror.gnu.org/gzip'] +source_urls = ['http://ftpmirror.gnu.org/gzip', + 'https://ftp.gnu.org/gnu/gzip'] # make sure the gzip and gunzip binaries are available after installation sanity_check_paths = {'files': ["bin/gunzip", "bin/gzip"], 'dirs': []} diff --git a/test/framework/easyconfigs/v2.0/gzip.eb b/test/framework/easyconfigs/v2.0/gzip.eb index 65d6aa0fc6..c0dc04f11e 100644 --- a/test/framework/easyconfigs/v2.0/gzip.eb +++ b/test/framework/easyconfigs/v2.0/gzip.eb @@ -26,7 +26,7 @@ docurls = ["http://www.gnu.org/software/gzip/manual/gzip.html"] software_license = GPLv3 software_license_urls = ['http://www.gnu.org/software/gzip/'] -source_urls = [GNU_SOURCE] +source_urls = GNU_FTP_URLS sources = [SOURCE_TAR_GZ] sanity_check_paths = {