Merge pull request #1549 from d-ylee/wmcore-dev-image

New Alma9 wmcore-dev image

docker/pypi/wmcore-dev/.dockerignore

@@ -0,0 +1,5 @@
+*
+!TestScripts
+!ContainerScripts
+!entrypoint.sh
+!etc

docker/pypi/wmcore-dev/.gitignore

@@ -0,0 +1,5 @@
+contents/
+ContainerScripts/LatestUnitTests
+.DS_Store
+*/**/venv
+requirements.txt

docker/pypi/wmcore-dev/ContainerScripts/AggregatePylint.py

@@ -0,0 +1,95 @@
+#! /usr/bin/env python3
+
+import json
+
+from optparse import OptionParser
+
+usage = "usage: %prog [options] message"
+parser = OptionParser(usage)
+(options, args) = parser.parse_args()
+if len(args) != 1:
+    parser.error("You must supply a label")
+
+label = args[0]
+
+try:
+    with open('pylintReport.json', 'r') as reportFile:
+        report = json.load(reportFile)
+except IOError:
+    report = {}
+
+warnings = 0
+errors = 0
+comments = 0
+refactors = 0
+score = 0
+
+with open('pylint.out', 'r') as pylintFile:
+    for line in pylintFile:
+        if line.startswith('Your code has been rated at '):
+            scorePart = line.strip('Your code has been rated at ')
+            score = scorePart.split('/')[0]
+            try:
+                if not filename in report:
+                    report[filename] = {}
+                if not label in report[filename]:
+                    report[filename][label] = {}
+                if filename and label:
+                    report[filename][label]['score'] = score
+            except NameError:
+                print("Score of %s found, but no filename" % score)
+
+        parts = line.split(':')
+        if len(parts) != 3:
+            continue
+        try:
+            newFilename, lineNumber, rawMessage = parts
+            newFilename = newFilename.strip()
+            if not newFilename:  # Don't update filename if we didn't find one
+                continue
+            lineNumber = int(lineNumber)
+            filename = newFilename
+            rmParts = rawMessage.split(']', 1)
+            rawCode = rmParts[0].strip()
+            message = rmParts[1].strip()
+            severity = rawCode[1:2]
+            code = rawCode[2:6]
+            shortMsg = rawCode[7:]
+            msgParts = shortMsg.split(',')
+            objectName = msgParts[1].strip()
+
+            if severity == 'R':
+                refactors += 1
+            elif severity == 'W':
+                warnings += 1
+            elif severity == 'E':
+                errors += 1
+            elif severity == 'C':
+                comments += 1
+
+            if not filename in report:
+                report[filename] = {}
+
+            if not label in report[filename]:
+                report[filename][label] = {}
+            if not 'events' in report[filename][label]:
+                report[filename][label]['events'] = []
+            report[filename][label]['events'].append((lineNumber, severity, code, objectName, message))
+
+            report[filename][label]['refactors'] = refactors
+            report[filename][label]['warnings'] = warnings
+            report[filename][label]['errors'] = errors
+            report[filename][label]['comments'] = comments
+
+        except ValueError:
+            continue
+
+with open('pylintReport.json', 'w') as reportFile:
+    json.dump(report, reportFile, indent=2)
+    reportFile.write('\n')
+
+
+
+
+
+

docker/pypi/wmcore-dev/ContainerScripts/AnalyzePyFuture.py

@@ -0,0 +1,20 @@
+#! /usr/bin/env python
+
+from __future__ import print_function, division
+
+import os
+
+with open('addedFiles.txt', 'r') as addedFiles:
+    for fileName in addedFiles:
+        fileName = fileName.strip()
+        if fileName.endswith('__init__.py'):
+            continue
+        with open(fileName, 'r') as pyFile:
+            pyLines = pyFile.readlines()
+            if fileName.endswith('.py') or 'python' in  pyLines[0]:
+                foundDivision = False
+                for line in pyLines:
+                    if '__future__' in line and 'division' in line:
+                        foundDivision = True
+                if not foundDivision:
+                    print ("* New file %s does not use python 3 division. Please add `from __future__ import division`.\n" % fileName)

docker/pypi/wmcore-dev/ContainerScripts/CompareTests.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import glob
+import os
+import sys
+
+try:
+    from github import Github
+except ImportError:
+    # DMWM-WMCore-UnitTests and DMWM-WMCorePy3-UnitTests don't really push anything!
+    Github = None
+
+import xunitparser
+
+testResults = {}
+
+unstableTests = []
+
+try:
+    with open('code/test/etc/UnstableTests.txt') as unstableFile:
+        for line in unstableFile:
+            unstableTests.append(line.strip())
+except:
+    print("Was not able to open list of unstable tests")
+
+# Parse all the various nose xunit test reports looking for changes
+filePattern = '*/nosetestspy3-*.xml'
+if len(sys.argv) == 2:
+    filePattern = "*/%s-*.xml" % sys.argv[1]
+for kind, directory in [('base', './MasterUnitTests/'), ('test', './LatestUnitTests/')]:
+    for xunitFile in glob.iglob(directory + filePattern):
+
+        ts, tr = xunitparser.parse(open(xunitFile))
+        for tc in ts:
+            testName = '%s:%s' % (tc.classname, tc.methodname)
+            if testName in testResults:
+                testResults[testName].update({kind: tc.result})
+            else:
+                testResults[testName] = {kind: tc.result}
+
+# Generate a Github report of any changes found
+
+issueID, mode = None, None
+
+if 'ghprbPullId' in os.environ:
+    issueID = os.environ['ghprbPullId']
+    mode = 'PR'
+elif 'TargetIssueID' in os.environ:
+    issueID = os.environ['TargetIssueID']
+    mode = 'Daily'
+
+print("Comparing tests for issueID: {} in mode: {}".format(issueID, mode))
+
+message = 'Unit test changes for pull request %s:\n' % issueID
+if mode == 'Daily':
+    message = 'Unit test changes for most recent test of master branch:\n'
+
+changed = False
+stableChanged = False
+failed = False
+errorConditions = ['error', 'failure']
+
+for testName, testResult in sorted(testResults.items()):
+    if 'base' in testResult and 'test' in testResult and testName in unstableTests:
+        if testResult['base'] != testResult['test']:
+            changed = True
+            message += "* %s (unstable) changed from %s to %s\n" % (testName, testResult['base'], testResult['test'])
+    elif 'base' in testResult and 'test' in testResult:
+        if testResult['base'] != testResult['test']:
+            changed = True
+            stableChanged = True
+            message += "* %s changed from %s to %s\n" % (testName, testResult['base'], testResult['test'])
+            if testResult['test'] in errorConditions:
+                failed = True
+    elif 'test' in testResult:
+        changed = True
+        stableChanged = True
+        message += "* %s was added. Status is %s\n" % (testName, testResult['test'])
+        if testResult['test'] in errorConditions:
+            failed = True
+    elif 'base' in testResult:
+        changed = True
+        stableChanged = True
+        message += "* %s was deleted. Prior status was %s\n" % (testName, testResult['base'])
+if failed:
+    message += '\n\nPreviously working unit tests have failed!\n'
+
+if mode == 'Daily':
+    # Alan on 25/may/2021: then there is nothing else to be done
+    print(message)
+    sys.exit(0)
+
+gh = Github(os.environ['DMWMBOT_TOKEN'])
+codeRepo = os.environ.get('CODE_REPO', 'WMCore')
+repoName = '%s/%s' % (os.environ['WMCORE_REPO'], codeRepo)
+
+issue = gh.get_repo(repoName).get_issue(int(issueID))
+
+if not changed and mode == 'Daily':
+    message = "No changes to unit tests for latest build\n"
+elif not changed:
+    message = "No changes to unit tests for pull request %s\n" % issueID
+
+if mode == 'Daily' and stableChanged:
+    issue.create_comment('%s' % message)
+elif mode != 'Daily':
+    issue.create_comment('%s' % message)
+
+if failed:
+    print('Testing of python code. DMWM-FAIL-UNIT')
+else:
+    print('Testing of python code. DMWM-SUCCEED-UNIT')

docker/pypi/wmcore-dev/ContainerScripts/IdentifyPythonFiles.py

@@ -0,0 +1,31 @@
+#! /usr/bin/env python3
+
+from __future__ import print_function, division
+
+import os
+from optparse import OptionParser
+
+usage = "usage: %prog [options] list_of_files.txt"
+parser = OptionParser(usage)
+(options, args) = parser.parse_args()
+if len(args) != 1:
+    parser.error("You must supply a file with a list of files to check")
+
+list_of_files = args[0]
+
+with open(list_of_files, 'r') as changedFiles:
+    for fileName in changedFiles:
+        fileName = fileName.strip()
+        if not fileName:
+            continue
+        if fileName.endswith('.py'):
+            print(fileName)
+            continue
+        try:
+            with open(fileName, 'r') as pyFile:
+                pyLines = pyFile.readlines()
+                if 'python' in pyLines[0]:
+                    print(fileName)
+                    continue
+        except IOError:
+            pass

docker/pypi/wmcore-dev/ContainerScripts/deployWMAgent.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+
+. ./cms-bot/DMWM/setup-secrets.sh
+. ./cms-bot/DMWM/update-deployment.sh
+. ./cms-bot/DMWM/latest-dmwm-versions.sh
+
+if [ -z "$WMAGENT_VERSION" ]; then
+  export WMAGENT_VERSION=$WMAGENT_LATEST
+fi
+
+. ./cms-bot/DMWM/deploy-wmagent.sh
+

docker/pypi/wmcore-dev/ContainerScripts/deploy_unittest_py3.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+###
+# usage
+# deploy current version of wmagent external library and use the dmwm/master WMCore code to set up unittest
+# sh ./deploy_unittest.sh
+#
+# Also optional values can be specified. -v deploy agent version, -r git repository for code to test,
+# -b branch name of the repository.
+# following
+# i.e) sh ./deploy_unittest.sh -v 1.0.6 -r ticonaan -v 1.0.5_wmagent
+#
+# for running the test check the tutorial, https://github.com/dmwm/WMCore/wiki/Setup-wmcore-unittest
+###
+DMWM_ARCH=slc7_amd64_gcc630
+VERSION=$(curl -s "http://cmsrep.cern.ch/cgi-bin/repos/comp/$DMWM_ARCH?C=M;O=D" | grep -oP "(?<=>cms\+wmagentpy3-dev\+).*(?=-1-1)" | head -1)
+
+REPOSITORY=dmwm
+BRANCH=
+UPDATE=false
+
+deploy_agent() {
+
+    git clone https://github.com/dmwm/deployment.git
+    curl -s https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/init.sh > init.sh
+    curl -s https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/env_unittest_py3.sh > env_unittest_py3.sh
+    curl -s https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/WMAgent_unittest.secrets > WMAgent_unittest.secrets
+    source ./init.sh
+    # set -e
+    for step in prep sw post; do
+        echo -e "\n*** Deploying WMAgent py3: running $step step ***"
+        $PWD/deployment/Deploy -R wmagentpy3-dev@$1 -r comp=comp -t $1 -A $DMWM_ARCH -s $step $INSTALL_DIR wmagentpy3/devtools
+        if [ $? -ne 0 ]; then
+            ls $INSTALL_DIR
+            cat $INSTALL_DIR/.deploy/*-$step.log
+            exit 1
+        fi
+    done
+    # set +e
+}
+
+setup_test_src() {
+    (
+     mkdir $TEST_DIR;
+     cd $TEST_DIR;
+     git clone https://github.com/$1/WMCore.git;
+     cd WMCore
+     # if branch is set check out the branch
+     if [ -n $2 ]
+     then
+        git checkout $2
+     fi;
+    )
+}
+
+update_src() {
+    (
+    rm -rf $TEST_DIR;
+    setup_test_src $1 $2
+    )
+}
+
+while [ $# -gt 0 ]
+do
+    case "$1" in
+        -v)  VERSION=$2; shift;;
+        -r)  REPOSITORY=$2; shift;;
+        -b)  BRANCH=$2; shift;;
+        -u)  UPDATE=true;;
+        *)  break;;	# terminate while loop
+    esac
+    shift
+done
+
+if [ $UPDATE = "true" ]
+then
+    source ./env_unittest_py3.sh
+    update_src $REPOSITORY $BRANCH
+else
+    echo "--- deploying agent $VERSION with local user $USER"
+    # deploy agent
+    deploy_agent $VERSION
+    echo "--- updating agent source repository $REPOSITORY, branch $BRANCH"
+    # checkout test source
+    setup_test_src $REPOSITORY $BRANCH
+
+    # swap the source code from deployed one test source
+    source ./env_unittest_py3.sh
+    echo "--- starting services"
+    $manage start-services
+
+    echo "--- creating MariaDB database"
+    mysql -u $USER --socket=$INSTALL_DIR/current/install/mysql/logs/mysql.sock --execute "create database wmcore_unittest"
+fi

docker/pypi/wmcore-dev/ContainerScripts/fixCertificates.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -x
+
+# Certificates are either not readable by the container or have the wrong permissions
+
+cp orig-certs/servicecert.pem.orig certs/servicecert.pem
+cp orig-certs/servicekey.pem.orig certs/servicekey.pem
+chmod 600 certs/servicecert.pem
+chmod 400 certs/servicekey.pem