From 801502d985878a6606648789018746f43bdd25bd Mon Sep 17 00:00:00 2001 From: fstagni Date: Tue, 1 Jun 2021 18:50:42 +0200 Subject: [PATCH 1/6] removing use of deprecated method --- src/DIRAC/Interfaces/API/Dirac.py | 4 ++-- tests/Jenkins/dirac-cfg-update.py | 16 +++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/DIRAC/Interfaces/API/Dirac.py b/src/DIRAC/Interfaces/API/Dirac.py index 08121fd620e..23ae43ca1d6 100755 --- a/src/DIRAC/Interfaces/API/Dirac.py +++ b/src/DIRAC/Interfaces/API/Dirac.py @@ -614,7 +614,7 @@ def runLocal(self, job): self.log.warn('Input data resolution failed') return result - softwarePolicy = self.__getVOPolicyModule('SoftwareDistModule') + softwarePolicy = Operations().getValue('SoftwareDistModule') if softwarePolicy: moduleFactory = ModuleFactory() moduleInstance = moduleFactory.getModule(softwarePolicy, {'Job': parameters}) @@ -628,7 +628,7 @@ def runLocal(self, job): self.log.warn('Software installation failed with result:\n%s' % (result)) return result else: - self.log.verbose('Could not retrieve DIRAC/VOPolicy/SoftwareDistModule for VO') + self.log.verbose('Could not retrieve SoftwareDistModule for VO') self.log.debug("Looking for resolving the input sandbox, if it is present") sandbox = parameters.get('InputSandbox') diff --git a/tests/Jenkins/dirac-cfg-update.py b/tests/Jenkins/dirac-cfg-update.py index a30befb4950..39fdcfc9964 100644 --- a/tests/Jenkins/dirac-cfg-update.py +++ b/tests/Jenkins/dirac-cfg-update.py @@ -66,17 +66,11 @@ setup = gConfig.getValue('/DIRAC/Setup') if not setup: setup = 'dirac-JenkinsSetup' - if not vo: - vo = gConfig.getValue('/DIRAC/VirtualOrganization') - if not vo: - vo = 'dirac' - if not localCfg.isSection('/DIRAC/VOPolicy'): - localCfg.createNewSection('/DIRAC/VOPolicy') - if not localCfg.isSection('/DIRAC/VOPolicy/%s' % vo): - localCfg.createNewSection('/DIRAC/VOPolicy/%s' % vo) - if not localCfg.isSection('/DIRAC/VOPolicy/%s/%s' % (vo, setup)): - localCfg.createNewSection('/DIRAC/VOPolicy/%s/%s' % (vo, setup)) - localCfg.setOption('/DIRAC/VOPolicy/%s/%s/SoftwareDistModule' % (vo, setup), '') + if not localCfg.isSection('/Operations'): + localCfg.createNewSection('/Operations') + if not localCfg.isSection('/Operations/%s' % setup): + localCfg.createNewSection('/Operations/%s' % setup) + localCfg.setOption('/Operations/%s/SoftwareDistModule' % setup, '') localCfg.writeToFile(localConfigFile) From 05c527dcb89631eefb142bfbf27fe224b0b7e5d0 Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 2 Jun 2021 12:44:31 +0200 Subject: [PATCH 2/6] fix for columns names --- .../FrameworkSystem/Client/SystemAdministratorClientCLI.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py index 4f73f1fdb9a..1333e293c58 100644 --- a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +++ b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py @@ -5,6 +5,7 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division + import sys import pprint import os @@ -553,8 +554,8 @@ def getInstallations(self, argss): gLogger.notice('Installed: '.rjust(20) + isInstalled) gLogger.notice('Host: '.rjust(20) + installation['Host']['HostName']) gLogger.notice('Name: '.rjust(20) + installation['Instance']) - gLogger.notice('Module: '.rjust(20) + installation['Component']['Module']) - gLogger.notice('System: '.rjust(20) + installation['Component']['System']) + gLogger.notice('Module: '.rjust(20) + installation['Component']['DIRACModule']) + gLogger.notice('System: '.rjust(20) + installation['Component']['DIRACSystem']) gLogger.notice('Type: '.rjust(20) + installation['Component']['Type']) gLogger.notice('Installed on: '.rjust(20) + installation['InstallationTime'].strftime("%d-%m-%Y %H:%M")) if installedBy != '': From a7dd5198790f1ee8499c4a2cd246f53490a5c61d Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 2 Jun 2021 12:45:01 +0200 Subject: [PATCH 3/6] using DIRAC.siteName instead of looking for /LocalSite/Site --- src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py | 4 ++-- src/DIRAC/WorkloadManagementSystem/Client/CPUNormalization.py | 4 ++-- src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py index 6d5b119457b..66e9c2c4eab 100755 --- a/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py @@ -19,7 +19,7 @@ from diraccfg import CFG -from DIRAC import S_OK, S_ERROR, gConfig, rootPath +from DIRAC import S_OK, S_ERROR, gConfig, rootPath, siteName from DIRAC.Core.Utilities.ModuleFactory import ModuleFactory from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd from DIRAC.Core.Base.AgentModule import AgentModule @@ -116,7 +116,7 @@ def initialize(self): self.initTimes = os.times() # Localsite options - self.siteName = gConfig.getValue('/LocalSite/Site', self.siteName) + self.siteName = siteName() self.pilotReference = gConfig.getValue('/LocalSite/PilotReference', self.pilotReference) self.defaultProxyLength = gConfig.getValue('/Registry/DefaultProxyLifeTime', self.defaultProxyLength) # Agent options diff --git a/src/DIRAC/WorkloadManagementSystem/Client/CPUNormalization.py b/src/DIRAC/WorkloadManagementSystem/Client/CPUNormalization.py index 8612b8fbd38..1bd95ef93c4 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/CPUNormalization.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/CPUNormalization.py @@ -13,7 +13,7 @@ import os from six.moves.urllib.request import urlopen -from DIRAC import gConfig, gLogger, S_OK, S_ERROR +from DIRAC import gConfig, gLogger, S_OK, S_ERROR, siteName from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping from DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft import TimeLeft from DIRAC.WorkloadManagementSystem.Client.DIRACbenchmark import singleDiracBenchmark @@ -218,7 +218,7 @@ def getCPUTime(cpuNormalizationFactor): # we have to look for a ceQueue in the CS # A bit hacky. We should better profit from something generic gLogger.warn("No CEQueue in local configuration, looking to find one in CS") - siteName = gConfig.getValue('/LocalSite/Site') + siteName = siteName() queueSection = '/Resources/Sites/%s/%s/CEs/%s/Queues' % (siteName.split('.')[0], siteName, gridCE) res = gConfig.getSections(queueSection) if not res['OK']: diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 27bb6011133..2a0d0c656d4 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -92,7 +92,7 @@ def __init__(self, jobID=None, jobReport=None): self.jobID = 0 else: self.jobID = jobID - self.siteName = gConfig.getValue('/LocalSite/Site', 'Unknown') + self.siteName = DIRAC.siteName() if jobReport: self.jobReport = jobReport else: From c3c366f344eaac0b04ad86a0553ca40160541059 Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 2 Jun 2021 13:49:59 +0200 Subject: [PATCH 4/6] workaround --- environment-py3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-py3.yml b/environment-py3.yml index 0306a2486e5..be40b508ea6 100644 --- a/environment-py3.yml +++ b/environment-py3.yml @@ -7,7 +7,7 @@ channels: dependencies: # Temporary workarounds - - astroid !=2.5 # https://github.com/PyCQA/astroid/issues/903 + - astroid 2.5.6 # https://github.com/PyCQA/astroid/issues/1006 and https://github.com/PyCQA/astroid/issues/1007 # runtime - python =3.9 - pip From d7a37d9960b09aac73e72b2e82024fc59548fed0 Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 2 Jun 2021 14:59:34 +0200 Subject: [PATCH 5/6] added one test --- .../Client/CPUNormalization.py | 5 +++-- .../Framework/Test_InstalledComponentsDB.py | 12 +++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/CPUNormalization.py b/src/DIRAC/WorkloadManagementSystem/Client/CPUNormalization.py index 1bd95ef93c4..5844ebe9b96 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/CPUNormalization.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/CPUNormalization.py @@ -13,7 +13,8 @@ import os from six.moves.urllib.request import urlopen -from DIRAC import gConfig, gLogger, S_OK, S_ERROR, siteName +import DIRAC +from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping from DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft import TimeLeft from DIRAC.WorkloadManagementSystem.Client.DIRACbenchmark import singleDiracBenchmark @@ -218,7 +219,7 @@ def getCPUTime(cpuNormalizationFactor): # we have to look for a ceQueue in the CS # A bit hacky. We should better profit from something generic gLogger.warn("No CEQueue in local configuration, looking to find one in CS") - siteName = siteName() + siteName = DIRAC.siteName() queueSection = '/Resources/Sites/%s/%s/CEs/%s/Queues' % (siteName.split('.')[0], siteName, gridCE) res = gConfig.getSections(queueSection) if not res['OK']: diff --git a/tests/Integration/Framework/Test_InstalledComponentsDB.py b/tests/Integration/Framework/Test_InstalledComponentsDB.py index ef2eceaeea9..142506cbebd 100644 --- a/tests/Integration/Framework/Test_InstalledComponentsDB.py +++ b/tests/Integration/Framework/Test_InstalledComponentsDB.py @@ -379,32 +379,26 @@ def testHostLogging(self): # Create a sample log result = self.client.updateLog('TestHost', {'DIRACVersion': 'v6r15'}) - self.assertTrue(result['OK']) # Check that the log exists result = self.client.getLog('TestHost') - self.assertTrue(result['OK'] and result['Value'][0]['DIRACVersion'] == 'v6r15') # Update the fields of the created log result = self.client.updateLog('TestHost', {'hostName': 'StillATestHost'}) - self.assertTrue(result['OK']) # Check if the log with the modified fields exists result = self.client.getLog('StillATestHost') - self.assertTrue(result['OK'] and result['Value'][0]['DIRACVersion'] == 'v6r15') # Remove the log result = self.client.removeLogs({'hostName': 'StillATestHost'}) - self.assertTrue(result['OK']) # Check that the log was actually removed result = self.client.getLog('StillATestHost') - self.assertFalse(result['OK']) # Multiple removal @@ -415,15 +409,15 @@ def testHostLogging(self): self.client.removeLogs({'DIRACVersion': 'v7r0'}) result = self.client.getLog('TestHostC') - self.assertTrue(result['OK'] and len(result['Value']) >= 1) result = self.client.getLog('TestHostB') - self.assertFalse(result['OK']) - result = self.client.removeLogs({'DIRACVersion': 'v7r1'}) + result = self.client.getLogs() + self.assertTrue(result['OK']) + result = self.client.removeLogs({'DIRACVersion': 'v7r1'}) self.assertTrue(result['OK']) From 4deeb7c35169dc3c6ef21bb0b9a4b654db98499b Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 2 Jun 2021 15:31:20 +0200 Subject: [PATCH 6/6] better formatting --- .../Client/SystemAdministratorClientCLI.py | 75 ++++++++++++------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py index 1333e293c58..d4eedca6e76 100644 --- a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +++ b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py @@ -505,18 +505,31 @@ def getInstallations(self, argss): installations = result['Value'] if installations: - if display == 'table': - gLogger.notice('') - gLogger.notice(' ' + 'Num'.center(5) + ' ' + - 'Host'.center(20) + ' ' + - 'Name'.center(20) + ' ' + - 'Module'.center(20) + ' ' + - 'System'.center(16) + ' ' + - 'Type'.center(12) + ' ' + - 'Installed on'.center(18) + ' ' + - 'Install by'.center(12) + ' ' + - 'Uninstalled on'.center(18) + ' ' + - 'Uninstall by'.center(12)) + if display == "table": + gLogger.notice("") + gLogger.notice( + " " + + "Num".center(5) + + " " + + "Host".center(20) + + " " + + "Name".center(20) + + " " + + "Module".center(20) + + " " + + "System".center(16) + + " " + + "Type".center(12) + + " " + + "Installed on".center(18) + + " " + + "Install by".center(12) + + " " + + "Uninstalled on".center(18) + + " " + + "Uninstall by".center(12) + ) + gLogger.notice(("-") * 164) gLogger.notice(('-') * 164) for i, installation in enumerate(installations): if not installation['InstalledBy']: @@ -536,18 +549,31 @@ def getInstallations(self, argss): uninstalledOn = '' isInstalled = 'Yes' - if display == 'table': - gLogger.notice('|' + str(i + 1).center(5) + '|' + - installation['Host']['HostName'].center(20) + '|' + - installation['Instance'].center(20) + '|' + - installation['Component']['Module'].center(20) + '|' + - installation['Component']['System'].center(16) + '|' + - installation['Component']['Type'].center(12) + '|' + - installation['InstallationTime'].strftime("%d-%m-%Y %H:%M").center(18) + '|' + - installedBy.center(12) + '|' + - uninstalledOn.center(18) + '|' + - uninstalledBy.center(12) + '|') - gLogger.notice(('-') * 164) + if display == "table": + gLogger.notice( + "|" + + str(i + 1).center(5) + + "|" + + installation["Host"]["HostName"].center(20) + + "|" + + installation["Instance"].center(20) + + "|" + + installation["Component"]["Module"].center(20) + + "|" + + installation["Component"]["System"].center(16) + + "|" + + installation["Component"]["Type"].center(12) + + "|" + + installation["InstallationTime"].strftime("%d-%m-%Y %H:%M").center(18) + + "|" + + installedBy.center(12) + + "|" + + uninstalledOn.center(18) + + "|" + + uninstalledBy.center(12) + + "|" + ) + gLogger.notice(("-") * 164) elif display == 'list': gLogger.notice('') gLogger.notice('Installation: '.rjust(20) + str(i + 1)) @@ -795,7 +821,6 @@ def do_uninstall(self, args): result = getProxyInfo() if not result['OK']: self._errMsg(result['Message']) - user = result['Value']['username'] option = argss[0] if option == 'db':