From 428cf61741e28bb6035dc271639b3edac4210d08 Mon Sep 17 00:00:00 2001 From: vagrant Date: Tue, 26 Jan 2021 14:28:31 +0000 Subject: [PATCH 1/3] Modifications to manage certificates to access the Elias (Elaasticsearch database) --- ConfigurationSystem/Client/Utilities.py | 56 +++++++++++++++++++++++++ Core/Base/ElasticDB.py | 11 ++++- Core/Utilities/ElasticSearchDB.py | 15 ++++++- 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/ConfigurationSystem/Client/Utilities.py b/ConfigurationSystem/Client/Utilities.py index e6f59e3a4cd..baf1f2b81b0 100644 --- a/ConfigurationSystem/Client/Utilities.py +++ b/ConfigurationSystem/Client/Utilities.py @@ -530,6 +530,62 @@ def getElasticDBParameters(fullname): ssl = False if result['Value'].lower() in ('false', 'no', 'n') else True parameters['SSL'] = ssl + # Elasticsearch use certs + result = gConfig.getOption(cs_path + '/CRT') + if not result['OK']: + # No host name found, try at the common place + result = gConfig.getOption('/Systems/NoSQLDatabases/CRT') + if not result['OK']: + gLogger.warn("Failed to get the configuration parameter: certs. Using False") + certs = False + else: + certs = result['Value'] + else: + certs = result['Value'] + parameters['CRT'] = certs + + # Elasticsearch ca_certs + result = gConfig.getOption(cs_path + '/ca_certs') + if not result['OK']: + # No host name found, try at the common place + result = gConfig.getOption('/Systems/NoSQLDatabases/ca_certs') + if not result['OK']: + gLogger.warn("Failed to get the configuration parameter: ca_certs. Using None") + ca_certs = None + else: + ca_certs = result['Value'] + else: + ca_certs = result['Value'] + parameters['ca_certs'] = ca_certs + + # Elasticsearch client_key + result = gConfig.getOption(cs_path + '/client_key') + if not result['OK']: + # No host name found, try at the common place + result = gConfig.getOption('/Systems/NoSQLDatabases/client_key') + if not result['OK']: + gLogger.warn("Failed to get the configuration parameter: client_key. Using None") + client_key = None + else: + client_key = result['Value'] + else: + client_key = result['Value'] + parameters['client_key'] = client_key + + # Elasticsearch client_cert + result = gConfig.getOption(cs_path + '/client_cert') + if not result['OK']: + # No host name found, try at the common place + result = gConfig.getOption('/Systems/NoSQLDatabases/client_cert') + if not result['OK']: + gLogger.warn("Failed to get the configuration parameter: client_cert. Using None") + client_cert = None + else: + client_cert = result['Value'] + else: + client_cert = result['Value'] + parameters['client_cert'] = client_cert + return S_OK(parameters) diff --git a/Core/Base/ElasticDB.py b/Core/Base/ElasticDB.py index 4dc0064e965..6c79196d73f 100644 --- a/Core/Base/ElasticDB.py +++ b/Core/Base/ElasticDB.py @@ -36,13 +36,22 @@ def __init__(self, dbname, fullName, indexPrefix=''): self.__user = dbParameters.get('User', '') self.__dbPassword = dbParameters.get('Password', '') self.__useSSL = dbParameters.get('SSL', True) + self.__useCRT = dbParameters.get('CRT', True) + self.__ca_certs = dbParameters.get('ca_certs', None) + self.__client_key = dbParameters.get('client_key', None) + self.__client_cert = dbParameters.get('client_cert', None) super(ElasticDB, self).__init__(self._dbHost, self._dbPort, self.__user, self.__dbPassword, indexPrefix, - useSSL=self.__useSSL) + useSSL=self.__useSSL, + useCRT=self.__useCRT, + ca_certs=self.__ca_certs, + client_key=self.__client_key, + client_cert=self.__client_cert) + if not self._connected: raise RuntimeError('Can not connect to ES cluster %s, exiting...' % self.clusterName) diff --git a/Core/Utilities/ElasticSearchDB.py b/Core/Utilities/ElasticSearchDB.py index 24015f3b0e3..077b6b8662f 100644 --- a/Core/Utilities/ElasticSearchDB.py +++ b/Core/Utilities/ElasticSearchDB.py @@ -58,7 +58,8 @@ class ElasticSearchDB(object): RESULT_SIZE = 10000 ######################################################################## - def __init__(self, host, port, user=None, password=None, indexPrefix='', useSSL=True): + def __init__(self, host, port, user=None, password=None, indexPrefix='', useSSL=True, + useCRT=False, ca_certs=None, client_key=None, client_cert=None): """ c'tor :param self: self reference @@ -68,6 +69,10 @@ def __init__(self, host, port, user=None, password=None, indexPrefix='', useSSL= :param str password: if the db is password protected we need to provide a password :param str indexPrefix: it is the indexPrefix used to get all indexes :param bool useSSL: We can disable using secure connection. By default we use secure connection. + :param bool useCRT: Use certificates. + :param str ca_certs: Server certificate. + :param str client_key: Client key. + :param str client_cert: Client certificate. """ self.__indexPrefix = indexPrefix @@ -105,6 +110,14 @@ def __init__(self, host, port, user=None, password=None, indexPrefix='', useSSL= use_ssl=True, verify_certs=True, ca_certs=casFile) + elif useCRT: + self.client = Elasticsearch(self.__url, + timeout=self.__timeout, + use_ssl=True, + verify_certs=True, + ca_certs=ca_certs, + client_cert=client_cert, + client_key=client_key) else: self.client = Elasticsearch(self.__url, timeout=self.__timeout) From 327bbe2acee348c1fa887a3e9d5140572e1c5fa0 Mon Sep 17 00:00:00 2001 From: sanguillon Date: Tue, 26 Jan 2021 15:08:32 +0000 Subject: [PATCH 2/3] Monitoring - Access to ElasticsearchDB - Fix bugs --- ConfigurationSystem/Client/Utilities.py | 2 +- Core/Utilities/ElasticSearchDB.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ConfigurationSystem/Client/Utilities.py b/ConfigurationSystem/Client/Utilities.py index baf1f2b81b0..f0ad8604d55 100644 --- a/ConfigurationSystem/Client/Utilities.py +++ b/ConfigurationSystem/Client/Utilities.py @@ -536,7 +536,7 @@ def getElasticDBParameters(fullname): # No host name found, try at the common place result = gConfig.getOption('/Systems/NoSQLDatabases/CRT') if not result['OK']: - gLogger.warn("Failed to get the configuration parameter: certs. Using False") + gLogger.warn("Failed to get the configuration parameter: CRT. Using False") certs = False else: certs = result['Value'] diff --git a/Core/Utilities/ElasticSearchDB.py b/Core/Utilities/ElasticSearchDB.py index 077b6b8662f..7169baa2c27 100644 --- a/Core/Utilities/ElasticSearchDB.py +++ b/Core/Utilities/ElasticSearchDB.py @@ -59,7 +59,7 @@ class ElasticSearchDB(object): ######################################################################## def __init__(self, host, port, user=None, password=None, indexPrefix='', useSSL=True, - useCRT=False, ca_certs=None, client_key=None, client_cert=None): + useCRT=False, ca_certs=None, client_key=None, client_cert=None): """ c'tor :param self: self reference @@ -112,12 +112,12 @@ def __init__(self, host, port, user=None, password=None, indexPrefix='', useSSL= ca_certs=casFile) elif useCRT: self.client = Elasticsearch(self.__url, - timeout=self.__timeout, - use_ssl=True, - verify_certs=True, - ca_certs=ca_certs, - client_cert=client_cert, - client_key=client_key) + timeout=self.__timeout, + use_ssl=True, + verify_certs=True, + ca_certs=ca_certs, + client_cert=client_cert, + client_key=client_key) else: self.client = Elasticsearch(self.__url, timeout=self.__timeout) From 64deb4931260ab85f50b2e4153e6685a5f4032e5 Mon Sep 17 00:00:00 2001 From: sanguillon Date: Fri, 29 Jan 2021 13:37:26 +0000 Subject: [PATCH 3/3] comments in ConfigurationSystem/Client/Utilities.py changed to be correct --- ConfigurationSystem/Client/Utilities.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ConfigurationSystem/Client/Utilities.py b/ConfigurationSystem/Client/Utilities.py index f0ad8604d55..0a3509e273e 100644 --- a/ConfigurationSystem/Client/Utilities.py +++ b/ConfigurationSystem/Client/Utilities.py @@ -533,7 +533,7 @@ def getElasticDBParameters(fullname): # Elasticsearch use certs result = gConfig.getOption(cs_path + '/CRT') if not result['OK']: - # No host name found, try at the common place + # No CRT option found, try at the common place result = gConfig.getOption('/Systems/NoSQLDatabases/CRT') if not result['OK']: gLogger.warn("Failed to get the configuration parameter: CRT. Using False") @@ -547,7 +547,7 @@ def getElasticDBParameters(fullname): # Elasticsearch ca_certs result = gConfig.getOption(cs_path + '/ca_certs') if not result['OK']: - # No host name found, try at the common place + # No CA certificate found, try at the common place result = gConfig.getOption('/Systems/NoSQLDatabases/ca_certs') if not result['OK']: gLogger.warn("Failed to get the configuration parameter: ca_certs. Using None") @@ -561,7 +561,7 @@ def getElasticDBParameters(fullname): # Elasticsearch client_key result = gConfig.getOption(cs_path + '/client_key') if not result['OK']: - # No host name found, try at the common place + # No client private key found, try at the common place result = gConfig.getOption('/Systems/NoSQLDatabases/client_key') if not result['OK']: gLogger.warn("Failed to get the configuration parameter: client_key. Using None") @@ -575,7 +575,7 @@ def getElasticDBParameters(fullname): # Elasticsearch client_cert result = gConfig.getOption(cs_path + '/client_cert') if not result['OK']: - # No host name found, try at the common place + # No cient certificate found, try at the common place result = gConfig.getOption('/Systems/NoSQLDatabases/client_cert') if not result['OK']: gLogger.warn("Failed to get the configuration parameter: client_cert. Using None")