diff --git a/README.md b/README.md index 47c3690..181b802 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Matrix Corporal Policy Specification Using LDAP Groups + +* This scripts takes as input a specification in the form of a YAML file and generates a so-called *Policy* file for Matrix Corporal reconciliator tool (see https://github.com/devture/matrix-corporal/blob/master/docs/policy.md) with data coming from an LDAP server in which groups are specified containing `memberUid` entries. +* It also generates any rooms and spaces mentioned in the yaml file, that do not exist on the Synapse Homeserver yet. -Generates a Matrix Corporal JSON policy file with data coming from: -- an LDAP server in which groups are specified containing `memberUid` entries, and -- a YAML file in which a specification of which users must join which communities and/or rooms. ## Install with Pipenv @@ -10,55 +10,65 @@ Generates a Matrix Corporal JSON policy file with data coming from: * `$ export PATH=$HOME/.local/bin/:$PATH` * `$ pipenv sync` * `$ pipenv shell` +* `$pipenv run ./spec2policy input.yml policy.json` ## Install with Pip (not recommended) * Python >= 3.6 * `pip install -r requirements.txt` -## Usage -* This scripts takes as input a specification in the form of a YAML file -* and generates a so-called *Policy* file for Matrix Corporal reconciliator tool (see https://github.com/devture/matrix-corporal/blob/master/docs/policy.md) -* first of all you need to: +## Configuration * configure in `$HOME/.ldapsync.cfg` your server credentials (see `ldapsync.cfg` as example) * change (at least) DOMAIN and LDAP_SEARCH_BASE (potentially LDAP filters also) in `spec2policy.py` -* then to generate the policy file, type: `$ ./spec2policy.py matrix.yml policy.json` -* and finally to push the policy, type `$ curl -s --insecure -XPUT --data "@$(pwd)/policy.json" -H 'Authorization: Bearer ......' https://matrix.domain.com/_matrix/corporal/policy | jq .` + * If the shared secret provider is installed on the HomeServer, you can + For the room creation process to work it is recommended to access the homeserver directly thru port 8008 and not through matrix-corporal +## Usage +* To generate the policy file, type: `$ ./spec2policy.py input.yml policy.json` +* to push the policy to matrix-corporal, type `$ curl -s --insecure -XPUT --data "@$(pwd)/policy.json" -H 'Authorization: Bearer ......' https://matrix.domain.com/_matrix/corporal/policy | jq .` * also to automate the build and deployment of the policy file, it is possible to use a CI tool such as Jenkins, Gitlab CI, Travis CI etc. See the `.gitlab-ci.yml` as an example (it requires the definition of the credentials in a protected $CFG environment variable). ## Input specification format ~~~ -- matrixgroup: +group1:domain.com - rooms: - - room1 - - room2 +--- +#here set flags on user level based on LDAP Group memberships: +ldapgroups-forbidroomcreation: + - ldapgroup1 + - ldapgroup2 +ldapgroups-forbidencryptedroomcreation: + - ldapgroup1 +ldapgroups-forbidunencryptedroomcreation: + - ldapgroup2 +--- +#list of spaces and rooms. Rooms can be attached to a space with setting the entry childof +- space: SpaceName ldapgroups: - ldapgroup1 - ldapgroup2 ldapusers: - supplementaryuser1 - supplementaryuser2 - restricted: - - room: room3 - groups: - - ldapgroup3 - - ldapgroup4 - users: - - supplementaryuser1 - - supplementaryuser3 + - room: Roomname + ldapgroups: + - ldapgroup1 + ldapusers: + - ldapuser1 + childof: SpaceName ~~~ -* a list of *matrixgroup* can be specified, each in its own section -* for each matrix group, either a list of individual members is specified (*ldapusers*) -* or a list of members belonging to LDAP groups is specified (*ldapgroups*) -* or both -* also a *restricted* section can add rooms for which those specified members above do not have access. For each of these restricted rooms, there is a specific room-only member list, i.e., for each room a *users* or a *groups* list or both. +* On the first part of the yaml file, set the user based flags for the members of *ldapgroups-forbidroomcreation* and *ldapgroups-forbidencryptedroomcreation* and *forbidunencryptedroomcreation*. + +* On the second part, list all spaces/rooms with their ldapgroups, individual ldapusers and with *childof* the parent space of each room, tha tu want to place in a space. +All the rooms and spaces are created as private rooms by default. This can be adjusted by modifying the appropriate functions in spec2policy.py + ## Caveats -* neither matrix groups (aka communities) nor rooms are *created* by this script: they need to be create beforehand and corporal enabled on your Matrix server -* also rooms have to be inserted in their relevant Matrix group beforehand -* this script is generating a corporal `policy.json` file that you can either *push* or place in a policy provider (see https://github.com/devture/matrix-corporal/blob/master/docs/policy-providers.md for more details). +* There is no room deletion implemented. To delete rooms, use the synapse admin gui, for instance. +* A Matrix Administrator Account has to be enrolled in all of the room. This is necessary for the conciliation of matrix-corporal to work. + +## Future ideas: +* implement hooks +* set room-powerlevels by ldapgroup-membership diff --git a/ldapsync.cfg b/ldapsync.cfg index 13cb8c2..73f4680 100644 --- a/ldapsync.cfg +++ b/ldapsync.cfg @@ -5,4 +5,4 @@ BindPassword = ************ [matrix] AdminUser = admin AdminPassword = ************ -AdminToken = ************ +SharedSecret = *************** diff --git a/spec2policy.py b/spec2policy.py index e70b994..5451bab 100755 --- a/spec2policy.py +++ b/spec2policy.py @@ -24,6 +24,8 @@ import configparser import requests import yaml +import hmac +import hashlib from ldap3 import Server, Connection, ALL import json @@ -40,12 +42,14 @@ # constants ## rooms that will not be controlled -IGNORED_ROOMS = [ 'testchannel' ] +#IGNORED_ROOMS = [ 'testchannel' ] ## Matrix URL API constants -DOMAIN = 'domain.com' -MATRIX = 'https://matrix.' + DOMAIN + '/_matrix/client/r0' -ADMIN = 'https://matrix.' + DOMAIN + '/_synapse/admin/v1' +#ideally the policy is created on the matrixserver directly, so no security issue with shared-secret +DOMAIN = 'matrix.domain.com' +DOMAINURL = 'http://'+ DOMAIN +MATRIX = 'http://' + DOMAIN + ':8008/_matrix/client/r0' +ADMIN = 'http://' + DOMAIN + ':8008/_synapse/admin/v1' ## LDAP server constants LDAP_SERVER_IP = "10.10.10.10" @@ -58,23 +62,117 @@ config.read(user_config) MATRIX_ADMIN_ACCOUNT = config['matrix']['AdminUser'] MATRIX_ADMIN_PASSWORD = config['matrix']['AdminPassword'] -TOKEN = config['matrix']['AdminToken'] -AUTH = {'Authorization': 'Bearer ' + TOKEN} +#shared secret used instead of password +MATRIX_SHARED_SECRET = config['matrix']['SharedSecret'] + LDAP_BIND_ACCOUNT = config['ldap']['BindAccount'] LDAP_BIND_PASSWORD = config['ldap']['BindPassword'] # functions +#this function was created by devture.sharedsecretauth +def obtain_access_token(full_user_id, homeserver_api_url, shared_secret): + login_api_url = homeserver_api_url + ':8008/_matrix/client/r0/login' + token = hmac.new(shared_secret.encode('utf-8'), full_user_id.encode('utf-8'), hashlib.sha512).hexdigest() + payload = { + 'type': 'com.devture.shared_secret_auth', + 'identifier': { + 'type': 'm.id.user', + 'user': full_user_id, + }, + 'token': token, + } + #code for password login instead of shared secret + # If `m_login_password_support_enabled`, you can use `m.login.password`. + # The token goes into the `password` field for this login type, not the `token` field. + # + # payload = { + # 'type': 'm.login.password', + # 'identifier': { + # 'type': 'm.id.user', + # 'user': full_user_id, + # }, + # 'password': token, + # } + response = requests.post(login_api_url, data=json.dumps(payload)) + return {'Authorization': 'Bearer ' + response.json()['access_token']} + + +def create_space(roomname, AUTH): + #AUTH=obtain_access_token('@'+ roomcreator +':'+ DOMAIN, DOMAINURL, MATRIX_SHARED_SECRET) + url=DOMAINURL+':8008/_matrix/client/r0/createRoom' + payload = { + 'room_alias_name': roomname, + 'creation_content': { + 'type': 'm.space', + }, + 'name': roomname, + 'preset':'private_chat', + } + response=requests.post(url, data=json.dumps(payload), headers=AUTH) + if str(response) == '': + print("Space:"+ roomname +"has been successfully created") + return None + +def create_room(roomname, AUTH): + #AUTH=obtain_access_token('@'+ roomcreator +':'+ DOMAIN, DOMAINURL, MATRIX_SHARED_SECRET) + url=DOMAINURL+':8008/_matrix/client/r0/createRoom' + payload = { + 'room_alias_name': roomname, + 'name': roomname, + 'preset':'private_chat', + } + #print(payload) + response=requests.post(url, data=json.dumps(payload), headers=AUTH) + if str(response) == '': + print("Room"+ roomname +"has been successfully created") + return None + +def move_room(roomname, parentroom, roommap,AUTH): + # method=PUT uri="/_matrix/client/r0/rooms/!UjQqBRgxQUFzvwbjsf:matrix.wgs-albstadt.de/state/m.space.child/!QQMfqytsqBvNzexoUW:matrix.wgs-albstadt.de" + + # url=DOMAINURL+':8008/_matrix/client/r0/rooms/'+ o / + # payload={ + print(" ") + id_roomname= id_room(roomname,roommap) + id_parentroom = id_room(parentroom,roommap) + # print(id_parentroom) + # print(id_roomname) + url=DOMAINURL+':8008/_matrix/client/r0/rooms/'+ id_parentroom + '/state/m.space.child/'+ id_roomname + payload={ + 'via': [DOMAIN], + } + #print(list(roommap)) + #list(roommap).index({'name': roomname}) + + + + + #print(json.dumps(payload)) + response=requests.put(url,data=json.dumps(payload), headers=AUTH) + return None + + +def id_room(roomname,roommap): + return [room for room in roommap if room['name']== roomname][0]['id'] + + def spec_load_yaml(yamlfile): with open(yamlfile, 'r') as f: - spec = yaml.load(f, Loader=yaml.FullLoader) - return spec + spec = list(yaml.load_all(f, Loader=yaml.FullLoader)) + return spec[1] + +def spec_load_yaml_flags(yamlfile): + with open(yamlfile, 'r') as f: + spec = list(yaml.load_all(f, Loader=yaml.FullLoader)) + return spec[0] -def spec_groups(spec): - return [ i.get('matrixgroup', []) for i in spec ] + +def spec_rooms(spec): + return [ i.get('room', i.get('space',[])) for i in spec ] def matrix_whoami(): - response = requests.get(MATRIX + '/account/whoami', headers=AUTH) + response = requests.get(MATRIX + '/account/whoami', headers=ADMINAUTH) print(response.json()['user_id']) def matrix_compute_rooms_map(): @@ -152,17 +250,18 @@ def ldap_group_members(ldapgroup): def policy_update_schema(version): POLICY.update( { "schemaVersion": version } ) - -def policy_update_flags(change_name = True, change_avatar = True, forbid_create_room = False): +#define global flag settings +def policy_update_flags(): flags_data = {} flags_data['flags'] = {} - flags_data['flags']['allowCustomUserDisplayNames'] = change_name - flags_data['flags']['allowCustomUserAvatars'] = change_avatar - flags_data['flags']['forbidRoomCreation'] = forbid_create_room + flags_data['flags']['allowCustomUserDisplayNames'] = False + flags_data['flags']['allowCustomUserAvatars'] = False + flags_data['flags']['forbidRoomCreation'] = False + flags_data['flags']['allowCustomPassthroughUserPasswords'] = False + flags_data['flags']['forbidEncryptedRoomCreation'] = False + flags_data['flags']['forbidUnencryptedRoomCreation'] = False POLICY.update(flags_data) -def policy_update_groups(list_of_groups): - POLICY.update( { "managedCommunityIds": list_of_groups } ) def policy_update_rooms(list_of_rooms): POLICY.update( { "managedRoomIds": list_of_rooms } ) @@ -175,6 +274,8 @@ def policy_update_users(list_of_users): ## initial policy POLICY = {} +ADMINAUTH = obtain_access_token(MATRIX_ADMIN_ACCOUNT, DOMAINURL, MATRIX_SHARED_SECRET) + ## check admin user print ('Checking Matrix user...', end='') matrix_whoami() @@ -182,13 +283,10 @@ def policy_update_users(list_of_users): ## parse YAML spec file print ('Loading YAML specifications...', end='') spec = spec_load_yaml(yamlfile) +spec_flags = spec_load_yaml_flags(yamlfile) print('Done.') print('') -GROUPS = spec_groups(spec) - -print("=== Will generate policy for the following Matrix groups: ===") -print(GROUPS) -print('') +spec_rooms = spec_rooms(spec) ## add schemaVersion policy_update_schema(1) @@ -199,60 +297,84 @@ def policy_update_users(list_of_users): ## add managedCommunityIds policy_update_groups(GROUPS) -## GET the list of all rooms with a name +## GET the list of all existing rooms in matrix with a name print('Computing rooms map...', end='') -rooms = matrix_compute_rooms_map() - +existingrooms = matrix_compute_rooms_map() +#print(existingrooms) ## filter the ones that are not under control of corporal -ROOMS = [ r for r in rooms if matrix_room_name(r) not in IGNORED_ROOMS ] -print(str(len(ROOMS)) + " rooms found that are controlled.") -print('') -print("=== Will generate policy for the following Matrix rooms: ===") -names = list(map(lambda room:matrix_room_name(room), ROOMS)) -print(', '.join(names)) -print('') +## filter the ones that are listed in yaml but not in matrix + # compute rooms, a list of dicts of the form { 'name':"...", 'id':"..." } + +ROOMStobecreated = [ r for r in spec_rooms if r not in [i.get('name') for i in existingrooms] ] +print(' ') + +print(str(len(ROOMStobecreated)) + " rooms found in yaml that do not exist in Matrix yet.") + +if len(ROOMStobecreated) > 0: + print("=== Going to create the following rooms:"+ str(ROOMStobecreated)) + print('') + + for i in spec: + if i.get('space',[]) in [r for r in ROOMStobecreated]: + create_space(i.get('space',[]), ADMINAUTH) + if i.get('room',[]) in [r for r in ROOMStobecreated]: + create_room(i.get('room',[]),ADMINAUTH) + #theoretically the code could be easily changed in a way that the rooms could be directly created by a designated person that becomes then roomadmin of the room, + #however, matrix-corporal relies on the fact that a single admin accoount is enrolled in all the managed rooms + + print('Recomputing rooms map...', end='') + existingrooms = matrix_compute_rooms_map() + # print(existingrooms) + + print('===Move Rooms to their parent room/space...', end='') + print(' ') + for i in spec: + if i.get('room',[]) in [r for r in ROOMStobecreated]: + if i.get('childof',None) is not None: + move_room(i.get('room',[]),i.get('childof',[]),existingrooms,ADMINAUTH) + + + ## add managedRoomIds -ROOMS_IDS = [ matrix_room_id(r) for r in ROOMS ] +ROOMS=[r for r in existingrooms if r.get('name') in spec_rooms] +print(spec_rooms) +ROOMS_IDS = [ matrix_room_id(rs) for rs in ROOMS ] policy_update_rooms(ROOMS_IDS) -## compute RESTRICTED room list from spec for all matrixgroups -## (i.e., rooms from each matrixgroup for which not all matrixgroup members have access) -RESTRICTED = [] -for r in [ s['restricted'] for s in spec if 'restricted' in s ]: - RESTRICTED += r -RESTRICTEDUSERS = [] -RESTRICTEDGROUPS = [] -for room in RESTRICTED: - RESTRICTEDUSERS += room.get('users', []) - RESTRICTEDGROUPS += room.get('groups', []) -RESTRICTEDUSERS = sorted(set(RESTRICTEDUSERS)) -RESTRICTEDGROUPS = sorted(set(RESTRICTEDGROUPS)) - -## compute total LDAPGROUPS list from spec (starting with the restricted ones) + + +### compute total LDAPGROUPS list from spec (starting with the restricted ones) print('Computing LDAP groups...', end='') -LDAPGROUPS = RESTRICTEDGROUPS -for g in [ s['ldapgroups'] for s in spec if 'ldapgroups' in s ]: +LDAPGROUPS=[] +for i in spec: + LDAPGROUPS += i.get('ldapgroups',[]) +for g in [spec_flags.get('ldapgroups-forbidroomcreation',[])]: + LDAPGROUPS += g +for g in [spec_flags.get('ldapgroups-forbidencryptedroomcreation',[])]: + LDAPGROUPS += g +for g in [spec_flags.get('ldapgroups-forbidunencryptedroomcreation',[])]: LDAPGROUPS += g LDAPGROUPS = sorted(set(LDAPGROUPS)) print('Done.') print('') -print("=== LDAP groups that are specified in the rooms ===") +print("=== LDAP groups that are specified in any rooms or spaces ===") print(LDAPGROUPS) print('') -## compute total LDAPUSERS list from spec (starting with the restricted ones) +## compute total LDAPUSERS list from spec print('Computing LDAP users...', end='') -LDAPUSERS = RESTRICTEDUSERS +LDAPUSERS = [] for u in [ s['ldapusers'] for s in spec if 'ldapusers' in s ]: - LDAPUSERS += u + LDAPUSERS += u LDAPUSERS = sorted(set(LDAPUSERS)) print('Done.') print('') print("=== LDAP users that are specified in the rooms ===") -print(LDAPUSERS) +#print(LDAPUSERS) print('') + ## init LDAP connection print ('Connecting to LDAP server...', end='') ldap_conn = ldap_init() @@ -261,8 +383,9 @@ def policy_update_users(list_of_users): ## compute users of LDAP: list all users print ('Computing users list...', end='') active_users = ldap_get_usernames(LDAP_USER_QUERY) +#inactive_users= ldap_get_usernames('(&(objectclass=inetorgperson)(|(|(logindisabled=true)(ou=Rossental))(initials=noLehrer)))') print(str(len(active_users)) + " users found.") - +#print(active_users) ## then for each LDAP group mentioned in the spec, compute LDAP users that are in required group print ('Computing LDAP group users for each group...') USERSINGROUP = {} @@ -277,48 +400,80 @@ def policy_update_users(list_of_users): USERS += LDAPUSERS USERS = sorted(set(USERS)) print('') -print(f'=== Will generate policy for {len(USERS)} Matrix users ===') +#print(f'=== Will generate policy for {len(USERS)} Matrix users ===') print("") ## compute all rooms IDs for each USERS and store it in USER_DATA: dict { : dict {:[] :[]} } USER_DATA = {} for user in USERS: USER_DATA[user] = {} - USER_DATA[user]['matrix-groups'] = [] + # USER_DATA[user]['matrix-groups'] = [] USER_DATA[user]['matrix-rooms'] = [] + USER_DATA[user]['displayName'] = {} + USER_DATA[user]['forbidroomcreation'] = False + USER_DATA[user]['forbidencryptedroomcreation'] = False + USER_DATA[user]['forbidunencryptedroomcreation'] = False + for i in spec: - # get all data fields of a given matrixgroup - matrixgroup = i.get('matrixgroup') - matrixrooms = i.get('rooms', []) - ldapusers = i.get('ldapusers', []) - ldapgroups = i.get('ldapgroups', []) - restrictedrooms = i.get('restricted', []) - print(f'Matrix group {matrixgroup} includes rooms {matrixrooms}, LDAP groups {ldapgroups}...') + # # get all data fields of a given matrixgroup + # matrixgroup = i.get('matrixgroup') + ROOM = i.get('room', i.get('space',[])) + ldapusers = i.get('ldapusers', []) + ldapgroups = i.get('ldapgroups', []) + # restrictedrooms = i.get('restricted', []) + print(f'Room {ROOM} includes LDAP groups {ldapgroups}...') # compute a global user list of all the groups - for group in ldapgroups: - ldapusers += USERSINGROUP[group] + for group in ldapgroups: + ldapusers += USERSINGROUP[group] # for each user: add group+rooms membership to its data - for user in ldapusers: - if matrixgroup not in USER_DATA[user]['matrix-groups']: - USER_DATA[user]['matrix-groups'].append(matrixgroup) - rooms_names = list(map(lambda name:matrix_room_id(ROOMS,name), matrixrooms)) - USER_DATA[user]['matrix-rooms'] += [ i for i in rooms_names if i is not None ] - # for each restricted room, for each of its user, add group+rooms membership - for room in restrictedrooms: - print(f'Restricted room: {room}...') - restrictedusers = room.get('users', []) - for restrictedgroup in room.get('groups', []): - #print(f' + adding restricted users from {restrictedgroup}: ', end='') - #print(USERSINGROUP[restrictedgroup]) - restrictedusers += USERSINGROUP[restrictedgroup] - for user in restrictedusers: - #print(f' + adding additional restricted users {user}:') - if matrixgroup not in USER_DATA[user]['matrix-groups']: - USER_DATA[user]['matrix-groups'].append(matrixgroup) - id = matrix_room_id(ROOMS,room.get('room')) - if id is not None and id not in USER_DATA[user]['matrix-rooms']: + for user in ldapusers: + # if matrixgroup not in USER_DATA[user]['matrix-groups']: + # USER_DATA[user]['matrix-groups'].append(matrixgroup) + id = matrix_room_id(existingrooms, ROOM) + if id is not None and id not in USER_DATA[user]['matrix-rooms']: USER_DATA[user]['matrix-rooms'].append(id) + +#old restricted group code + # if matrixgroup not in USER_DATA[user]['matrix-groups']: + # USER_DATA[user]['matrix-groups'].append(matrixgroup) + # rooms_names = list(map(lambda name:matrix_room_id(ROOMS,name), matrixrooms)) + # USER_DATA[user]['matrix-rooms'] += [ i for i in rooms_names if i is not None ] + # for each restricted room, for each of its user, add group+rooms membership + # for room in restrictedrooms: + # print(f'Restricted room: {room}...') + # restrictedusers = room.get('users', []) + # for restrictedgroup in room.get('groups', []): + # restrictedusers += USERSINGROUP[restrictedgroup] + # for user in restrictedusers: + #print(f' + adding additional restricted users {user}:') + # if matrixgroup not in USER_DATA[user]['matrix-groups']: + # USER_DATA[user]['matrix-groups'].append(matrixgroup) + # id = matrix_room_id(ROOMS,room.get('room')) + # if id is not None and id not in USER_DATA[user]['matrix-rooms']: + # USER_DATA[user]['matrix-rooms'].append(id) + +#set flags for users +forbidroomcreationusers=[] +forbidencryptedroomcreationusers=[] +forbidunencryptedroomcreationusers=[] + +for group in spec_flags.get('ldapgroups-forbidroomcreation',[]): + forbidroomcreationusers += USERSINGROUP[group] +for user in forbidroomcreationusers: + USER_DATA[user]['forbidroomcreation'] = True +for group in spec_flags.get('ldapgroups-forbidencryptedroomcreation',[]): + forbidencryptedroomcreationusers += USERSINGROUP[group] +for user in forbidencryptedroomcreationusers: + USER_DATA[user]['forbidencryptedroomcreation'] = True +for group in spec_flags.get('ldapgroups-forbidunencryptedroomcreation',[]): + forbidunencryptedroomcreationusers += USERSINGROUP[group] +for user in forbidunencryptedroomcreationusers: + USER_DATA[user]['forbidunencryptedroomcreation'] = True + + + + ## For each user, add its policy section list = [] for user in USERS: @@ -329,6 +484,9 @@ def policy_update_users(list_of_users): user_data['authType'] = "rest" user_data['joinedCommunityIds'] = USER_DATA[user]['matrix-groups'] user_data['joinedRoomIds'] = USER_DATA[user]['matrix-rooms'] + user_data['forbidRoomCreation'] = USER_DATA[user]['forbidroomcreation'] + user_data['forbidUnencryptedRoomCreation'] = USER_DATA[user]['forbidunencryptedroomcreation'] + user_data['forbidEncryptedRoomCreation'] = USER_DATA[user]['forbidencryptedroomcreation'] list.append(user_data) policy_update_users(list)