From c329e53811d640e21ee82cabb3bea747af58197b Mon Sep 17 00:00:00 2001
From: Carpentier Pierre-Francois <carpentier.pf@gmail.com>
Date: Wed, 7 Feb 2018 19:52:29 +0100
Subject: [PATCH 01/82] Update README.rst

---
 README.rst | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/README.rst b/README.rst
index 3685447..266d152 100644
--- a/README.rst
+++ b/README.rst
@@ -30,6 +30,21 @@ Nice and simple application to manage users and groups in multiple directory ser
 
 ----
 
+********
+  Demo
+********
+
+A demo is accessible there: https://ldapcherry.kakwalab.ovh
+
+The Credential are:
+
+* as administrator: admin/admin
+* as user: user/user
+
+Please take note that it's not possible to modify/delete the 'admin' and 'user' users.
+
+Also take note that the service will be reseted once per day.
+
 ****************
   Presentation
 ****************

From 1ed654c91bf2caeedbad34944bf837b6c21a2292 Mon Sep 17 00:00:00 2001
From: Carpentier Pierre-Francois <carpentier.pf@gmail.com>
Date: Wed, 7 Feb 2018 19:54:23 +0100
Subject: [PATCH 02/82] Update README.rst

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 266d152..330eb88 100644
--- a/README.rst
+++ b/README.rst
@@ -34,9 +34,9 @@ Nice and simple application to manage users and groups in multiple directory ser
   Demo
 ********
 
-A demo is accessible there: https://ldapcherry.kakwalab.ovh
+A demo is accessible here: https://ldapcherry.kakwalab.ovh
 
-The Credential are:
+The credentials are:
 
 * as administrator: admin/admin
 * as user: user/user

From 6f98076281e9452fdb1adcd1bcbb70a6f968ade9 Mon Sep 17 00:00:00 2001
From: John Thiltges <jthiltges2@unl.edu>
Date: Wed, 2 Jan 2019 14:31:10 -0600
Subject: [PATCH 03/82] Protect against XSS vulnerabilities in URL redirection

- Switch from base64 to URL encoding for the passing the URL, using the built-in Mako filtering
- Apply HTML filtering to Mako output by default
- Disable HTML filtering for nested templates in adduser, modify, and selfmodify
---
 ldapcherry/__init__.py              | 23 ++++++++++++-----------
 resources/templates/adduser.tmpl    |  4 ++--
 resources/templates/login.tmpl      | 14 +++++++-------
 resources/templates/modify.tmpl     |  4 ++--
 resources/templates/selfmodify.tmpl |  2 +-
 5 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index 60ce654..a7d5c18 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -15,7 +15,7 @@
 import logging.handlers
 from operator import itemgetter
 from socket import error as socket_error
-import base64
+import urllib
 import cgi
 
 from exceptions import *
@@ -387,7 +387,8 @@ def _load_templates(self, config):
         )
         # preload templates
         self.temp_lookup = lookup.TemplateLookup(
-            directories=self.template_dir, input_encoding='utf-8'
+            directories=self.template_dir, input_encoding='utf-8',
+            default_filters=['unicode', 'h']
             )
         # load each template
         self.temp = {}
@@ -573,7 +574,7 @@ def _check_session(self):
 
     def _check_auth(self, must_admin, redir_login=True):
         """ check if a user is autheticated and, optionnaly an administrator
-        if user not authentifaced -> redirection to login page (with base64
+        if user not authenticated -> redirect to login page (with escaped URL
             of the originaly requested page (redirection after login)
         if user authenticated, not admin and must_admin enabled -> 403 error
         @boolean must_admin: flag "user must be an administrator to access
@@ -588,13 +589,13 @@ def _check_auth(self, must_admin, redir_login=True):
             qs = ''
         else:
             qs = '?' + cherrypy.request.query_string
-        # base64 of the requested URL
-        b64requrl = base64.b64encode(cherrypy.url() + qs)
+        # Escaped version of the requested URL
+        quoted_requrl = urllib.quote_plus(cherrypy.url() + qs)
         if not username:
-            # return to login page (with base64 of the url in query string
+            # return to login page (with quoted url in query string)
             if redir_login:
                 raise cherrypy.HTTPRedirect(
-                    "/signin?url=%(url)s" % {'url': b64requrl},
+                    "/signin?url=%(url)s" % {'url': quoted_requrl},
                     )
             else:
                 raise cherrypy.HTTPError(
@@ -606,7 +607,7 @@ def _check_auth(self, must_admin, redir_login=True):
                 or not cherrypy.session['connected']:
             if redir_login:
                 raise cherrypy.HTTPRedirect(
-                    "/signin?url=%(url)s" % {'url': b64requrl},
+                    "/signin?url=%(url)s" % {'url': quoted_requrl},
                     )
             else:
                 raise cherrypy.HTTPError(
@@ -631,7 +632,7 @@ def _check_auth(self, must_admin, redir_login=True):
         else:
             if redir_login:
                 raise cherrypy.HTTPRedirect(
-                    "/signin?url=%(url)s" % {'url': b64requrl},
+                    "/signin?url=%(url)s" % {'url': quoted_requrl},
                     )
             else:
                 raise cherrypy.HTTPError(
@@ -919,7 +920,7 @@ def login(self, login, password, url=None):
             if url is None:
                 redirect = "/"
             else:
-                redirect = base64.b64decode(url)
+                redirect = url
             raise cherrypy.HTTPRedirect(redirect)
         else:
             message = "login failed for user '%(user)s'" % {
@@ -932,7 +933,7 @@ def login(self, login, password, url=None):
             if url is None:
                 qs = ''
             else:
-                qs = '?url=' + url
+                qs = '?url=' + urllib.quote_plus(url)
             raise cherrypy.HTTPRedirect("/signin" + qs)
 
     @cherrypy.expose
diff --git a/resources/templates/adduser.tmpl b/resources/templates/adduser.tmpl
index 7b690fc..8b27f75 100644
--- a/resources/templates/adduser.tmpl
+++ b/resources/templates/adduser.tmpl
@@ -9,11 +9,11 @@
               <form method='POST' autocomplete="off" action='/adduser' role="form" class="form-signin" id=form>
               <fieldset>
               <legend>Fill new user's attributes:</legend>
-              ${form}
+              ${form | n}
               </fieldset>
               <fieldset>
               <legend>Enable/Disable user's roles:</legend>
-              ${roles}
+              ${roles | n}
               </fieldset>
               <div class="form-group">
                 <div class="input-group">
diff --git a/resources/templates/login.tmpl b/resources/templates/login.tmpl
index d325702..3bc6ca0 100644
--- a/resources/templates/login.tmpl
+++ b/resources/templates/login.tmpl
@@ -4,13 +4,13 @@
     <div class="row clearfix" style="margin-top:30px">
         <div class="col-md-4 column"></div>
         <div class="col-md-4 column well">
-<%
-if url is None:
-    qs=''
-else:
-    qs='?url=' + url
-%>
-            <form method='POST' action='/login${qs}' role="form" class="form-signin">
+            <form method='POST' role="form" class="form-signin"
+% if url:
+                action='login?url=${url | u}'
+% else:
+                action='login'
+% endif
+            >
               <div class="form-group">
               <h2 class="form-signin-heading">Please sign in</h2>
                 <div class="input-group">
diff --git a/resources/templates/modify.tmpl b/resources/templates/modify.tmpl
index dfeab49..b494fcc 100644
--- a/resources/templates/modify.tmpl
+++ b/resources/templates/modify.tmpl
@@ -9,11 +9,11 @@
               <form method='POST' action='/modify' role="form" class="form-signin" id="form">
               <fieldset>
               <legend>Modify user's attributes:</legend>
-              ${form}
+              ${form | n}
               </fieldset>
               <fieldset>
               <legend>Enable/Disable user's roles:</legend>
-              ${roles}
+              ${roles | n}
               </fieldset>
               % if len(standalone_groups) != 0:
               <fieldset>
diff --git a/resources/templates/selfmodify.tmpl b/resources/templates/selfmodify.tmpl
index ee58c77..95bd806 100644
--- a/resources/templates/selfmodify.tmpl
+++ b/resources/templates/selfmodify.tmpl
@@ -8,7 +8,7 @@
             <div class="well well-sm">
               <form method='POST' action='/selfmodify' autocomplete="off" role="form" class="form-signin" id="form">
               <legend>Modify your attributes:</legend>
-              ${form}
+              ${form | n}
               </fieldset>
               <div class="form-group">
                 <div class="input-group">

From 1f79648d5739e6f604b40daec4c19ca706b64a46 Mon Sep 17 00:00:00 2001
From: Carpentier Pierre-Francois <carpentier.pf@gmail.com>
Date: Wed, 2 Jan 2019 23:59:03 +0100
Subject: [PATCH 04/82] Update ChangeLog.rst

---
 ChangeLog.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 2597668..1c009fa 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,9 @@
 Dev
 ***
 
+* [sec ] fix XSS injection in the url redirect in the login page (thanks to jthiltges)
+* [impr] more systematic use of html and url escaping in the html rendering to prevent against content injection (thanks to jthiltges)
+
 Version 0.5.2
 *************
 

From c6cce54d5f2496b686a7102306c9e8c08859a078 Mon Sep 17 00:00:00 2001
From: John Thiltges <jthiltges2@unl.edu>
Date: Thu, 3 Jan 2019 13:12:53 -0600
Subject: [PATCH 05/82] Escape form values with markupsafe

- Use markupsafe to format escaped HTML fragments
- Correct the formatting problems introduced with the XSS fixes
---
 resources/templates/form.tmpl | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/resources/templates/form.tmpl b/resources/templates/form.tmpl
index c984641..243b173 100644
--- a/resources/templates/form.tmpl
+++ b/resources/templates/form.tmpl
@@ -1,5 +1,6 @@
 ## -*- coding: utf-8 -*-
 <% 
+from markupsafe import Markup
 len_attr = len(attributes)
 switch = len_attr / 2
 if not switch * 2 == len_attr:
@@ -31,32 +32,32 @@ for a in sorted(attributes.keys(), key=lambda attr: attributes[attr]['weight']):
             raw_value = values[a]
         if raw_value is None:
            raw_value = ''
-        value = ' value="'+ raw_value + '"'
-        value2 = '<option>'+ raw_value +'</option>'
+        value = Markup(' value="{}"').format(raw_value)
+        value2 = Markup('<option>{}</option>').format(raw_value)
     else:
         raw_value = ''
         value = ''
         value2 = ''
     if 'default' in attr and value == '':
-        value = ' value="'+ attr['default'] + '"'
+        value = Markup(' value="{}"').format(attr['default'])
   %>
 
   <span class="input-group-addon" id="basic-addon-${a}">${attr['display_name']}</span>
     % if modify and a == keyattr:
-  <input type="hidden" id="attr.${a}" name="attr.${a}" class="form-control" autocomplete='off' aria-describedby="basic-addon-${a}" ${required} ${value} readonly  onfocus="this.removeAttribute('readonly');">
+  <input type="hidden" id="attr.${a}" name="attr.${a}" class="form-control" autocomplete='off' aria-describedby="basic-addon-${a}" ${required} ${value | n} readonly  onfocus="this.removeAttribute('readonly');">
   <span class="form-control" aria-describedby="basic-addon-${a}">${raw_value}</span>
     % elif attr['type'] == 'string':
-  <input type="text"   id="attr.${a}" name="attr.${a}" class="form-control" autocomplete='off' placeholder="${attr['description']}" aria-describedby="basic-addon-${a}" ${required} ${value} readonly  onfocus="this.removeAttribute('readonly');">
+  <input type="text"   id="attr.${a}" name="attr.${a}" class="form-control" autocomplete='off' placeholder="${attr['description']}" aria-describedby="basic-addon-${a}" ${required} ${value | n} readonly  onfocus="this.removeAttribute('readonly');">
     % elif attr['type'] == 'email':
-  <input type="email"  id="attr.${a}" name="attr.${a}" class="form-control" autocomplete='off' placeholder="${attr['description']}" aria-describedby="basic-addon-${a}" ${required} ${value} data-error="email address is invalid" readonly  onfocus="this.removeAttribute('readonly');">
+  <input type="email"  id="attr.${a}" name="attr.${a}" class="form-control" autocomplete='off' placeholder="${attr['description']}" aria-describedby="basic-addon-${a}" ${required} ${value | n} data-error="email address is invalid" readonly  onfocus="this.removeAttribute('readonly');">
     % elif attr['type'] == 'int':
-  <input type="number" id="attr.${a}" name="attr.${a}" class="form-control" autocomplete='off' placeholder="${attr['description']}" aria-describedby="basic-addon-${a}" ${required} ${value} readonly  onfocus="this.removeAttribute('readonly');">
+  <input type="number" id="attr.${a}" name="attr.${a}" class="form-control" autocomplete='off' placeholder="${attr['description']}" aria-describedby="basic-addon-${a}" ${required} ${value | n} readonly  onfocus="this.removeAttribute('readonly');">
     % elif attr['type'] == 'fix':
   <input type="hidden" id="attr.${a}" name="attr.${a}" class="form-control" autocomplete='off' aria-describedby="basic-addon-${a}" ${required} value="${attr['value']}" readonly  onfocus="this.removeAttribute('readonly');">
   <span class="form-control" placeholder="${attr['description']}" aria-describedby="basic-addon-${a}">${attr['value']}</span>
     % elif attr['type'] == 'stringlist':
   <select class="form-control" id="attr.${a}" name="attr.${a}">
-        ${value2}
+        ${value2 | n}
         %for val in attr['values']:
         %if '<option>' + val + '</option>' != value2:
         <option>${val}</option>

From 2df56d2de23229b670d9fbc8b345aadaf0812707 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 6 Feb 2019 21:38:11 +0100
Subject: [PATCH 06/82] fix template over-escaping + python 3 support

The templates were html escaping the generated js code for the
autofill and the role management. This was breaking these features.
It's okay to not escape these as they are coming from a trusted source
(configuration file).

Also make the templates python3 compatible (not need to import Set in
python 3)
---
 resources/templates/base.tmpl  | 2 +-
 resources/templates/form.tmpl  | 9 ++++++---
 resources/templates/roles.tmpl | 4 ++--
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/resources/templates/base.tmpl b/resources/templates/base.tmpl
index eed3e54..2c25f8a 100644
--- a/resources/templates/base.tmpl
+++ b/resources/templates/base.tmpl
@@ -58,7 +58,7 @@
 <body>
 % if notifications:
 % for notif in notifications:
-<script type="text/javascript">$.notify('${notif}')</script>
+<script type="text/javascript">$.notify('${notif | n}')</script>
 % endfor
 % endif
 <div class="container">
diff --git a/resources/templates/form.tmpl b/resources/templates/form.tmpl
index 243b173..3c87e11 100644
--- a/resources/templates/form.tmpl
+++ b/resources/templates/form.tmpl
@@ -86,8 +86,11 @@ ${form_col(lc2)}
 </div>
 % if autofill:
 <%
-from sets import Set
-attr_set = Set([])
+import sys
+if sys.version < '3':
+    from sets import Set as set
+
+attr_set = set([])
 attr_events = {}
 functions = {}
 for attrid in attributes:
@@ -122,7 +125,7 @@ if (fields['${attrid}'] != null) {
     fields['${attrid}'].onchange = function () {
     % for tuple in attr_events[attrid]:
         if (typeof(${tuple[1]}) == "function") {
-            fields['${tuple[0]}'].value = ${tuple[1]}(${', '.join(functions[tuple])});
+            fields['${tuple[0]}'].value = ${tuple[1] | n}(${', '.join(functions[tuple]) | n});
         }
     % endfor
     };
diff --git a/resources/templates/roles.tmpl b/resources/templates/roles.tmpl
index d97457d..5fa6ac3 100644
--- a/resources/templates/roles.tmpl
+++ b/resources/templates/roles.tmpl
@@ -1,7 +1,7 @@
 ## -*- coding: utf-8 -*-
 <script type="text/javascript">
-    var graph = ${graph_js};
-    var roles = ${roles_js};
+    var graph = ${graph_js | n};
+    var roles = ${roles_js | n};
     function enableParentRoles(roleid){
         var parentRoles = graph[roleid]['parent_roles'];
         var DnRole = roles[roleid];

From 921a0820f4a307bd1ac696204e057578ecd7da03 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 6 Feb 2019 22:26:46 +0100
Subject: [PATCH 07/82] switch to using lists in templates

Sets are not available in mako templates when using python3.
Reverting to using lists with 'if not in' checks to avoid duplication.
---
 resources/templates/form.tmpl | 42 +++++++++++++++++------------------
 1 file changed, 20 insertions(+), 22 deletions(-)

diff --git a/resources/templates/form.tmpl b/resources/templates/form.tmpl
index 3c87e11..068eb61 100644
--- a/resources/templates/form.tmpl
+++ b/resources/templates/form.tmpl
@@ -86,33 +86,31 @@ ${form_col(lc2)}
 </div>
 % if autofill:
 <%
-import sys
-if sys.version < '3':
-    from sets import Set as set
-
-attr_set = set([])
+attr_set = []
 attr_events = {}
 functions = {}
 for attrid in attributes:
     attr = attributes[attrid]
     field = 'attr.' + attrid
-    attr_set.add(field)
-    if 'autofill' in attr:
-        function = attr['autofill']['function']
-        tuple = (field, function)
-        if not tuple in functions:
-            functions[tuple] = []
-        for arg in attr['autofill']['args']:
-            if arg[0] == '$':
-                field_arg = 'attr.' + arg[1:]
-                attr_set.add(field_arg)
-                functions[tuple].append("fields['" + field_arg + "'].value")
-                if not field_arg in attr_events:
-                    attr_events[field_arg] = []
-                attr_events[field_arg].append(tuple) 
-            else:
-                value = arg
-                functions[tuple].append("'" + value + "'")
+    if field not in attr_set:
+        attr_set.append(field)
+        if 'autofill' in attr:
+            function = attr['autofill']['function']
+            tuple = (field, function)
+            if not tuple in functions:
+                functions[tuple] = []
+            for arg in attr['autofill']['args']:
+                if arg[0] == '$':
+                    field_arg = 'attr.' + arg[1:]
+                    if field_arg not in attr_set:
+                        attr_set.append(field_arg)
+                    functions[tuple].append("fields['" + field_arg + "'].value")
+                    if not field_arg in attr_events:
+                        attr_events[field_arg] = []
+                    attr_events[field_arg].append(tuple)
+                else:
+                    value = arg
+                    functions[tuple].append("'" + value + "'")
 %>
 <script>
 var fields = new Object();

From 69526610f38b604b3c72fb230182ee3c29efa0d2 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 6 Feb 2019 22:30:59 +0100
Subject: [PATCH 08/82] add a small script to generate a local dev config

---
 goodies/gen-dev-conf.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100755 goodies/gen-dev-conf.sh

diff --git a/goodies/gen-dev-conf.sh b/goodies/gen-dev-conf.sh
new file mode 100755
index 0000000..432da6f
--- /dev/null
+++ b/goodies/gen-dev-conf.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+ROOT=$(readlink -f $(dirname $0)/../)
+
+cp $ROOT/conf/ldapcherry.ini $ROOT/ldapcherry-dev.ini
+
+sed -i "s|/etc/ldapcherry/|$ROOT/conf/|" $ROOT/ldapcherry-dev.ini
+sed -i "s|/usr/share/ldapcherry/|$ROOT/resources/|" $ROOT/ldapcherry-dev.ini
+sed -i "s|^ldap\.|#ldap.|" $ROOT/ldapcherry-dev.ini
+sed -i "s|#demo\.|ldap.|" $ROOT/ldapcherry-dev.ini
+
+GROUPS='cn=nagios admins\\,ou=Group\\,dc=example\\,dc=org, cn=users\\,ou=Group\\,dc=example\\,dc=org'
+sed -i "s|ldap.admin.groups.*|ldap.admin.groups = '$GROUPS'|" $ROOT/ldapcherry-dev.ini
+
+
+sed -i "s|^min_length.*|min_length = 3|" $ROOT/ldapcherry-dev.ini
+sed -i "s|^min_upper.*|min_upper = 0|" $ROOT/ldapcherry-dev.ini
+sed -i "s|^min_digit.*|min_digit = 0|" $ROOT/ldapcherry-dev.ini

From 74dc6c58941b61f838ee4e9cfcbc25df3c7b86c7 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 6 Feb 2019 22:32:40 +0100
Subject: [PATCH 09/82] various changes to support python3

* changes in urllib imports since quote_plus in urllib with python 2 and
in urllib.parse in python 3
* changes in imports for Sets since set is a native type in python 3 and
doesn't requires an import
* fix in __import__, '-1' level for module path discovery is not supported
anymore, switching to 0 (absolute import only).
---
 ldapcherry/__init__.py            | 39 +++++++++++++++++-------------
 ldapcherry/attributes.py          |  6 +++--
 ldapcherry/backend/backendDemo.py | 24 ++++++++++++-------
 ldapcherry/backend/backendLdap.py | 11 +++++----
 ldapcherry/roles.py               | 40 ++++++++++++++++---------------
 5 files changed, 70 insertions(+), 50 deletions(-)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index a7d5c18..33b2063 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -15,10 +15,9 @@
 import logging.handlers
 from operator import itemgetter
 from socket import error as socket_error
-import urllib
 import cgi
 
-from exceptions import *
+from ldapcherry.exceptions import *
 from ldapcherry.lclogging import *
 from ldapcherry.roles import Roles
 from ldapcherry.attributes import Attributes
@@ -31,7 +30,13 @@
 from mako.template import Template
 from mako import lookup
 from mako import exceptions
-from sets import Set
+
+
+if sys.version < '3':
+    from sets import Set as set
+    from urllib import quote_plus
+else:
+    from urllib.parse import quote_plus
 
 SESSION_KEY = '_cp_username'
 
@@ -68,8 +73,8 @@ def _escape_dict(self, data):
                 data[d] = self._escape_list(data[d])
             elif isinstance(data[d], dict):
                 data[d] = self._escape_dict(data[d])
-            elif isinstance(data[d], Set):
-                data[d] = Set(self._escape_list(data[d]))
+            elif isinstance(data[d], set):
+                data[d] = set(self._escape_list(data[d]))
             else:
                 data[d] = cgi.escape(data[d], True)
         return data
@@ -178,7 +183,7 @@ def _init_backends(self, config):
             except Exception as e:
                 raise MissingParameter('backends', backend + '.module')
             try:
-                bc = __import__(module, globals(), locals(), ['Backend'], -1)
+                bc = __import__(module, globals(), locals(), ['Backend'], 0)
             except Exception as e:
                 self._handle_exception(e)
                 raise BackendModuleLoadingFail(module)
@@ -219,7 +224,7 @@ def _init_ppolicy(self, config):
             'ldapcherry.ppolicy'
         )
         try:
-            pp = __import__(module, globals(), locals(), ['PPolicy'], -1)
+            pp = __import__(module, globals(), locals(), ['PPolicy'], 0)
         except:
             raise BackendModuleLoadingFail(module)
         if 'ppolicy' in config:
@@ -590,7 +595,7 @@ def _check_auth(self, must_admin, redir_login=True):
         else:
             qs = '?' + cherrypy.request.query_string
         # Escaped version of the requested URL
-        quoted_requrl = urllib.quote_plus(cherrypy.url() + qs)
+        quoted_requrl = quote_plus(cherrypy.url() + qs)
         if not username:
             # return to login page (with quoted url in query string)
             if redir_login:
@@ -695,7 +700,7 @@ def _adduser(self, params):
                 roles.append(r)
         groups = self.roles.get_groups(roles)
         for b in groups:
-            self.backends[b].add_to_groups(username, Set(groups[b]))
+            self.backends[b].add_to_groups(username, set(groups[b]))
 
         cherrypy.log.error(
             msg="user '" + username + "' made member of " +
@@ -823,10 +828,10 @@ def _modify(self, params):
                 if b not in g:
                     g[b] = []
             tmp = \
-                Set(groups_add[b]) - \
-                Set(groups_keep[b]) - \
-                Set(groups_current[b]) - \
-                Set(lonely_groups[b])
+                set(groups_add[b]) - \
+                set(groups_keep[b]) - \
+                set(groups_current[b]) - \
+                set(lonely_groups[b])
             cherrypy.log.error(
                 msg="user '" + username + "' added to groups: " +
                     str(list(tmp)) + " in backend '" + b + "'",
@@ -840,11 +845,11 @@ def _modify(self, params):
                     g[b] = []
             tmp = \
                 (
-                    (Set(groups_rm[b]) | Set(groups_remove[b])) -
-                    (Set(groups_keep[b]) | Set(groups_add[b]))
+                    (set(groups_rm[b]) | set(groups_remove[b])) -
+                    (set(groups_keep[b]) | set(groups_add[b]))
                 ) & \
                 (
-                    Set(groups_current[b]) | Set(lonely_groups[b])
+                    set(groups_current[b]) | set(lonely_groups[b])
                 )
             cherrypy.log.error(
                 msg="user '" + username + "' removed from groups: " +
@@ -933,7 +938,7 @@ def login(self, login, password, url=None):
             if url is None:
                 qs = ''
             else:
-                qs = '?url=' + urllib.quote_plus(url)
+                qs = '?url=' + quote_plus(url)
             raise cherrypy.HTTPRedirect("/signin" + qs)
 
     @cherrypy.expose
diff --git a/ldapcherry/attributes.py b/ldapcherry/attributes.py
index 01fa396..75c863e 100644
--- a/ldapcherry/attributes.py
+++ b/ldapcherry/attributes.py
@@ -12,9 +12,11 @@
 from ldapcherry.pyyamlwrapper import loadNoDump
 from ldapcherry.pyyamlwrapper import DumplicatedKey
 from ldapcherry.exceptions import *
-from sets import Set
 import yaml
 
+if sys.version < '3':
+    from sets import Set as set
+
 # List of available types for form
 types = ['string', 'textfield', 'email', 'int', 'stringlist',
          'fix', 'password']
@@ -24,7 +26,7 @@ class Attributes:
 
     def __init__(self, attributes_file):
         self.attributes_file = attributes_file
-        self.backends = Set([])
+        self.backends = set([])
         self.self_attributes = {}
         self.backend_attributes = {}
         self.displayed_attributes = {}
diff --git a/ldapcherry/backend/backendDemo.py b/ldapcherry/backend/backendDemo.py
index db3559b..0377bad 100644
--- a/ldapcherry/backend/backendDemo.py
+++ b/ldapcherry/backend/backendDemo.py
@@ -7,7 +7,11 @@
 
 # This is a demo backend
 
-from sets import Set
+
+import sys
+if sys.version < '3':
+    from sets import Set as set
+
 import ldapcherry.backend
 from ldapcherry.exceptions import UserDoesntExist, \
     GroupDoesntExist, MissingParameter, \
@@ -37,12 +41,12 @@ def __init__(self, config, logger, name, attrslist, key):
         self.backend_name = name
         admin_user = self.get_param('admin.user', 'admin')
         admin_password = self.get_param('admin.password', 'admin')
-        admin_groups = Set(re.split('\W+', self.get_param('admin.groups')))
+        admin_groups = set(self._basic_splitter(self.get_param('admin.groups')))
         basic_user = self.get_param('basic.user', 'user')
         basic_password = self.get_param('basic.password', 'user')
-        basic_groups = Set(re.split('\W+', self.get_param('basic.groups')))
+        basic_groups = set(self._basic_splitter(self.get_param('basic.groups')))
         pwd_attr = self.get_param('pwd_attr')
-        self.search_attrs = Set(
+        self.search_attrs = set(
             re.split('\W+', self.get_param('search_attributes')),
             )
         self.pwd_attr = pwd_attr
@@ -60,6 +64,10 @@ def __init__(self, config, logger, name, attrslist, key):
                 'groups': basic_groups,
                 }
 
+    @staticmethod
+    def _basic_splitter(in_str):
+        return [re.sub(r'(?<!\\)\\', '', x) for x in re.split(r'(?<!\\),\W*', in_str)]
+
     def _check_fix_users(self, username):
         if self.admin_user == username or self.basic_user == username:
             raise Exception('User cannot be modified')
@@ -91,7 +99,7 @@ def add_user(self, attrs):
         if username in self.users:
             raise UserAlreadyExists(username, self.backend_name)
         self.users[username] = attrs
-        self.users[username]['groups'] = Set([])
+        self.users[username]['groups'] = set([])
 
     def del_user(self, username):
         """ Delete a user from the backend
@@ -107,7 +115,7 @@ def del_user(self, username):
             raise UserDoesntExist(username, self.backend_name)
 
     def set_attrs(self, username, attrs):
-        """ Set a list of attributes for a given user
+        """ set a list of attributes for a given user
 
         :param username: 'key' attribute of the user
         :type username: string
@@ -128,7 +136,7 @@ def add_to_groups(self, username, groups):
         """
         self._check_fix_users(username)
         current_groups = self.users[username]['groups']
-        new_groups = current_groups | Set(groups)
+        new_groups = current_groups | set(groups)
         self.users[username]['groups'] = new_groups
 
     def del_from_groups(self, username, groups):
@@ -143,7 +151,7 @@ def del_from_groups(self, username, groups):
         """
         self._check_fix_users(username)
         current_groups = self.users[username]['groups']
-        new_groups = current_groups - Set(groups)
+        new_groups = current_groups - set(groups)
         self.users[username]['groups'] = new_groups
 
     def search(self, searchstring):
diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index c36ba9f..5ac16e1 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -11,7 +11,10 @@
 import ldap.filter
 import logging
 import ldapcherry.backend
-from sets import Set
+import sys
+if sys.version < '3':
+    from sets import Set as set
+
 from ldapcherry.exceptions import UserDoesntExist, \
     GroupDoesntExist, \
     UserAlreadyExists
@@ -74,12 +77,12 @@ def __init__(self, config, logger, name, attrslist, key):
         for o in re.split('\W+', self.get_param('objectclasses')):
             self.objectclasses.append(self._str(o))
         self.group_attrs = {}
-        self.group_attrs_keys = Set([])
+        self.group_attrs_keys = set([])
         for param in config:
             name, sep, group = param.partition('.')
             if name == 'group_attr':
                 self.group_attrs[group] = self.get_param(param)
-                self.group_attrs_keys |= Set(
+                self.group_attrs_keys |= set(
                     self._extract_format_keys(self.get_param(param))
                 )
 
@@ -393,7 +396,7 @@ def del_user(self, username):
         ldap_client.unbind_s()
 
     def set_attrs(self, username, attrs):
-        """ Set user attributes"""
+        """ set user attributes"""
         ldap_client = self._bind()
         tmp = self._get_user(self._str(username), ALL_ATTRS)
         if tmp is None:
diff --git a/ldapcherry/roles.py b/ldapcherry/roles.py
index 0bd1dd6..1c3bd91 100644
--- a/ldapcherry/roles.py
+++ b/ldapcherry/roles.py
@@ -9,7 +9,9 @@
 import sys
 import copy
 
-from sets import Set
+if sys.version < '3':
+    from sets import Set as set
+
 from ldapcherry.pyyamlwrapper import loadNoDump
 from ldapcherry.pyyamlwrapper import DumplicatedKey
 from ldapcherry.exceptions import *
@@ -27,7 +29,7 @@ class Roles:
 
     def __init__(self, role_file):
         self.role_file = role_file
-        self.backends = Set([])
+        self.backends = set([])
         try:
             stream = open(role_file, 'r')
         except:
@@ -51,7 +53,7 @@ def _merge_groups(self, backends_list):
         for backends in backends_list:
             for b in backends:
                 if b not in ret:
-                    ret[b] = Set([])
+                    ret[b] = set([])
                 for group in backends[b]:
                     ret[b].add(group)
         for b in ret:
@@ -134,8 +136,8 @@ def _nest(self):
 
             if roleid not in self.graph:
                 self.graph[roleid] = {
-                    'parent_roles': Set([]),
-                    'sub_roles': Set([])
+                    'parent_roles': set([]),
+                    'sub_roles': set([])
                     }
 
         # Create the nested groups
@@ -147,7 +149,7 @@ def _nest(self):
                     if b not in self.group2roles:
                         self.group2roles[b] = {}
                     if g not in self.group2roles[b]:
-                        self.group2roles[b][g] = Set([])
+                        self.group2roles[b][g] = set([])
                     self.group2roles[b][g].add(roleid)
 
             parent_roles[roleid] = []
@@ -223,7 +225,7 @@ def _check_member(
         # add groups of the role to usedgroups
         for b in self.roles[role]['backends_groups']:
             if b not in usedgroups:
-                usedgroups[b] = Set([])
+                usedgroups[b] = set([])
             for g in self.roles[role]['backends_groups'][b]:
                 usedgroups[b].add(g)
 
@@ -254,11 +256,11 @@ def get_groups_to_remove(self, current_roles, roles_to_remove):
         """get groups to remove from list of
         roles to remove and current roles
         """
-        current_roles = Set(current_roles)
+        current_roles = set(current_roles)
 
         ret = {}
-        roles_to_remove = Set(roles_to_remove)
-        tmp = Set([])
+        roles_to_remove = set(roles_to_remove)
+        tmp = set([])
         # get sub roles of the role to remove that the user belongs to
         # if we remove a role, there is no reason to keep the sub roles
         for r in roles_to_remove:
@@ -267,7 +269,7 @@ def get_groups_to_remove(self, current_roles, roles_to_remove):
                     tmp.add(sr)
 
         roles_to_remove = roles_to_remove.union(tmp)
-        roles = current_roles.difference(Set(roles_to_remove))
+        roles = current_roles.difference(set(roles_to_remove))
         groups_roles = self._get_groups(roles)
         groups_roles_to_remove = self._get_groups(roles_to_remove)
 
@@ -284,12 +286,12 @@ def _get_groups(self, roles):
             for b in self.flatten[r]['backends_groups']:
                 groups = self.flatten[r]['backends_groups'][b]
                 if b not in ret:
-                    ret[b] = Set(groups)
-                ret[b] = ret[b].union(Set(groups))
+                    ret[b] = set(groups)
+                ret[b] = ret[b].union(set(groups))
         return ret
 
     def _get_subroles(self, role):
-        ret = Set([])
+        ret = set([])
         for sr in self.graph[role]['sub_roles']:
             tmp = self._get_subroles(sr)
             tmp.add(sr)
@@ -298,10 +300,10 @@ def _get_subroles(self, role):
 
     def get_roles(self, groups):
         """get list of roles and list of standalone groups"""
-        roles = Set([])
-        parentroles = Set([])
-        notroles = Set([])
-        tmp = Set([])
+        roles = set([])
+        parentroles = set([])
+        notroles = set([])
+        tmp = set([])
         usedgroups = {}
         unusedgroups = {}
         ret = {}
@@ -316,7 +318,7 @@ def get_roles(self, groups):
             for g in groups[b]:
                 if b not in usedgroups or g not in usedgroups[b]:
                     if b not in unusedgroups:
-                        unusedgroups[b] = Set([])
+                        unusedgroups[b] = set([])
                     unusedgroups[b].add(g)
 
         ret['roles'] = roles

From 3beedc8d4d449f2e91374dd5e9388a12d57696bc Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 6 Feb 2019 23:03:55 +0100
Subject: [PATCH 10/82] add an ignore on the local dev conf file

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index cecc2c1..4c7c380 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,3 +54,4 @@ coverage.xml
 # Sphinx documentation
 docs/_build/
 
+ldapcherry-dev.ini

From ccc252965d3ad9fe31abb3a08ef50451fe02ccef Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 6 Feb 2019 23:04:23 +0100
Subject: [PATCH 11/82] fix another __import__

---
 ldapcherry/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index 33b2063..2a6a15a 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -243,7 +243,7 @@ def _init_auth(self, config):
         elif self.auth_mode == 'custom':
             # load custom auth module
             auth_module = self._get_param('auth', 'auth.module', config)
-            auth = __import__(auth_module, globals(), locals(), ['Auth'], -1)
+            auth = __import__(auth_module, globals(), locals(), ['Auth'], 0)
             self.auth = auth.Auth(config['auth'], cherrypy.log)
         else:
             raise WrongParamValue(

From be598b0129f1d364430273f5a50e831b487166b0 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 6 Feb 2019 23:55:03 +0100
Subject: [PATCH 12/82] slightly cleaner testenv deploy script

---
 tests/test_env/deploy.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/test_env/deploy.sh b/tests/test_env/deploy.sh
index a5136d4..b7f8aef 100755
--- a/tests/test_env/deploy.sh
+++ b/tests/test_env/deploy.sh
@@ -4,7 +4,13 @@ DEBIAN_FRONTEND=noninteractive apt-get install ldap-utils slapd -o Dpkg::Options
 DEBIAN_FRONTEND=noninteractive apt-get install samba -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 DEBIAN_FRONTEND=noninteractive apt-get install winbind -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 
-rsync -a `dirname $0`/ /
+[ -e '/etc/default/slapd' ] && mv /etc/default/slapd.ori-prelc
+cp -r `dirname $0`/etc/default/slapd /etc/default/slapd
+[ -e '/etc/ldap' ] && mv /etc/ldap.ori-prelc
+cp -r `dirname $0`/etc/ldap /etc/ldap
+[ -e '/etc/ldapcherry' ] && mv /etc/ldapcherry.ori-prelc
+cp -r `dirname $0`/etc/ldapcherry /etc/ldapcherry
+
 cd `dirname $0`/../../
 sudo sed -i "s%template_dir.*%template_dir = '`pwd`/resources/templates/'%" /etc/ldapcherry/ldapcherry.ini
 sudo sed -i "s%tools.staticdir.dir.*%tools.staticdir.dir = '`pwd`/resources/static/'%" /etc/ldapcherry/ldapcherry.ini
@@ -13,7 +19,7 @@ chown -R openldap:openldap /etc/ldap/
 rm /etc/ldap/slapd.d/cn\=config/*mdb*
 /etc/init.d/slapd restart
 ldapadd -c -H ldap://localhost:390  -x -D "cn=admin,dc=example,dc=org" -f /etc/ldap/content.ldif -w password
-if grep -q '127.0.0.1' /etc/hosts
+if grep -q '127.0.0.1' /etc/hosts && ! grep -q 'ldap.ldapcherry.org' /etc/hosts
 then
     sed -i "s/\(127.0.0.1.*\)/\1 ldap.ldapcherry.org ad.ldapcherry.org ldap.dnscherry.org/" /etc/hosts
 else

From 3d6e24eb7369478bd8834357c0c39059e5c84859 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 20:16:39 +0100
Subject: [PATCH 13/82] pep8

---
 ldapcherry/__init__.py            |  8 ++++----
 ldapcherry/attributes.py          |  4 ++--
 ldapcherry/backend/backendDemo.py | 24 ++++++++++++++----------
 ldapcherry/backend/backendLdap.py |  7 +++----
 ldapcherry/roles.py               |  8 ++++----
 5 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index 2a6a15a..642ee2f 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -173,7 +173,7 @@ def _init_backends(self, config):
             try:
                 self.backends_display_names[backend] = \
                     self.backends_params[backend]['display_name']
-            except:
+            except Exception as e:
                 self.backends_display_names[backend] = backend
                 self.backends_params[backend]['display_name'] = backend
             params = self.backends_params[backend]
@@ -225,7 +225,7 @@ def _init_ppolicy(self, config):
         )
         try:
             pp = __import__(module, globals(), locals(), ['PPolicy'], 0)
-        except:
+        except Exception as e:
             raise BackendModuleLoadingFail(module)
         if 'ppolicy' in config:
             ppcfg = config['ppolicy']
@@ -1091,7 +1091,7 @@ def delete(self, user):
         is_admin = self._check_admin()
         try:
             referer = cherrypy.request.headers['Referer']
-        except:
+        except Exception as e:
             referer = '/'
         self._deleteuser(user)
         self._add_notification('User Deleted')
@@ -1110,7 +1110,7 @@ def modify(self, user=None, **params):
             self._add_notification("User modified")
             try:
                 referer = cherrypy.request.headers['Referer']
-            except:
+            except Exception as e:
                 referer = '/'
             raise cherrypy.HTTPRedirect(referer)
 
diff --git a/ldapcherry/attributes.py b/ldapcherry/attributes.py
index 75c863e..989bf14 100644
--- a/ldapcherry/attributes.py
+++ b/ldapcherry/attributes.py
@@ -33,7 +33,7 @@ def __init__(self, attributes_file):
         self.key = None
         try:
             stream = open(attributes_file, 'r')
-        except:
+        except Exception as e:
             raise MissingAttributesFile(attributes_file)
         try:
             self.attributes = loadNoDump(stream)
@@ -71,7 +71,7 @@ def __init__(self, attributes_file):
             raise MissingUserKey()
 
     def _is_email(self, email):
-        pattern = '[\.\w]{1,}[@]\w+[.]\w+'
+        pattern = r'[\.\w]{1,}[@]\w+[.]\w+'
         if re.match(pattern, email):
             return True
         else:
diff --git a/ldapcherry/backend/backendDemo.py b/ldapcherry/backend/backendDemo.py
index 0377bad..6e0e0b4 100644
--- a/ldapcherry/backend/backendDemo.py
+++ b/ldapcherry/backend/backendDemo.py
@@ -9,14 +9,13 @@
 
 
 import sys
-if sys.version < '3':
-    from sets import Set as set
-
 import ldapcherry.backend
 from ldapcherry.exceptions import UserDoesntExist, \
     GroupDoesntExist, MissingParameter, \
     UserAlreadyExists
 import re
+if sys.version < '3':
+    from sets import Set as set
 
 
 class Backend(ldapcherry.backend.Backend):
@@ -41,13 +40,17 @@ def __init__(self, config, logger, name, attrslist, key):
         self.backend_name = name
         admin_user = self.get_param('admin.user', 'admin')
         admin_password = self.get_param('admin.password', 'admin')
-        admin_groups = set(self._basic_splitter(self.get_param('admin.groups')))
+        admin_groups = set(
+            self._basic_splitter(self.get_param('admin.groups'))
+            )
         basic_user = self.get_param('basic.user', 'user')
         basic_password = self.get_param('basic.password', 'user')
-        basic_groups = set(self._basic_splitter(self.get_param('basic.groups')))
+        basic_groups = set(
+            self._basic_splitter(self.get_param('basic.groups'))
+            )
         pwd_attr = self.get_param('pwd_attr')
         self.search_attrs = set(
-            re.split('\W+', self.get_param('search_attributes')),
+            re.split(r'\W+', self.get_param('search_attributes')),
             )
         self.pwd_attr = pwd_attr
         self.admin_user = admin_user
@@ -66,7 +69,8 @@ def __init__(self, config, logger, name, attrslist, key):
 
     @staticmethod
     def _basic_splitter(in_str):
-        return [re.sub(r'(?<!\\)\\', '', x) for x in re.split(r'(?<!\\),\W*', in_str)]
+        return [re.sub(r'(?<!\\)\\', '', x)
+                for x in re.split(r'(?<!\\),\W*', in_str)]
 
     def _check_fix_users(self, username):
         if self.admin_user == username or self.basic_user == username:
@@ -111,7 +115,7 @@ def del_user(self, username):
         self._check_fix_users(username)
         try:
             del self.users[username]
-        except:
+        except Exception as e:
             raise UserDoesntExist(username, self.backend_name)
 
     def set_attrs(self, username, attrs):
@@ -184,7 +188,7 @@ def get_user(self, username):
         """
         try:
             return self.users[username]
-        except:
+        except Exception as e:
             raise UserDoesntExist(username, self.backend_name)
 
     def get_groups(self, username):
@@ -196,5 +200,5 @@ def get_groups(self, username):
         """
         try:
             return self.users[username]['groups']
-        except:
+        except Exception as e:
             raise UserDoesntExist(username, self.backend_name)
diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index 5ac16e1..9bf177c 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -12,14 +12,13 @@
 import logging
 import ldapcherry.backend
 import sys
-if sys.version < '3':
-    from sets import Set as set
-
 from ldapcherry.exceptions import UserDoesntExist, \
     GroupDoesntExist, \
     UserAlreadyExists
 import os
 import re
+if sys.version < '3':
+    from sets import Set as set
 
 
 class CaFileDontExist(Exception):
@@ -74,7 +73,7 @@ def __init__(self, config, logger, name, attrslist, key):
         self.key = key
         # objectclasses parameter is a coma separated list in configuration
         # split it to get a real list, and convert it to bytes
-        for o in re.split('\W+', self.get_param('objectclasses')):
+        for o in re.split(r'\W+', self.get_param('objectclasses')):
             self.objectclasses.append(self._str(o))
         self.group_attrs = {}
         self.group_attrs_keys = set([])
diff --git a/ldapcherry/roles.py b/ldapcherry/roles.py
index 1c3bd91..9fc6a92 100644
--- a/ldapcherry/roles.py
+++ b/ldapcherry/roles.py
@@ -9,14 +9,14 @@
 import sys
 import copy
 
-if sys.version < '3':
-    from sets import Set as set
-
 from ldapcherry.pyyamlwrapper import loadNoDump
 from ldapcherry.pyyamlwrapper import DumplicatedKey
 from ldapcherry.exceptions import *
 import yaml
 
+if sys.version < '3':
+    from sets import Set as set
+
 
 class CustomDumper(yaml.SafeDumper):
     "A custom YAML dumper that never emits aliases"
@@ -32,7 +32,7 @@ def __init__(self, role_file):
         self.backends = set([])
         try:
             stream = open(role_file, 'r')
-        except:
+        except Exception as e:
             raise MissingRolesFile(role_file)
         try:
             self.roles_raw = loadNoDump(stream)

From c81429a870d84bfeda4975ec5da3db43e06ba006 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 20:34:49 +0100
Subject: [PATCH 14/82] few tweaks for python3 support

* switch from script to entry_points in setup.py
* move the cli script in ldapcherry (to be used as a module)
* put the __main__ code in a dedicated function constituting the entry
point
* add a few python3 environments in travis file
---
 .travis.yml                              | 19 ++++++++++++++-----
 scripts/ldapcherryd => ldapcherry/cli.py |  5 ++++-
 setup.py                                 | 12 +++++++++---
 3 files changed, 27 insertions(+), 9 deletions(-)
 rename scripts/ldapcherryd => ldapcherry/cli.py (99%)

diff --git a/.travis.yml b/.travis.yml
index fd0f42a..2881fe1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,12 +8,9 @@ language: python
 before_install:
   - '[ "$TEST_PEP8" == "1" ] || sudo ./tests/test_env/deploy.sh'
 
-python:
-  - "2.7"
-
 install: 
   - pip install -e .
-  - "if [[ $TEST_PEP8 == '1' ]]; then pip install pep8; fi"
+  - "if [[ $TEST_PEP8 == '1' ]]; then pip install pycodestyle; fi"
   - pip install passlib
   - pip install coveralls
 
@@ -21,11 +18,23 @@ install:
 #
 #script: 
 #  - coverage run --source=ldapcherry setup.py test
-script: "if [[ $TEST_PEP8 == '1' ]]; then pep8 --repeat --show-source --exclude=.venv,.tox,dist,docs,build,*.egg,tests,misc,setup.py . scripts/ldapcherryd; else coverage run --source=ldapcherry setup.py test; fi"
+script: "if [[ $TEST_PEP8 == '1' ]]; then pycodestyle --repeat --show-source --exclude=.venv,.tox,dist,docs,build,*.egg,tests,misc,setup.py . scripts/ldapcherryd; else coverage run --source=ldapcherry setup.py test; fi"
 matrix:
   include:
     - python: "2.7"
       env: TEST_PEP8=1
+    - python: "2.7"
+      env: TEST_PEP8=0
+    - python: "3.5"
+      env: TEST_PEP8=0
+    - python: "3.6"
+      env: TEST_PEP8=0
+    - python: "3.4"
+      env: TEST_PEP8=0
+    - python: "3.7"
+      env: TEST_PEP8=0
+
+
 after_success:
   - coveralls
 after_failure:
diff --git a/scripts/ldapcherryd b/ldapcherry/cli.py
similarity index 99%
rename from scripts/ldapcherryd
rename to ldapcherry/cli.py
index 1547caf..0b11a05 100755
--- a/scripts/ldapcherryd
+++ b/ldapcherry/cli.py
@@ -102,7 +102,7 @@ def new_as_dict(self, raw=True, vars=None):
         engine.block()
 
 
-if __name__ == '__main__':
+def main():
     from optparse import OptionParser
 
     p = OptionParser()
@@ -142,3 +142,6 @@ def new_as_dict(self, raw=True, vars=None):
     start(options.config, options.daemonize,
           options.environment, options.fastcgi, options.scgi,
           options.pidfile, options.cgi, options.debug)
+
+if __name__ == '__main__':
+    main()
diff --git a/setup.py b/setup.py
index d47a27f..ace0cdc 100755
--- a/setup.py
+++ b/setup.py
@@ -29,8 +29,12 @@
         'Mako'
         ],
 elif sys.version_info[0] == 3:
-    print('unsupported version')
-    exit(1)
+    install_requires = [
+        'CherryPy >= 3.0.0',
+        'python-ldap',
+        'PyYAML',
+        'Mako'
+        ],
 else:
     print('unsupported version')
     exit(1)
@@ -120,7 +124,9 @@ def get_list_files(basedir, targetdir):
         'ldapcherry.ppolicy'
         ],
     data_files=resources_files,
-    scripts=['scripts/ldapcherryd'],
+    entry_points = {
+        'console_scripts': ['ldapcherryd = ldapcherry.cli:main']
+    },
     url='https://github.com/kakwa/ldapcherry',
     license=license,
     description=small_description,

From 5bdcc5522ae7ff751955eb63fb403eb308459d4c Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 20:40:22 +0100
Subject: [PATCH 15/82] switch to xenial in travis configuration

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 2881fe1..c7484e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 sudo: required
-dist: trusty
+dist: xenial
 language: python
 
 #env:

From 9f6af580cda661a4c0db0b900cf5e92e9606d23e Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 20:41:22 +0100
Subject: [PATCH 16/82] remove env that doesn't exist

---
 .travis.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c7484e5..74425c9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,9 +31,6 @@ matrix:
       env: TEST_PEP8=0
     - python: "3.4"
       env: TEST_PEP8=0
-    - python: "3.7"
-      env: TEST_PEP8=0
-
 
 after_success:
   - coveralls

From 86fb6c1dd2dba5c2ef43006be80ea4e19170a1b2 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 20:44:19 +0100
Subject: [PATCH 17/82] adding an update as the first step of the deploy script

---
 tests/test_env/deploy.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_env/deploy.sh b/tests/test_env/deploy.sh
index b7f8aef..423f078 100755
--- a/tests/test_env/deploy.sh
+++ b/tests/test_env/deploy.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+apt update
+
 DEBIAN_FRONTEND=noninteractive apt-get install ldap-utils slapd -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 DEBIAN_FRONTEND=noninteractive apt-get install samba -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 DEBIAN_FRONTEND=noninteractive apt-get install winbind -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y

From c3feafdb2cd9791c36fb13be8140fa82cb15ce1e Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 20:48:06 +0100
Subject: [PATCH 18/82] pep8

---
 ldapcherry/cli.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ldapcherry/cli.py b/ldapcherry/cli.py
index 0b11a05..862111d 100755
--- a/ldapcherry/cli.py
+++ b/ldapcherry/cli.py
@@ -143,5 +143,6 @@ def main():
           options.environment, options.fastcgi, options.scgi,
           options.pidfile, options.cgi, options.debug)
 
+
 if __name__ == '__main__':
     main()

From 2a2864a3065eaddf15a1a6a23e4b3b8a670db11c Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 20:55:50 +0100
Subject: [PATCH 19/82] porting the tests over to python3

---
 tests/disable.py          |  4 ++--
 tests/test_Attributes.py  |  6 ++++--
 tests/test_BackendAD.py   | 10 ++++++----
 tests/test_BackendDemo.py |  8 +++++---
 tests/test_BackendLdap.py |  4 ++--
 tests/test_LdapCherry.py  |  5 +++--
 tests/test_Roles.py       | 11 ++++++-----
 7 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/tests/disable.py b/tests/disable.py
index e42f3dc..19d48f1 100644
--- a/tests/disable.py
+++ b/tests/disable.py
@@ -1,7 +1,7 @@
 import os
 def travis_disabled(f):
     def _decorator(f):
-            print 'test has been disabled on travis'
+            print('test has been disabled on travis')
     if 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'yes':
         return _decorator
     else:
@@ -9,7 +9,7 @@ def _decorator(f):
 
 def slow_disabled(f):
     def _decorator(f):
-            print 'test has been disabled by env var LCNOSLOW'
+            print('test has been disabled by env var LCNOSLOW')
     if 'LCNOSLOW' in os.environ and os.environ['LCNOSLOW'] == 'yes':
         return _decorator
     else:
diff --git a/tests/test_Attributes.py b/tests/test_Attributes.py
index 7220946..f7f42b2 100644
--- a/tests/test_Attributes.py
+++ b/tests/test_Attributes.py
@@ -6,10 +6,12 @@
 
 import pytest
 import sys
-from sets import Set
 from ldapcherry.attributes import Attributes
 from ldapcherry.exceptions import *
 from ldapcherry.pyyamlwrapper import DumplicatedKey, RelationError
+if sys.version < '3':
+    from sets import Set as set
+
 
 class TestError(object):
 
@@ -27,7 +29,7 @@ def testGetSelfAttributes(self):
     def testGetSelfAttributes(self):
         inv = Attributes('./tests/cfg/attributes.yml')
         ret = inv.get_backends()
-        expected = Set(['ldap', 'ad'])
+        expected = set(['ldap', 'ad'])
         assert ret == expected
 
     def testGetSearchAttributes(self):
diff --git a/tests/test_BackendAD.py b/tests/test_BackendAD.py
index 0d7b870..416dde6 100644
--- a/tests/test_BackendAD.py
+++ b/tests/test_BackendAD.py
@@ -6,12 +6,14 @@
 
 import pytest
 import sys
-from sets import Set
 from ldapcherry.backend.backendAD import Backend
 from ldapcherry.exceptions import *
 from disable import travis_disabled
 import cherrypy
 import logging
+if sys.version < '3':
+    from sets import Set as set
+
 
 cfg = {
     'display_name': u'test☭',
@@ -74,14 +76,14 @@ def testAuthFailure(self):
         assert res == False
 
     @travis_disabled
-    def testSetPassword(self):
+    def testsetPassword(self):
         inv = Backend(cfg, cherrypy.log, u'test☭', attr, 'sAMAccountName')
         try:
             inv.add_user(default_user.copy())
             inv.add_to_groups(u'☭default_user', default_groups)
         except:
             pass
-	inv.set_attrs(u'☭default_user', {'unicodePwd': u'test☭P66642$'})
+        inv.set_attrs(u'☭default_user', {'unicodePwd': u'test☭P66642$'})
         ret = inv.auth(u'☭default_user', u'test☭P66642$')
         inv.del_user(u'☭default_user')
         assert ret == True
@@ -126,7 +128,7 @@ def testSearchUser(self):
         expected = [u'☭default_user', u'☭default_user2']
         inv.del_user(u'☭default_user')
         inv.del_user(u'☭default_user2')
-        assert Set(ret.keys()) == Set(expected)
+        assert set(ret.keys()) == set(expected)
 
     @travis_disabled
     def testAddUser(self):
diff --git a/tests/test_BackendDemo.py b/tests/test_BackendDemo.py
index e7d94a8..7c2ac4e 100644
--- a/tests/test_BackendDemo.py
+++ b/tests/test_BackendDemo.py
@@ -6,12 +6,14 @@
 
 import pytest
 import sys
-from sets import Set
 from ldapcherry.backend.backendDemo import Backend
 from ldapcherry.exceptions import *
 from disable import travis_disabled
 import cherrypy
 import logging
+if sys.version < '3':
+    from sets import Set as set
+
 
 cfg = {
     'display_name': 'test',
@@ -82,7 +84,7 @@ def testGetGroups(self):
         inv.add_user(default_user)
         inv.add_to_groups('default_user', default_groups)
         ret = inv.get_groups('default_user')
-        expected = Set(default_groups)
+        expected = set(default_groups)
         assert ret == expected
 
     def testSearchUser(self):
@@ -91,7 +93,7 @@ def testSearchUser(self):
         inv.add_user(default_user2)
         ret = inv.search('default')
         expected = ['default_user', 'default_user2']
-        assert Set(ret.keys()) == Set(expected)
+        assert set(ret.keys()) == set(expected)
 
     def testAddUser(self):
         try:
diff --git a/tests/test_BackendLdap.py b/tests/test_BackendLdap.py
index f969324..b9e874f 100644
--- a/tests/test_BackendLdap.py
+++ b/tests/test_BackendLdap.py
@@ -6,13 +6,14 @@
 
 import pytest
 import sys
-from sets import Set
 from ldapcherry.backend.backendLdap import Backend, CaFileDontExist
 from ldapcherry.exceptions import *
 from disable import travis_disabled
 import cherrypy
 import logging
 import ldap
+if sys.version < '3':
+    from sets import Set as set
 
 cfg = {
 'module'             : 'ldapcherry.backend.ldap',
@@ -148,7 +149,6 @@ def testAddDeleteGroups(self):
         ]
         inv.add_to_groups(u'jwatsoné', groups)
         ret = inv.get_groups(u'jwatsoné')
-        print ret
         inv.del_from_groups(u'jwatsoné', ['cn=hrpeople,ou=Groups,dc=example,dc=org'])
         inv.del_from_groups(u'jwatsoné', ['cn=hrpeople,ou=Groups,dc=example,dc=org'])
         assert ret == ['cn=itpeople,ou=Groups,dc=example,dc=org', 'cn=hrpeople,ou=Groups,dc=example,dc=org']
diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 9c5667b..54611f2 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -9,7 +9,6 @@
 from tempfile import NamedTemporaryFile as tempfile
 import re
 
-from sets import Set
 from ldapcherry import LdapCherry
 from ldapcherry.exceptions import *
 from ldapcherry.pyyamlwrapper import DumplicatedKey, RelationError
@@ -21,6 +20,8 @@
 from ldapcherry.lclogging import *
 from disable import *
 import json
+if sys.version < '3':
+    from sets import Set as set
 
 cherrypy.session = {}
 
@@ -337,7 +338,7 @@ def testModifyUserOneBackend(self):
         app = LdapCherry()
         loadconf('./tests/cfg/ldapcherry_adldap.cfg', app)
         inv = ldapcherry.backend.backendAD.Backend(adcfg, cherrypy.log, u'test☭', adattr, 'sAMAccountName')
-	try:
+        try:
             app._deleteuser(u'☭default_user')
         except:
             pass
diff --git a/tests/test_Roles.py b/tests/test_Roles.py
index 5cebc08..72bddc1 100644
--- a/tests/test_Roles.py
+++ b/tests/test_Roles.py
@@ -6,16 +6,17 @@
 
 import pytest
 import sys
-from sets import Set
 from ldapcherry.roles import Roles
 from ldapcherry.exceptions import DumplicateRoleKey, MissingKey, DumplicateRoleContent, MissingRolesFile, MissingRole
 from ldapcherry.pyyamlwrapper import DumplicatedKey, RelationError
+if sys.version < '3':
+    from sets import Set as set
 
 class TestError(object):
 
     def testNominal(self):
         inv = Roles('./tests/cfg/roles.yml')
-        print inv.roles
+        print(inv.roles)
         return True
 
     def testMissingDisplayName(self):
@@ -64,7 +65,7 @@ def testGroupsRemove(self):
                 ['admin-lv2', 'admin-lv3', 'users'],
                 ['admin-lv2']
         )
-        expected = {'ad': Set(['Administrators', 'Domain Controllers']), 'ldap': Set(['cn=nagios admins,ou=group,dc=example,dc=com', 'cn=puppet admins,ou=group,dc=example,dc=com', 'cn=dns admins,ou=group,dc=example,dc=com'])}
+        expected = {'ad': set(['Administrators', 'Domain Controllers']), 'ldap': set(['cn=nagios admins,ou=group,dc=example,dc=com', 'cn=puppet admins,ou=group,dc=example,dc=com', 'cn=dns admins,ou=group,dc=example,dc=com'])}
         assert groups == expected
 
     def testGetGroup(self):
@@ -114,7 +115,7 @@ def testGetAllRoles(self):
     def testGetAllRoles(self):
         inv = Roles('./tests/cfg/roles.yml')
         res = inv.get_backends()
-        expected = Set(['ad', 'ldap'])
+        expected = set(['ad', 'ldap'])
         assert res == expected
 
     def testDumpNested(self):
@@ -147,5 +148,5 @@ def testGetRole(self):
             ],
         'toto': ['not a group'],
         }
-        expected = {'unusedgroups': {'toto': Set(['not a group']), 'ad': Set(['Domain Users 2'])}, 'roles': Set(['developpers', 'admin-lv2', 'users'])}
+        expected = {'unusedgroups': {'toto': set(['not a group']), 'ad': set(['Domain Users 2'])}, 'roles': set(['developpers', 'admin-lv2', 'users'])}
         assert inv.get_roles(groups) == expected

From 8bd4afb23550bbc7813b1e814b345c461784f146 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 21:07:36 +0100
Subject: [PATCH 20/82] remove scripts from pycodestyle

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 74425c9..5ea306d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,7 +18,7 @@ install:
 #
 #script: 
 #  - coverage run --source=ldapcherry setup.py test
-script: "if [[ $TEST_PEP8 == '1' ]]; then pycodestyle --repeat --show-source --exclude=.venv,.tox,dist,docs,build,*.egg,tests,misc,setup.py . scripts/ldapcherryd; else coverage run --source=ldapcherry setup.py test; fi"
+script: "if [[ $TEST_PEP8 == '1' ]]; then pycodestyle --repeat --show-source --exclude=.venv,.tox,dist,docs,build,*.egg,tests,misc,setup.py .; else coverage run --source=ldapcherry setup.py test; fi"
 matrix:
   include:
     - python: "2.7"

From 70140f966ac4c3d8c2e4f3b038d72dc8a8feddc3 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 21:09:42 +0100
Subject: [PATCH 21/82] pep 8

---
 ldapcherry/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ldapcherry/cli.py b/ldapcherry/cli.py
index 862111d..626ef2e 100755
--- a/ldapcherry/cli.py
+++ b/ldapcherry/cli.py
@@ -95,7 +95,7 @@ def new_as_dict(self, raw=True, vars=None):
     # Always start the engine; this will start all other services
     try:
         engine.start()
-    except:
+    except Exception as e:
         # Assume the error has been logged already via bus.log.
         sys.exit(1)
     else:

From 13bfbdcbbcf57dc9c928716e28dd3db9ac1af635 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 22:07:28 +0100
Subject: [PATCH 22/82] add requirements files for simulating RHEL 7 and Debian
 9

---
 .travis.yml              | 32 +++++++++++++++++++++++++-------
 requirements-el7.txt     |  4 ++++
 requirements-stretch.txt |  4 ++++
 tests/test_env/deploy.sh |  6 ++++--
 4 files changed, 37 insertions(+), 9 deletions(-)
 create mode 100644 requirements-el7.txt
 create mode 100644 requirements-stretch.txt

diff --git a/.travis.yml b/.travis.yml
index 5ea306d..3054b57 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,7 @@ before_install:
   - '[ "$TEST_PEP8" == "1" ] || sudo ./tests/test_env/deploy.sh'
 
 install: 
-  - pip install -e .
+  - pip install -e -r $REQ_FILE
   - "if [[ $TEST_PEP8 == '1' ]]; then pip install pycodestyle; fi"
   - pip install passlib
   - pip install coveralls
@@ -22,15 +22,33 @@ script: "if [[ $TEST_PEP8 == '1' ]]; then pycodestyle --repeat --show-source --e
 matrix:
   include:
     - python: "2.7"
-      env: TEST_PEP8=1
+      env:
+        TEST_PEP8=1
+        REQ_FILE=requirements.txt
     - python: "2.7"
-      env: TEST_PEP8=0
+      env:
+        TEST_PEP8=0
+        REQ_FILE=requirements-el7.txt
+    - python: "2.7"
+      env:
+        TEST_PEP8=0
+        REQ_FILE=requirements-stretch.txt
+    - python: "2.7"
+      env:
+        TEST_PEP8=0
+        REQ_FILE=requirements.txt
+    - python: "3.4"
+      env:
+        TEST_PEP8=0
+        REQ_FILE=requirements-el7.txt
     - python: "3.5"
-      env: TEST_PEP8=0
+      env:
+        TEST_PEP8=0
+        REQ_FILE=requirements-stretch.txt
     - python: "3.6"
-      env: TEST_PEP8=0
-    - python: "3.4"
-      env: TEST_PEP8=0
+      env:
+        TEST_PEP8=0
+        REQ_FILE=requirements.txt
 
 after_success:
   - coveralls
diff --git a/requirements-el7.txt b/requirements-el7.txt
new file mode 100644
index 0000000..109ab48
--- /dev/null
+++ b/requirements-el7.txt
@@ -0,0 +1,4 @@
+CherryPy>=3.0.0
+PyYAML
+Mako
+python-ldap==2.4.15
diff --git a/requirements-stretch.txt b/requirements-stretch.txt
new file mode 100644
index 0000000..dcdcf36
--- /dev/null
+++ b/requirements-stretch.txt
@@ -0,0 +1,4 @@
+CherryPy>=3.0.0
+PyYAML
+Mako
+python-ldap==2.4.28
diff --git a/tests/test_env/deploy.sh b/tests/test_env/deploy.sh
index 423f078..19e0d83 100755
--- a/tests/test_env/deploy.sh
+++ b/tests/test_env/deploy.sh
@@ -48,8 +48,9 @@ role=dc
 sambacmd=samba-tool
 adpass=qwertyP455
 
+systemctl unmask samba-ad-dc
+
 hostname ad.ldapcherry.org 
-/etc/init.d/dnsmasq stop
 pkill -9 dnsmasq
 
 kill -9 `cat /var/run/samba/smbd.pid` 
@@ -75,10 +76,11 @@ mv /var/lib/samba/private/krb5.conf /etc/krb5.conf
 
 sleep 5
 
+systemctl start samba-ad-dc
 /etc/init.d/samba-ad-dc start
 
 cat /var/log/samba/*
 
 sleep 5
 
-netstat -apn | grep samba
+ss -apn | grep samba

From ab9cd664ec87d8917d14dde52409ac129eb12c0c Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 22:12:49 +0100
Subject: [PATCH 23/82] fix pip install

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 3054b57..35db8f9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,7 @@ before_install:
   - '[ "$TEST_PEP8" == "1" ] || sudo ./tests/test_env/deploy.sh'
 
 install: 
-  - pip install -e -r $REQ_FILE
+  - "pip install -e . -r $REQ_FILE"
   - "if [[ $TEST_PEP8 == '1' ]]; then pip install pycodestyle; fi"
   - pip install passlib
   - pip install coveralls

From fc98b1bd7070ce83e02df67a127a64fe0d6741ac Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 22:34:47 +0100
Subject: [PATCH 24/82] fixing the test env deploy script + small fix in unit
 tests

---
 tests/test_BackendLdap.py |  3 ++-
 tests/test_env/deploy.sh  | 12 ++++--------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/tests/test_BackendLdap.py b/tests/test_BackendLdap.py
index b9e874f..eefabef 100644
--- a/tests/test_BackendLdap.py
+++ b/tests/test_BackendLdap.py
@@ -105,7 +105,8 @@ def testConnectSSLWrongCA(self):
         try:
             ldapc.simple_bind_s(inv.binddn, inv.bindpassword)
         except ldap.SERVER_DOWN as e:
-            assert e[0]['info'] == 'TLS: hostname does not match CN in peer certificate'
+            assert e[0]['info'] == 'TLS: hostname does not match CN in peer certificate' or \
+                    e[0]['info'] == '(unknown error code)'
         else:
             raise AssertionError("expected an exception")
 
diff --git a/tests/test_env/deploy.sh b/tests/test_env/deploy.sh
index 19e0d83..353df4b 100755
--- a/tests/test_env/deploy.sh
+++ b/tests/test_env/deploy.sh
@@ -6,11 +6,11 @@ DEBIAN_FRONTEND=noninteractive apt-get install ldap-utils slapd -o Dpkg::Options
 DEBIAN_FRONTEND=noninteractive apt-get install samba -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 DEBIAN_FRONTEND=noninteractive apt-get install winbind -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 
-[ -e '/etc/default/slapd' ] && mv /etc/default/slapd.ori-prelc
+[ -e '/etc/default/slapd' ] && rm -rf /etc/default/slapd
 cp -r `dirname $0`/etc/default/slapd /etc/default/slapd
-[ -e '/etc/ldap' ] && mv /etc/ldap.ori-prelc
+[ -e '/etc/ldap' ] && rm -rf /etc/ldap
 cp -r `dirname $0`/etc/ldap /etc/ldap
-[ -e '/etc/ldapcherry' ] && mv /etc/ldapcherry.ori-prelc
+[ -e '/etc/ldapcherry' ] && rm -rf /etc/ldapcherry
 cp -r `dirname $0`/etc/ldapcherry /etc/ldapcherry
 
 cd `dirname $0`/../../
@@ -32,11 +32,6 @@ cat /etc/hosts
 
 df -h
 
-/etc/init.d/samba stop 
-/etc/init.d/smbd stop
-/etc/init.d/nmbd stop 
-/etc/init.d/samba-ad-dc stop
-
 find /var/log/samba/ -type f -exec rm -f {} \;
 
 smbconffile=/etc/samba/smb.conf
@@ -52,6 +47,7 @@ systemctl unmask samba-ad-dc
 
 hostname ad.ldapcherry.org 
 pkill -9 dnsmasq
+pkill -9 samba
 
 kill -9 `cat /var/run/samba/smbd.pid` 
 rm -f /var/run/samba/smbd.pid

From 9989f97091a1665acaeddd5f7b4e1e0f4b0426b7 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 22:38:30 +0100
Subject: [PATCH 25/82] remove python3 test env for el7 and stretch (no
 python3-ldap lib anyway)

---
 .travis.yml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 35db8f9..1244290 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -37,14 +37,6 @@ matrix:
       env:
         TEST_PEP8=0
         REQ_FILE=requirements.txt
-    - python: "3.4"
-      env:
-        TEST_PEP8=0
-        REQ_FILE=requirements-el7.txt
-    - python: "3.5"
-      env:
-        TEST_PEP8=0
-        REQ_FILE=requirements-stretch.txt
     - python: "3.6"
       env:
         TEST_PEP8=0

From bc0f3aceb50dbebb27b3314039e4ffddb9cf2cb5 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 22:42:30 +0100
Subject: [PATCH 26/82] adding another dependency for the samba/ad test

---
 tests/test_env/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_env/deploy.sh b/tests/test_env/deploy.sh
index 353df4b..3f9c530 100755
--- a/tests/test_env/deploy.sh
+++ b/tests/test_env/deploy.sh
@@ -3,7 +3,7 @@
 apt update
 
 DEBIAN_FRONTEND=noninteractive apt-get install ldap-utils slapd -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
-DEBIAN_FRONTEND=noninteractive apt-get install samba -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
+DEBIAN_FRONTEND=noninteractive apt-get install samba-vfs-modules samba -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 DEBIAN_FRONTEND=noninteractive apt-get install winbind -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 
 [ -e '/etc/default/slapd' ] && rm -rf /etc/default/slapd

From 7430af5ffcd84e6d3f8c4426e861e5a7e0d70271 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 22:44:51 +0100
Subject: [PATCH 27/82] adding another samba package in test env

---
 tests/test_env/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_env/deploy.sh b/tests/test_env/deploy.sh
index 3f9c530..c37f642 100755
--- a/tests/test_env/deploy.sh
+++ b/tests/test_env/deploy.sh
@@ -3,7 +3,7 @@
 apt update
 
 DEBIAN_FRONTEND=noninteractive apt-get install ldap-utils slapd -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
-DEBIAN_FRONTEND=noninteractive apt-get install samba-vfs-modules samba -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
+DEBIAN_FRONTEND=noninteractive apt-get install samba-dsdb-modules samba-vfs-modules samba -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 DEBIAN_FRONTEND=noninteractive apt-get install winbind -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  -f -q -y
 
 [ -e '/etc/default/slapd' ] && rm -rf /etc/default/slapd

From 8b48a1f024b3db6add8dc67e69025bf791d66692 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 22:59:57 +0100
Subject: [PATCH 28/82] cleanup in travis file

---
 .travis.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 1244290..557f5e3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,9 +2,6 @@ sudo: required
 dist: xenial
 language: python
 
-#env:
-#  - TRAVIS="yes"
-
 before_install:
   - '[ "$TEST_PEP8" == "1" ] || sudo ./tests/test_env/deploy.sh'
 
@@ -14,10 +11,6 @@ install:
   - pip install passlib
   - pip install coveralls
 
-# command to run tests
-#
-#script: 
-#  - coverage run --source=ldapcherry setup.py test
 script: "if [[ $TEST_PEP8 == '1' ]]; then pycodestyle --repeat --show-source --exclude=.venv,.tox,dist,docs,build,*.egg,tests,misc,setup.py .; else coverage run --source=ldapcherry setup.py test; fi"
 matrix:
   include:

From d25ceef2d3c92b4349cbaec2a12ced519bb714ca Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Thu, 7 Feb 2019 23:46:10 +0100
Subject: [PATCH 29/82] trying to fix samba/AD setup

---
 tests/test_env/deploy.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/test_env/deploy.sh b/tests/test_env/deploy.sh
index c37f642..c2f748f 100755
--- a/tests/test_env/deploy.sh
+++ b/tests/test_env/deploy.sh
@@ -70,13 +70,14 @@ cat ${smbconffile}
 
 mv /var/lib/samba/private/krb5.conf /etc/krb5.conf
 
-sleep 5
+sleep 15
 
-systemctl start samba-ad-dc
-/etc/init.d/samba-ad-dc start
+systemctl restart samba-ad-dc
+/etc/init.d/samba-ad-dc restart
 
 cat /var/log/samba/*
 
 sleep 5
 
+systemctl status samba-ad-dc
 ss -apn | grep samba

From 12c511b53775fea213e1903eecf3aa400908e9bd Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Fri, 8 Feb 2019 00:11:01 +0100
Subject: [PATCH 30/82] switch to explicit bytearray for checking missing
 params

---
 ldapcherry/backend/backendAD.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ldapcherry/backend/backendAD.py b/ldapcherry/backend/backendAD.py
index 37c194c..5e34715 100644
--- a/ldapcherry/backend/backendAD.py
+++ b/ldapcherry/backend/backendAD.py
@@ -144,10 +144,10 @@ def __init__(self, config, logger, name, attrslist, key):
         for a in attrslist:
             self.attrlist.append(self._str(a))
 
-        if 'cn' not in self.attrlist:
+        if b'cn' not in self.attrlist:
             raise MissingAttr()
 
-        if 'unicodePwd' not in self.attrlist:
+        if b'unicodePwd' not in self.attrlist:
             raise MissingAttr()
 
     def _search_group(self, searchfilter, groupdn):

From 18fdeb483e422f8c97bae5d371e61459be95d162 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Fri, 8 Feb 2019 20:33:58 +0100
Subject: [PATCH 31/82] better handling of the str/byte mess for python3

* add dedicated methods for python 3 in handling of bytearrays/strings
* using them to compare attributes checks in AD backend
---
 ldapcherry/backend/backendAD.py   |  4 ++--
 ldapcherry/backend/backendLdap.py | 37 +++++++++++++++++++++----------
 2 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/ldapcherry/backend/backendAD.py b/ldapcherry/backend/backendAD.py
index 5e34715..57bf47a 100644
--- a/ldapcherry/backend/backendAD.py
+++ b/ldapcherry/backend/backendAD.py
@@ -144,10 +144,10 @@ def __init__(self, config, logger, name, attrslist, key):
         for a in attrslist:
             self.attrlist.append(self._str(a))
 
-        if b'cn' not in self.attrlist:
+        if self._str('cn') not in self.attrlist:
             raise MissingAttr()
 
-        if b'unicodePwd' not in self.attrlist:
+        if self._str('unicodePwd') not in self.attrlist:
             raise MissingAttr()
 
     def _search_group(self, searchfilter, groupdn):
diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index 9bf177c..b777929 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -312,22 +312,35 @@ def _get_user(self, username, attrs=ALL_ATTRS):
         else:
             dn_entry = r[0]
         return dn_entry
+
     # python-ldap talks in bytes,
     # as the rest of ldapcherry talks in unicode utf-8:
     # * everything passed to python-ldap must be converted to bytes
     # * everything coming from python-ldap must be converted to unicode
-
-    def _str(self, s):
-        """unicode -> bytes conversion"""
-        if s is None:
-            return None
-        return s.encode('utf-8')
-
-    def _uni(self, s):
-        """bytes -> unicode conversion"""
-        if s is None:
-            return None
-        return s.decode('utf-8', 'ignore')
+    if sys.version < '3':
+        def _str(self, s):
+            """unicode -> bytes conversion"""
+            if s is None:
+                return None
+            return s.encode('utf-8')
+        def _uni(self, s):
+            """bytes -> unicode conversion"""
+            if s is None:
+                return None
+            return s.decode('utf-8', 'ignore')
+    else:
+        def _str(self, s):
+            """unicode -> bytes conversion"""
+            return s
+
+        def _uni(self, s):
+            """bytes -> unicode conversion"""
+            if s is None:
+                return None
+            if type(s) is not str:
+                return s.decode('utf-8', 'ignore')
+            else:
+                return s
 
     def auth(self, username, password):
         """Authentication of a user"""

From 42759f1cc4519a75ccfd55c32ced6ca5fb6bd13a Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Fri, 8 Feb 2019 20:38:03 +0100
Subject: [PATCH 32/82] pep8

---
 ldapcherry/backend/backendLdap.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index b777929..30d48d4 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -323,6 +323,7 @@ def _str(self, s):
             if s is None:
                 return None
             return s.encode('utf-8')
+
         def _uni(self, s):
             """bytes -> unicode conversion"""
             if s is None:

From 8c0bf94904c2254f99d307a13b9b9c48a1150856 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Fri, 8 Feb 2019 20:38:29 +0100
Subject: [PATCH 33/82] better log+fix in conf checking + fix in ppolicy
 handler

* log where the backend is declared (role or attribute) when
inconsistency with main .ini file
* fix check of configuration, only role file was checked 2 times instead
on checking role one time and attribute one time
* <dict>.keys() seems to have a different behavior between 2 (return
"list") and 3 (return "dict_keys"), casting to "list" to avoid that.
---
 ldapcherry/__init__.py   | 8 ++++----
 ldapcherry/exceptions.py | 7 ++++---
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index 642ee2f..d058d80 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -149,10 +149,10 @@ def _check_backends(self):
         backends = self.backends_params.keys()
         for b in self.roles.get_backends():
             if b not in backends:
-                raise MissingBackend(b)
-        for b in self.roles.get_backends():
+                raise MissingBackend(b, 'role')
+        for b in self.attributes.get_backends():
             if b not in backends:
-                raise MissingBackend(b)
+                raise MissingBackend(b, 'attribute')
 
     def _init_backends(self, config):
         """ Init all backends
@@ -1003,7 +1003,7 @@ def searchuser(self, searchstring=None):
     def checkppolicy(self, **params):
         """ search user page """
         self._check_auth(must_admin=False, redir_login=False)
-        keys = params.keys()
+        keys = list(params.keys())
         if len(keys) != 1:
             cherrypy.response.status = 400
             return "bad argument"
diff --git a/ldapcherry/exceptions.py b/ldapcherry/exceptions.py
index 0bd39b5..cb3ac54 100644
--- a/ldapcherry/exceptions.py
+++ b/ldapcherry/exceptions.py
@@ -46,11 +46,12 @@ def __init__(self, role):
 
 
 class MissingBackend(Exception):
-    def __init__(self, backend):
+    def __init__(self, backend, type_conf):
         self.backend = backend
         self.log = \
-            "backend '%(backend)s' does not exist in main config file" % \
-            {'backend': backend}
+            "backend '%(backend)s' does not exist in main config file " \
+            "but is still declared in '%(type_conf)s' file" % \
+            {'backend': backend, 'type_conf': type_conf}
 
 
 class WrongBackend(Exception):

From 60d57d8530c6e0e773600340edb93c161054a5b3 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Fri, 8 Feb 2019 20:47:15 +0100
Subject: [PATCH 34/82] changelog

---
 ChangeLog.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 1c009fa..12f087d 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -2,6 +2,8 @@ Dev
 ***
 
 * [sec ] fix XSS injection in the url redirect in the login page (thanks to jthiltges)
+* [fix ] fix configuration consistency check for attribute file (error if a given backend is not declared in main .ini file but in attributes)
+* [impr] better log error message if inconsistency between role, attribute and main .ini file for backends
 * [impr] more systematic use of html and url escaping in the html rendering to prevent against content injection (thanks to jthiltges)
 
 Version 0.5.2

From b9437abefb98650d4ddca0dc8b285de2544f95a9 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 11:58:09 +0100
Subject: [PATCH 35/82] * support for python-ldap 2 and 3

* python-ldap 3 is slightly different than 2 on how it handles modify
the modified attributes used to be transmitted as a dict, now it should
be transmitted as a list of dict)
---
 ldapcherry/backend/backendAD.py   |  4 +--
 ldapcherry/backend/backendLdap.py | 42 ++++++++++++++++++++-----------
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/ldapcherry/backend/backendAD.py b/ldapcherry/backend/backendAD.py
index 57bf47a..ace67e3 100644
--- a/ldapcherry/backend/backendAD.py
+++ b/ldapcherry/backend/backendAD.py
@@ -192,13 +192,13 @@ def _set_password(self, name, password, by_cn=True):
 
         attrs = {}
 
-        attrs['unicodePwd'] = self._str(password_value)
+        attrs['unicodePwd'] = self._modlist(self._str(password_value))
 
         ldif = modlist.modifyModlist({'unicodePwd': 'tmp'}, attrs)
         ldap_client.modify_s(dn, ldif)
 
         del(attrs['unicodePwd'])
-        attrs['UserAccountControl'] = str(NORMAL_ACCOUNT)
+        attrs['UserAccountControl'] = self._modlist(str(NORMAL_ACCOUNT))
         ldif = modlist.modifyModlist({'UserAccountControl': 'tmp'}, attrs)
         ldap_client.modify_s(dn, ldif)
 
diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index 30d48d4..0955a83 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -20,6 +20,7 @@
 if sys.version < '3':
     from sets import Set as set
 
+PYTHON_LDAP_MAJOR_VERSION = ldap.__version__[0]
 
 class CaFileDontExist(Exception):
     def __init__(self, cafile):
@@ -362,10 +363,19 @@ def auth(self, username, password):
         else:
             return False
 
+    if PYTHON_LDAP_MAJOR_VERSION == '2':
+        @staticmethod
+        def _modlist(in_attr):
+            return in_attr
+    else:
+        @staticmethod
+        def _modlist(in_attr):
+            return [in_attr]
+
     def attrs_pretreatment(self, attrs):
         attrs_str = {}
         for a in attrs:
-            attrs_str[self._str(a)] = self._str(attrs[a])
+            attrs_str[self._str(a)] = self._modlist(self._str(attrs[a]))
         return attrs_str
 
     def add_user(self, attrs):
@@ -374,17 +384,18 @@ def add_user(self, attrs):
         # encoding crap
         attrs_str = self.attrs_pretreatment(attrs)
 
-        attrs_str['objectClass'] = self.objectclasses
+        attrs_str[self._str('objectClass')] = self.objectclasses
         # construct is DN
         dn = \
             self._str(self.dn_user_attr) + \
-            '=' + \
-            ldap.dn.escape_dn_chars(
-                self._str(attrs[self.dn_user_attr])
+            self._str('=') + \
+            self._str(ldap.dn.escape_dn_chars(
+                          attrs[self.dn_user_attr]
+                      )
                 ) + \
-            ',' + \
+            self._str(',') + \
             self._str(self.userdn)
-        # gen the ldif fir add_s and add the user
+        # gen the ldif first add_s and add the user
         ldif = modlist.addModlist(attrs_str)
         try:
             ldap_client.add_s(dn, ldif)
@@ -419,7 +430,7 @@ def set_attrs(self, username, attrs):
         for attr in attrs:
             bcontent = self._str(attrs[attr])
             battr = self._str(attr)
-            new = {battr: bcontent}
+            new = {battr: self._modlist(bcontent)}
             # if attr is dn entry, use rename
             if attr.lower() == self.dn_user_attr.lower():
                 ldap_client.rename_s(
@@ -439,17 +450,18 @@ def set_attrs(self, username, attrs):
                             tmp.append(self._str(value))
                         bold_value = tmp
                     else:
-                        bold_value = self._str(old_attrs[attr])
+                        bold_value = self._modlist(self._str(old_attrs[attr]))
                     old = {battr: bold_value}
                 # attribute is not set, just add it
                 else:
                     old = {}
                 ldif = modlist.modifyModlist(old, new)
-                try:
-                    ldap_client.modify_s(dn, ldif)
-                except Exception as e:
-                    ldap_client.unbind_s()
-                    self._exception_handler(e)
+                if ldif:
+                    try:
+                        ldap_client.modify_s(dn, ldif)
+                    except Exception as e:
+                        ldap_client.unbind_s()
+                        self._exception_handler(e)
 
         ldap_client.unbind_s()
 
@@ -482,7 +494,7 @@ def add_to_groups(self, username, groups):
                             'backend': self.backend_name
                             }
                 )
-                ldif = modlist.modifyModlist({}, {attr: content})
+                ldif = modlist.modifyModlist({}, {attr: self._modlist(content)})
                 try:
                     ldap_client.modify_s(group, ldif)
                 # if already member, not a big deal, just log it and continue

From bbfe96d4f780bb21894348915bc1eae7abbb263c Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 12:05:09 +0100
Subject: [PATCH 36/82] pep8

---
 ldapcherry/backend/backendLdap.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index 0955a83..2444cf8 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -22,6 +22,7 @@
 
 PYTHON_LDAP_MAJOR_VERSION = ldap.__version__[0]
 
+
 class CaFileDontExist(Exception):
     def __init__(self, cafile):
         self.cafile = cafile
@@ -391,7 +392,7 @@ def add_user(self, attrs):
             self._str('=') + \
             self._str(ldap.dn.escape_dn_chars(
                           attrs[self.dn_user_attr]
-                      )
+                     )
                 ) + \
             self._str(',') + \
             self._str(self.userdn)
@@ -494,7 +495,10 @@ def add_to_groups(self, username, groups):
                             'backend': self.backend_name
                             }
                 )
-                ldif = modlist.modifyModlist({}, {attr: self._modlist(content)})
+                ldif = modlist.modifyModlist(
+                        {},
+                        {attr: self._modlist(content)}
+                       )
                 try:
                     ldap_client.modify_s(group, ldif)
                 # if already member, not a big deal, just log it and continue

From fb6b0a5d3146d18c3601fbb526fcf75f3be0c0fd Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 12:12:24 +0100
Subject: [PATCH 37/82] limit cherrypy to < 18.0.0 in setup.py

cherrypy dropped support for python2 with 18.0.0, 17.X is the last
version usable with python 2.
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ace0cdc..a52e92c 100755
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,7 @@
 # change requirements according to python version
 if sys.version_info[0] == 2:
     install_requires = [
-        'CherryPy >= 3.0.0',
+        'CherryPy >= 3.0.0,< 18.0.0',
         'python-ldap',
         'PyYAML',
         'Mako'

From 979d4eeda8026e57d391024d6c0559f5e89e3265 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 15:42:48 +0100
Subject: [PATCH 38/82] disable ppolicy in samba test deployment

---
 tests/test_env/deploy.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_env/deploy.sh b/tests/test_env/deploy.sh
index c2f748f..f009070 100755
--- a/tests/test_env/deploy.sh
+++ b/tests/test_env/deploy.sh
@@ -79,5 +79,7 @@ cat /var/log/samba/*
 
 sleep 5
 
+samba-tool domain passwordsettings set -d 1 --complexity off
+samba-tool domain passwordsettings set -d 1 --min-pwd-length 0
 systemctl status samba-ad-dc
 ss -apn | grep samba

From 10747cff93b502f08dfa383ade2ce452d00f2994 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 16:08:18 +0100
Subject: [PATCH 39/82] add some python 3 support in the LDAP and AD backends

python-ldap talks in bytes,
as the rest of ldapcherry talks in unicode utf-8:
* everything passed to python-ldap must be converted to bytes
* everything coming from python-ldap must be converted to unicode

The previous statement was true for python-ldap < version 3.X.
With versions > 3.0.0 and python 3, it gets tricky,
some parts of python-ldap takes string, specially the filters/escaper.

so we have now:
*_byte_p2 (unicode -> bytes conversion for python 2)
*_byte_p3 (unicode -> bytes conversion for python 3)
*_byte_p23 (unicode -> bytes conversion for python AND 3)
---
 ldapcherry/backend/backendAD.py   |  48 ++++++----
 ldapcherry/backend/backendLdap.py | 140 +++++++++++++++++++-----------
 2 files changed, 121 insertions(+), 67 deletions(-)

diff --git a/ldapcherry/backend/backendAD.py b/ldapcherry/backend/backendAD.py
index ace67e3..9483390 100644
--- a/ldapcherry/backend/backendAD.py
+++ b/ldapcherry/backend/backendAD.py
@@ -15,6 +15,7 @@
 from ldapcherry.exceptions import UserDoesntExist, GroupDoesntExist
 import os
 import re
+import sys
 
 
 class CaFileDontExist(Exception):
@@ -129,11 +130,11 @@ def __init__(self, config, logger, name, attrslist, key):
         self.dn_user_attr = 'cn'
         self.key = 'sAMAccountName'
         self.objectclasses = [
-            'top',
-            'person',
-            'organizationalPerson',
-            'user',
-            'posixAccount',
+            self._byte_p23('top'),
+            self._byte_p23('person'),
+            self._byte_p23('organizationalPerson'),
+            self._byte_p23('user'),
+            self._byte_p23('posixAccount'),
             ]
         self.group_attrs = {
             'member': "%(dn)s"
@@ -142,16 +143,25 @@ def __init__(self, config, logger, name, attrslist, key):
         self.attrlist = []
         self.group_attrs_keys = []
         for a in attrslist:
-            self.attrlist.append(self._str(a))
+            self.attrlist.append(self._byte_p2(a))
 
-        if self._str('cn') not in self.attrlist:
+        if self._byte_p2('cn') not in self.attrlist:
             raise MissingAttr()
 
-        if self._str('unicodePwd') not in self.attrlist:
+        if self._byte_p2('unicodePwd') not in self.attrlist:
             raise MissingAttr()
 
+    if sys.version < '3':
+        @staticmethod
+        def _tobyte(in_int):
+            return str(in_int)
+    else:
+        @staticmethod
+        def _tobyte(in_int):
+            return in_int.to_bytes(4, byteorder='big')
+
     def _search_group(self, searchfilter, groupdn):
-        searchfilter = self._str(searchfilter)
+        searchfilter = self._byte_p2(searchfilter)
         ldap_client = self._bind()
         try:
             r = ldap_client.search_s(
@@ -183,22 +193,24 @@ def _set_password(self, name, password, by_cn=True):
         ldap_client = self._bind()
 
         if by_cn:
-            dn = self._str('CN=%(cn)s,%(user_dn)s' % {
+            dn = self._byte_p2('CN=%(cn)s,%(user_dn)s' % {
                         'cn': name,
                         'user_dn': self.userdn
                        })
         else:
-            dn = self._str(name)
+            dn = self._byte_p2(name)
 
         attrs = {}
 
-        attrs['unicodePwd'] = self._modlist(self._str(password_value))
+        attrs['unicodePwd'] = self._modlist(self._byte_p2(password_value))
 
         ldif = modlist.modifyModlist({'unicodePwd': 'tmp'}, attrs)
         ldap_client.modify_s(dn, ldif)
 
         del(attrs['unicodePwd'])
-        attrs['UserAccountControl'] = self._modlist(str(NORMAL_ACCOUNT))
+        attrs['UserAccountControl'] = self._modlist(
+            self._tobyte(NORMAL_ACCOUNT)
+        )
         ldif = modlist.modifyModlist({'UserAccountControl': 'tmp'}, attrs)
         ldap_client.modify_s(dn, ldif)
 
@@ -212,7 +224,7 @@ def set_attrs(self, username, attrs):
         if 'unicodePwd' in attrs:
             password = attrs['unicodePwd']
             del(attrs['unicodePwd'])
-            userdn = self._get_user(self._str(username), NO_ATTR)
+            userdn = self._get_user(self._byte_p2(username), NO_ATTR)
             self._set_password(userdn, password, False)
         super(Backend, self).set_attrs(username, attrs)
 
@@ -226,7 +238,7 @@ def del_from_groups(self, username, groups):
 
     def get_groups(self, username):
         username = ldap.filter.escape_filter_chars(username)
-        userdn = self._get_user(self._str(username), NO_ATTR)
+        userdn = self._get_user(self._byte_p2(username), NO_ATTR)
 
         searchfilter = self.group_filter_tmpl % {
             'userdn': userdn,
@@ -246,7 +258,7 @@ def get_groups(self, username):
         )
 
         for entry in groups:
-            ret.append(entry[1]['cn'][0])
+            ret.append(self._uni(entry[1]['cn'][0]))
         return ret
 
     def auth(self, username, password):
@@ -256,8 +268,8 @@ def auth(self, username, password):
             ldap_client = self._connect()
             try:
                 ldap_client.simple_bind_s(
-                    self._str(binddn),
-                    self._str(password)
+                    self._byte_p2(binddn),
+                    self._byte_p2(password)
                 )
             except ldap.INVALID_CREDENTIALS:
                 ldap_client.unbind_s()
diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index 2444cf8..e60ae98 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -76,7 +76,7 @@ def __init__(self, config, logger, name, attrslist, key):
         # objectclasses parameter is a coma separated list in configuration
         # split it to get a real list, and convert it to bytes
         for o in re.split(r'\W+', self.get_param('objectclasses')):
-            self.objectclasses.append(self._str(o))
+            self.objectclasses.append(self._byte_p23(o))
         self.group_attrs = {}
         self.group_attrs_keys = set([])
         for param in config:
@@ -89,7 +89,7 @@ def __init__(self, config, logger, name, attrslist, key):
 
         self.attrlist = []
         for a in attrslist:
-            self.attrlist.append(self._str(a))
+            self.attrlist.append(self._byte_p2(a))
 
     # exception handler (mainly to log something meaningful)
     def _exception_handler(self, e):
@@ -302,7 +302,7 @@ def _get_user(self, username, attrs=ALL_ATTRS):
         user_filter = self.user_filter_tmpl % {
             'username': self._uni(username)
         }
-        r = self._search(self._str(user_filter), attrs, self.userdn)
+        r = self._search(self._byte_p2(user_filter), attrs, self.userdn)
 
         if len(r) == 0:
             return None
@@ -319,23 +319,56 @@ def _get_user(self, username, attrs=ALL_ATTRS):
     # as the rest of ldapcherry talks in unicode utf-8:
     # * everything passed to python-ldap must be converted to bytes
     # * everything coming from python-ldap must be converted to unicode
+    #
+    # The previous statement was true for python-ldap < version 3.X.
+    # With versions > 3.0.0 and python 3, it gets tricky,
+    # some parts of python-ldap takes string, specially the filters/escaper.
+    #
+    # so we have now:
+    # *_byte_p2 (unicode -> bytes conversion for python 2)
+    # *_byte_p3 (unicode -> bytes conversion for python 3)
+    # *_byte_p23 (unicode -> bytes conversion for python AND 3)
+    def _byte_p23(self, s):
+        """unicode -> bytes conversion"""
+        if s is None:
+            return None
+        return s.encode('utf-8')
+
     if sys.version < '3':
-        def _str(self, s):
-            """unicode -> bytes conversion"""
+        def _byte_p2(self, s):
+            """unicode -> bytes conversion (python 2)"""
             if s is None:
                 return None
             return s.encode('utf-8')
 
+        def _byte_p3(self, s):
+            """pass through (does something in python 3)"""
+            return s
+
         def _uni(self, s):
             """bytes -> unicode conversion"""
             if s is None:
                 return None
             return s.decode('utf-8', 'ignore')
+
+        def attrs_pretreatment(self, attrs):
+            attrs_srt = {}
+            for a in attrs:
+                attrs_srt[self._byte_p2(a)] = self._modlist(
+                    self._byte_p2(attrs[a])
+                )
+            return attrs_srt
     else:
-        def _str(self, s):
-            """unicode -> bytes conversion"""
+        def _byte_p2(self, s):
+            """pass through (does something in python 2)"""
             return s
 
+        def _byte_p3(self, s):
+            """unicode -> bytes conversion"""
+            if s is None:
+                return None
+            return s.encode('utf-8')
+
         def _uni(self, s):
             """bytes -> unicode conversion"""
             if s is None:
@@ -345,16 +378,24 @@ def _uni(self, s):
             else:
                 return s
 
+        def attrs_pretreatment(self, attrs):
+            attrs_srt = {}
+            for a in attrs:
+                attrs_srt[self._byte_p2(a)] = self._modlist(
+                    self._byte_p3(attrs[a])
+                )
+            return attrs_srt
+
     def auth(self, username, password):
         """Authentication of a user"""
 
-        binddn = self._get_user(self._str(username), NO_ATTR)
+        binddn = self._get_user(self._byte_p2(username), NO_ATTR)
         if binddn is not None:
             ldap_client = self._connect()
             try:
                 ldap_client.simple_bind_s(
-                        self._str(binddn),
-                        self._str(password)
+                        self._byte_p2(binddn),
+                        self._byte_p2(password)
                         )
             except ldap.INVALID_CREDENTIALS:
                 ldap_client.unbind_s()
@@ -368,36 +409,31 @@ def auth(self, username, password):
         @staticmethod
         def _modlist(in_attr):
             return in_attr
+
     else:
         @staticmethod
         def _modlist(in_attr):
             return [in_attr]
 
-    def attrs_pretreatment(self, attrs):
-        attrs_str = {}
-        for a in attrs:
-            attrs_str[self._str(a)] = self._modlist(self._str(attrs[a]))
-        return attrs_str
-
     def add_user(self, attrs):
         """add a user"""
         ldap_client = self._bind()
         # encoding crap
-        attrs_str = self.attrs_pretreatment(attrs)
+        attrs_srt = self.attrs_pretreatment(attrs)
 
-        attrs_str[self._str('objectClass')] = self.objectclasses
+        attrs_srt[self._byte_p2('objectClass')] = self.objectclasses
         # construct is DN
         dn = \
-            self._str(self.dn_user_attr) + \
-            self._str('=') + \
-            self._str(ldap.dn.escape_dn_chars(
-                          attrs[self.dn_user_attr]
-                     )
+            self._byte_p2(self.dn_user_attr) + \
+            self._byte_p2('=') + \
+            self._byte_p2(ldap.dn.escape_dn_chars(
+                        attrs[self.dn_user_attr]
+                    )
                 ) + \
-            self._str(',') + \
-            self._str(self.userdn)
+            self._byte_p2(',') + \
+            self._byte_p2(self.userdn)
         # gen the ldif first add_s and add the user
-        ldif = modlist.addModlist(attrs_str)
+        ldif = modlist.addModlist(attrs_srt)
         try:
             ldap_client.add_s(dn, ldif)
         except ldap.ALREADY_EXISTS as e:
@@ -411,7 +447,7 @@ def del_user(self, username):
         """delete a user"""
         ldap_client = self._bind()
         # recover the user dn
-        dn = self._str(self._get_user(self._str(username), NO_ATTR))
+        dn = self._byte_p2(self._get_user(self._byte_p2(username), NO_ATTR))
         # delete
         if dn is not None:
             ldap_client.delete_s(dn)
@@ -423,15 +459,15 @@ def del_user(self, username):
     def set_attrs(self, username, attrs):
         """ set user attributes"""
         ldap_client = self._bind()
-        tmp = self._get_user(self._str(username), ALL_ATTRS)
+        tmp = self._get_user(self._byte_p2(username), ALL_ATTRS)
         if tmp is None:
             raise UserDoesntExist(username, self.backend_name)
-        dn = self._str(tmp[0])
+        dn = self._byte_p2(tmp[0])
         old_attrs = tmp[1]
         for attr in attrs:
-            bcontent = self._str(attrs[attr])
-            battr = self._str(attr)
-            new = {battr: self._modlist(bcontent)}
+            bcontent = self._byte_p2(attrs[attr])
+            battr = self._byte_p2(attr)
+            new = {battr: self._modlist(self._byte_p3(bcontent))}
             # if attr is dn entry, use rename
             if attr.lower() == self.dn_user_attr.lower():
                 ldap_client.rename_s(
@@ -448,10 +484,12 @@ def set_attrs(self, username, attrs):
                     if type(old_attrs[attr]) is list:
                         tmp = []
                         for value in old_attrs[attr]:
-                            tmp.append(self._str(value))
+                            tmp.append(self._byte_p2(value))
                         bold_value = tmp
                     else:
-                        bold_value = self._modlist(self._str(old_attrs[attr]))
+                        bold_value = self._modlist(
+                            self._byte_p3(old_attrs[attr])
+                        )
                     old = {battr: bold_value}
                 # attribute is not set, just add it
                 else:
@@ -469,19 +507,19 @@ def set_attrs(self, username, attrs):
     def add_to_groups(self, username, groups):
         ldap_client = self._bind()
         # recover dn of the user and his attributes
-        tmp = self._get_user(self._str(username), ALL_ATTRS)
+        tmp = self._get_user(self._byte_p2(username), ALL_ATTRS)
         dn = tmp[0]
         attrs = tmp[1]
         attrs['dn'] = dn
         self._normalize_group_attrs(attrs)
-        dn = self._str(tmp[0])
+        dn = self._byte_p2(tmp[0])
         # add user to all groups
         for group in groups:
-            group = self._str(group)
+            group = self._byte_p2(group)
             # iterate on group membership attributes
             for attr in self.group_attrs:
                 # fill the content template
-                content = self._str(self.group_attrs[attr] % attrs)
+                content = self._byte_p2(self.group_attrs[attr] % attrs)
                 self._logger(
                     severity=logging.DEBUG,
                     msg="%(backend)s: adding user '%(user)s'"
@@ -497,12 +535,14 @@ def add_to_groups(self, username, groups):
                 )
                 ldif = modlist.modifyModlist(
                         {},
-                        {attr: self._modlist(content)}
+                        {attr: self._modlist(self._byte_p3(content))}
                        )
                 try:
+                    print(ldif)
+                    print(group)
                     ldap_client.modify_s(group, ldif)
                 # if already member, not a big deal, just log it and continue
-                except ldap.TYPE_OR_VALUE_EXISTS as e:
+                except (ldap.TYPE_OR_VALUE_EXISTS, ldap.ALREADY_EXISTS) as e:
                     self._logger(
                         severity=logging.INFO,
                         msg="%(backend)s: user '%(user)s'"
@@ -526,19 +566,19 @@ def del_from_groups(self, username, groups):
         # it follows the same logic than add_to_groups
         # but with MOD_DELETE
         ldap_client = self._bind()
-        tmp = self._get_user(self._str(username), ALL_ATTRS)
+        tmp = self._get_user(self._byte_p2(username), ALL_ATTRS)
         if tmp is None:
             raise UserDoesntExist(username, self.backend_name)
         dn = tmp[0]
         attrs = tmp[1]
         attrs['dn'] = dn
         self._normalize_group_attrs(attrs)
-        dn = self._str(tmp[0])
+        dn = self._byte_p2(tmp[0])
         for group in groups:
-            group = self._str(group)
+            group = self._byte_p2(group)
             for attr in self.group_attrs:
-                content = self._str(self.group_attrs[attr] % attrs)
-                ldif = [(ldap.MOD_DELETE, attr, content)]
+                content = self._byte_p2(self.group_attrs[attr] % attrs)
+                ldif = [(ldap.MOD_DELETE, attr, self._byte_p3(content))]
                 try:
                     ldap_client.modify_s(group, ldif)
                 except ldap.NO_SUCH_ATTRIBUTE as e:
@@ -561,7 +601,9 @@ def del_from_groups(self, username, groups):
     def search(self, searchstring):
         """Search users"""
         # escape special char to avoid injection
-        searchstring = ldap.filter.escape_filter_chars(self._str(searchstring))
+        searchstring = ldap.filter.escape_filter_chars(
+            self._byte_p2(searchstring)
+        )
         # fill the search string template
         searchfilter = self.search_filter_tmpl % {
             'searchstring': searchstring
@@ -586,7 +628,7 @@ def search(self, searchstring):
     def get_user(self, username):
         """Gest a specific user"""
         ret = {}
-        tmp = self._get_user(self._str(username), ALL_ATTRS)
+        tmp = self._get_user(self._byte_p2(username), ALL_ATTRS)
         if tmp is None:
             raise UserDoesntExist(username, self.backend_name)
         attrs_tmp = tmp[1]
@@ -600,7 +642,7 @@ def get_user(self, username):
 
     def get_groups(self, username):
         """Get all groups of a user"""
-        username = ldap.filter.escape_filter_chars(self._str(username))
+        username = ldap.filter.escape_filter_chars(self._byte_p2(username))
         userdn = self._get_user(username, NO_ATTR)
 
         searchfilter = self.group_filter_tmpl % {
@@ -611,5 +653,5 @@ def get_groups(self, username):
         groups = self._search(searchfilter, NO_ATTR, self.groupdn)
         ret = []
         for entry in groups:
-            ret.append(entry[0])
+            ret.append(self._uni(entry[0]))
         return ret

From 79983c078f77957ffe2c9f81352d89afbcc4e593 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 16:22:42 +0100
Subject: [PATCH 40/82] fix behavior of get_attributes()

* make sure it returns an ordered list in both python 2 and python 3
---
 ldapcherry/attributes.py | 4 +++-
 tests/test_Attributes.py | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ldapcherry/attributes.py b/ldapcherry/attributes.py
index 989bf14..5ddea82 100644
--- a/ldapcherry/attributes.py
+++ b/ldapcherry/attributes.py
@@ -130,7 +130,9 @@ def get_backends(self):
     def get_backend_attributes(self, backend):
         if backend not in self.backends:
             raise WrongBackend(backend)
-        return self.backend_attributes[backend].keys()
+        ret = list(self.backend_attributes[backend].keys())
+        ret.sort()
+        return ret
 
     def get_backend_key(self, backend):
         if backend not in self.backends:
diff --git a/tests/test_Attributes.py b/tests/test_Attributes.py
index f7f42b2..aab7342 100644
--- a/tests/test_Attributes.py
+++ b/tests/test_Attributes.py
@@ -42,6 +42,7 @@ def testGetBackendAttributes(self):
         inv = Attributes('./tests/cfg/attributes.yml')
         ret = inv.get_backend_attributes('ldap')
         expected = ['shell', 'cn', 'userPassword', 'uidNumber', 'gidNumber', 'sn', 'home', 'givenName', 'email', 'uid']
+        expected.sort()
         assert ret == expected
 
     def testGetKey(self):

From 90ff69586b0579e6d059d789da9a9ad37031c372 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 16:29:16 +0100
Subject: [PATCH 41/82] remove deprecation warning for html escape

in python 2, (html) escape is part of the cgi module
in python 3, it's part of the html module

we now do a conditional import depending on the version, and name the
function html_escape.
---
 ldapcherry/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index d058d80..0acb0e0 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -15,7 +15,6 @@
 import logging.handlers
 from operator import itemgetter
 from socket import error as socket_error
-import cgi
 
 from ldapcherry.exceptions import *
 from ldapcherry.lclogging import *
@@ -35,8 +34,10 @@
 if sys.version < '3':
     from sets import Set as set
     from urllib import quote_plus
+    from cgi import escape as html_escape
 else:
     from urllib.parse import quote_plus
+    from html import escape as html_escape
 
 SESSION_KEY = '_cp_username'
 
@@ -64,7 +65,7 @@ def _handle_exception(self, e):
     def _escape_list(self, data):
         ret = []
         for i in data:
-            ret.append(cgi.escape(i, True))
+            ret.append(html_escape(i, True))
         return ret
 
     def _escape_dict(self, data):
@@ -76,7 +77,7 @@ def _escape_dict(self, data):
             elif isinstance(data[d], set):
                 data[d] = set(self._escape_list(data[d]))
             else:
-                data[d] = cgi.escape(data[d], True)
+                data[d] = html_escape(data[d], True)
         return data
 
     def _escape(self, data, dtype):

From baa3430e637589814267eb88dc0741eefe81f621 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 17:12:39 +0100
Subject: [PATCH 42/82] fix test and exception handling in code

With python 2 it was possible to do exception[0][...] to recover
details about an exception.
It's no longer authorized with python 3.
Now, we must do something like exception.args or exception.urls.
fortunately this syntax also works with python 2.
So we use it for both.
---
 ldapcherry/backend/backendLdap.py | 6 +++---
 tests/test_BackendLdap.py         | 4 ++--
 tests/test_LdapCherry.py          | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index e60ae98..cceaa96 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -132,8 +132,8 @@ def _exception_handler(self, e):
                     ".groupdn'",
                 )
         elif et is ldap.OBJECT_CLASS_VIOLATION:
-            info = e[0]['info']
-            desc = e[0]['desc']
+            info = e.args[0]['info']
+            desc = e.args[0]['desc']
             self._logger(
                 severity=logging.ERROR,
                 msg="Configuration error, " + desc + ", " + info,
@@ -147,7 +147,7 @@ def _exception_handler(self, e):
                     self.backend_name,
                 )
         elif et is ldap.ALREADY_EXISTS:
-            desc = e[0]['desc']
+            desc = e.args[0]['desc']
             self._logger(
                 severity=logging.ERROR,
                 msg="adding user failed, " + desc,
diff --git a/tests/test_BackendLdap.py b/tests/test_BackendLdap.py
index eefabef..3f80930 100644
--- a/tests/test_BackendLdap.py
+++ b/tests/test_BackendLdap.py
@@ -105,8 +105,8 @@ def testConnectSSLWrongCA(self):
         try:
             ldapc.simple_bind_s(inv.binddn, inv.bindpassword)
         except ldap.SERVER_DOWN as e:
-            assert e[0]['info'] == 'TLS: hostname does not match CN in peer certificate' or \
-                    e[0]['info'] == '(unknown error code)'
+            assert e.args[0]['info'] == 'TLS: hostname does not match CN in peer certificate' or \
+                    e.args[0]['info'] == '(unknown error code)'
         else:
             raise AssertionError("expected an exception")
 
diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 54611f2..536f80d 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -185,7 +185,7 @@ def testLogin(self):
             app.login(u'jwatsoné', u'passwordwatsoné')
         except cherrypy.HTTPRedirect as e:
             expected = 'http://127.0.0.1:8080/'
-            assert e[0][0] == expected
+            assert e.urls[0] == expected
         else:
             raise AssertionError("expected an exception")
 
@@ -197,7 +197,7 @@ def testLoginFailure(self):
             app.login(u'jwatsoné', u'wrongPasswordé')
         except cherrypy.HTTPRedirect as e:
             expected = 'http://127.0.0.1:8080/signin'
-            assert e[0][0] == expected
+            assert e.urls[0] == expected
         else:
             raise AssertionError("expected an exception")
 

From 05aace0e9da4535a8c0822c552fe5ae1a0e15424 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 17:36:01 +0100
Subject: [PATCH 43/82] force the groups in flatten roles to be sorted

* sorting the groups helps debuggability and also permits testing
that doesn't rely on python ordering (which is different between 2 and
3).
---
 ldapcherry/roles.py |  1 +
 tests/test_Roles.py | 45 ++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/ldapcherry/roles.py b/ldapcherry/roles.py
index 9fc6a92..a12fd70 100644
--- a/ldapcherry/roles.py
+++ b/ldapcherry/roles.py
@@ -58,6 +58,7 @@ def _merge_groups(self, backends_list):
                     ret[b].add(group)
         for b in ret:
             ret[b] = list(ret[b])
+            ret[b].sort()
         return ret
 
     def _flatten(self, roles=None, groups=None):
diff --git a/tests/test_Roles.py b/tests/test_Roles.py
index 72bddc1..84e9994 100644
--- a/tests/test_Roles.py
+++ b/tests/test_Roles.py
@@ -79,7 +79,50 @@ def testGetGroup(self):
 
     def testNested(self):
         inv = Roles('./tests/cfg/nested.yml')
-        expected = {'developpers': {'backends_groups': {'ad': ['Domain Users'], 'ldap': ['cn=developpers,ou=group,dc=example,dc=com', 'cn=users,ou=group,dc=example,dc=com']}, 'display_name': 'Developpers', 'description': 'description'}, 'admin-lv3': {'backends_groups': {'ad': ['Domain Users', 'Administrators', 'Domain Controllers'], 'ldap': ['cn=nagios admins,ou=group,dc=example,dc=com', 'cn=users,ou=group,dc=example,dc=com', 'cn=puppet admins,ou=group,dc=example,dc=com', 'cn=dns admins,ou=group,dc=example,dc=com']}, 'display_name': 'Administrators Level 3', 'description': 'description'}, 'admin-lv2': {'backends_groups': {'ad': ['Domain Users'], 'ldap': ['cn=nagios admins,ou=group,dc=example,dc=com', 'cn=users,ou=group,dc=example,dc=com']}, 'display_name': 'Administrators Level 2', 'description': 'description', 'LC_admins': True}, 'users': {'backends_groups': {'ad': ['Domain Users'], 'ldap': ['cn=users,ou=group,dc=example,dc=com']}, 'display_name': 'Simple Users', 'description': 'description'}}
+        expected = {
+            'admin-lv2': {
+                'LC_admins': True,
+                'backends_groups': {
+                    'ad': ['Domain Users'],
+                     'ldap': ['cn=nagios '
+                              'admins,ou=group,dc=example,dc=com',
+                              'cn=users,ou=group,dc=example,dc=com']
+                },
+                'description': 'description',
+                'display_name': 'Administrators Level 2'
+            },
+            'admin-lv3': {
+                'backends_groups': {
+                    'ad': ['Administrators',
+                           'Domain Controllers',
+                           'Domain Users'],
+                    'ldap': ['cn=dns '
+                             'admins,ou=group,dc=example,dc=com',
+                             'cn=nagios '
+                             'admins,ou=group,dc=example,dc=com',
+                             'cn=puppet '
+                             'admins,ou=group,dc=example,dc=com',
+                             'cn=users,ou=group,dc=example,dc=com']
+                    },
+                'description': 'description',
+                'display_name': 'Administrators Level 3'
+            },
+            'developpers': {
+                'backends_groups': {
+                    'ad': ['Domain Users'],
+                    'ldap': ['cn=developpers,ou=group,dc=example,dc=com',
+                             'cn=users,ou=group,dc=example,dc=com']},
+                'description': 'description',
+                'display_name': 'Developpers'
+            },
+            'users': {
+                'backends_groups': {
+                    'ad': ['Domain Users'],
+                    'ldap': ['cn=users,ou=group,dc=example,dc=com']},
+            'description': 'description',
+            'display_name': 'Simple Users'
+            }
+        }
         assert expected == inv.flatten
 
     def testGetGroupMissingRole(self):

From 263e6be5478155e7f93580e7d0580d59995d38fe Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 17:40:43 +0100
Subject: [PATCH 44/82] fix html validator test for python 3

---
 tests/test_LdapCherry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 536f80d..014a2fc 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -90,7 +90,7 @@ def htmlvalidator(page):
     f.close()
     stdout.close()
     print(out)
-    if not re.search(r'Error:.*', out) is None:
+    if not re.search(r'Error:.*', str(out)) is None:
         raise HtmlValidationFailed(out)
 
 class BadModule():

From 02357d886aef20224c91c24b6c92baa6579c6941 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 18:18:58 +0100
Subject: [PATCH 45/82] remove debug print

---
 ldapcherry/backend/backendLdap.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index cceaa96..83fb857 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -538,8 +538,6 @@ def add_to_groups(self, username, groups):
                         {attr: self._modlist(self._byte_p3(content))}
                        )
                 try:
-                    print(ldif)
-                    print(group)
                     ldap_client.modify_s(group, ldif)
                 # if already member, not a big deal, just log it and continue
                 except (ldap.TYPE_OR_VALUE_EXISTS, ldap.ALREADY_EXISTS) as e:

From a56c491ee1a59f59a52182d4c204fa4bbf4bae78 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 18:31:37 +0100
Subject: [PATCH 46/82] cleanup in html template + tidylib

* few small cleanup in html template (avoid empty tbody, put id between
quotes)
* switch to tidylib to validate the html instead of the previous hack
calling an external service (https://html5.validator.nu/)
* remove the previous validator script
* add exception for tidylib on empty <span> (these are required by
bootstrap)
---
 resources/templates/adduser.tmpl |   2 +-
 resources/templates/base.tmpl    |   2 +-
 resources/templates/index.tmpl   |   4 +-
 setup.py                         |   2 +-
 tests/html_validator.py          | 243 -------------------------------
 tests/test_LdapCherry.py         |  26 ++--
 6 files changed, 19 insertions(+), 260 deletions(-)
 delete mode 100755 tests/html_validator.py

diff --git a/resources/templates/adduser.tmpl b/resources/templates/adduser.tmpl
index 8b27f75..cd8e31f 100644
--- a/resources/templates/adduser.tmpl
+++ b/resources/templates/adduser.tmpl
@@ -6,7 +6,7 @@
         </div>
         <div class="col-md-12 column">
             <div class="well well-sm">
-              <form method='POST' autocomplete="off" action='/adduser' role="form" class="form-signin" id=form>
+              <form method='POST' autocomplete="off" action='/adduser' role="form" class="form-signin" id="form">
               <fieldset>
               <legend>Fill new user's attributes:</legend>
               ${form | n}
diff --git a/resources/templates/base.tmpl b/resources/templates/base.tmpl
index 2c25f8a..5a66230 100644
--- a/resources/templates/base.tmpl
+++ b/resources/templates/base.tmpl
@@ -70,6 +70,6 @@
         <p class="muted credit"><a href="http://ldapcherry.readthedocs.org" target="_blank">LdapCherry</a> • © 2016 • Pierre-François Carpentier • Released under the MIT License</p>
       </div>
     </div>
-</body>
 <script type="text/javascript" src="/static/js/alignforms.js"></script>
+</body>
 </html>
diff --git a/resources/templates/index.tmpl b/resources/templates/index.tmpl
index 3765855..2f1b3ef 100644
--- a/resources/templates/index.tmpl
+++ b/resources/templates/index.tmpl
@@ -11,8 +11,8 @@
               </div>
               <div class="panel-body">
 		<table id="RecordTable" class="table table-hover table-condensed">
-		<tbody>
 		% if not searchresult is None:
+		<tbody>
                      %for attr in sorted(attrs_list.keys(), key=lambda attr: attrs_list[attr]['weight']):
                      <tr>
                              % if attr in searchresult:
@@ -26,8 +26,8 @@
                              % endif
                      </tr>
                      % endfor
-		%endif
 		</tbody>
+		%endif
                 </table>
                 </div>
               </div>
diff --git a/setup.py b/setup.py
index a52e92c..55d0383 100755
--- a/setup.py
+++ b/setup.py
@@ -132,7 +132,7 @@ def get_list_files(basedir, targetdir):
     description=small_description,
     long_description=description,
     install_requires=install_requires,
-    tests_require=['pytest', 'pep8'],
+    tests_require=['pytest', 'pep8', 'pytidylib'],
     cmdclass={'test': PyTest},
     classifiers=[
         'Development Status :: 3 - Alpha',
diff --git a/tests/html_validator.py b/tests/html_validator.py
deleted file mode 100755
index 559f0ff..0000000
--- a/tests/html_validator.py
+++ /dev/null
@@ -1,243 +0,0 @@
-#!/usr/bin/python
-
-# Copyright (c) 2007-2008 Mozilla Foundation
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-# DEALINGS IN THE SOFTWARE.
-
-from __future__ import print_function, with_statement
-
-import os
-import sys
-import re
-import string
-import gzip
-
-# Several "try" blocks for python2/3 differences (@secretrobotron)
-try:
-  import httplib
-except ImportError:
-  import http.client as httplib
-
-try:
-  import urlparse
-except ImportError:
-  import urllib.parse as urlparse
-
-try:
-  from BytesIO import BytesIO
-except ImportError:
-  from io import BytesIO
-
-try:
-  maketrans = str.maketrans
-except AttributeError:
-  maketrans = string.maketrans
-
-#
-# Begin
-#
-extPat = re.compile(r'^.*\.([A-Za-z]+)$')
-extDict = {
-  'html' : 'text/html',
-  'htm' : 'text/html',
-  'xhtml' : 'application/xhtml+xml',
-  'xht' : 'application/xhtml+xml',
-  'xml' : 'application/xml',
-}
-
-forceXml = False
-forceHtml = False
-gnu = False
-errorsOnly = False
-encoding = None
-fileName = None
-contentType = None
-inputHandle = None
-service = 'https://html5.validator.nu/'
-
-argv = sys.argv[1:]
-
-#
-# Parse command line input
-#
-for arg in argv:
-  if '--help' == arg:
-    print('-h : force text/html')
-    print('-x : force application/xhtml+xml')
-    print('-g : GNU output')
-    print('-e : errors only (no info or warnings)')
-    print('--encoding=foo : declare encoding foo')
-    print('--service=url  : the address of the HTML5 validator')
-    print('One file argument allowed. Leave out to read from stdin.')
-    sys.exit(0)
-  elif arg.startswith('--encoding='):
-    encoding = arg[11:]
-  elif arg.startswith('--service='):
-    service = arg[10:]
-  elif arg.startswith('--'):
-      sys.stderr.write('Unknown argument %s.\n' % arg)
-      sys.exit(2)
-  elif arg.startswith('-'):
-    for c in arg[1:]:
-      if 'x' == c:
-        forceXml = True
-      elif 'h' == c:
-        forceHtml = True
-      elif 'g' == c:
-        gnu = True
-      elif 'e' == c:
-        errorsOnly = True
-      else:
-        sys.stderr.write('Unknown argument %s.\n' % arg)
-        sys.exit(3)
-  else:
-    if fileName:
-      sys.stderr.write('Cannot have more than one input file.\n')
-      sys.exit(1)
-    fileName = arg
-
-#
-# Ensure a maximum of one forced output type
-#
-if forceXml and forceHtml:
-  sys.stderr.write('Cannot force HTML and XHTML at the same time.\n')
-  sys.exit(2)
-
-#
-# Set contentType
-#
-if forceXml:
-  contentType = 'application/xhtml+xml'
-elif forceHtml:
-  contentType = 'text/html'
-elif fileName:
-  m = extPat.match(fileName)
-  if m:
-    ext = m.group(1)
-    ext = ext.translate(maketrans(string.ascii_uppercase, string.ascii_lowercase))
-    if ext in extDict:
-      contentType = extDict[ext]
-    else:
-      sys.stderr.write('Unable to guess Content-Type from file name. Please force the type.\n')
-      sys.exit(3)
-  else:
-    sys.stderr.write('Could not extract a filename extension. Please force the type.\n')
-    sys.exit(6)
-else:
-  sys.stderr.write('Need to force HTML or XHTML when reading from stdin.\n')
-  sys.exit(4)
-
-if encoding:
-  contentType = '%s; charset=%s' % (contentType, encoding)
-
-#
-# Read the file argument (or STDIN)
-#
-if fileName:
-  inputHandle = fileName
-else:
-  inputHandle = sys.stdin
-
-with open(inputHandle, mode='rb') as inFile:
-  data = inFile.read()
-  with BytesIO() as buf:
-    # we could use another with block here, but it requires Python 2.7+
-    zipFile = gzip.GzipFile(fileobj=buf, mode='wb')
-    zipFile.write(data)
-    zipFile.close()
-    gzippeddata = buf.getvalue()
-
-#
-# Prepare the request
-#
-url = service
-
-if gnu:
-  url = url + '?out=gnu'
-else:
-  url = url + '?out=text'
-
-if errorsOnly:
-  url = url + '&level=error'
-
-connection = None
-response = None
-status = 302
-redirectCount = 0
-
-#
-# Make the request
-#
-while status in (302,301,307) and redirectCount < 10:
-  if redirectCount > 0:
-    url = response.getheader('Location')
-  parsed = urlparse.urlsplit(url)
-
-  if redirectCount > 0:
-    connection.close() # previous connection
-    print('Redirecting to %s' % url)
-    print('Please press enter to continue or type \'stop\' followed by enter to stop.')
-    if raw_input() != '':
-      sys.exit(0)
-
-  if parsed.scheme == 'https':
-    connection = httplib.HTTPSConnection(parsed[1])
-  else:
-    connection = httplib.HTTPConnection(parsed[1])
-
-  headers = {
-    'Accept-Encoding': 'gzip',
-    'Content-Type': contentType,
-    'Content-Encoding': 'gzip',
-    'Content-Length': len(gzippeddata),
-  }
-  urlSuffix = '%s?%s' % (parsed[2], parsed[3])
-
-  connection.connect()
-  connection.request('POST', urlSuffix, body=gzippeddata, headers=headers)
-
-  response = connection.getresponse()
-  status = response.status
-
-  redirectCount += 1
-
-#
-# Handle the response
-#
-if status != 200:
-  sys.stderr.write('%s %s\n' % (status, response.reason))
-  sys.exit(5)
-
-if response.getheader('Content-Encoding', 'identity').lower() == 'gzip':
-  response = gzip.GzipFile(fileobj=BytesIO(response.read()))
-
-if fileName and gnu:
-  quotedName = '"%s"' % fileName.replace("'", '\\042')
-  for line in response.read().split('\n'):
-    if line:
-      sys.stdout.write(quotedName)
-      sys.stdout.write(line + '\n')
-else:
-  output = response.read()
-  # python2/3 difference in output's type
-  if not isinstance(output, str):
-    output = output.decode('utf-8')
-  sys.stdout.write(output)
-
-connection.close()
diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 014a2fc..ff94bda 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -20,6 +20,7 @@
 from ldapcherry.lclogging import *
 from disable import *
 import json
+from tidylib import tidy_document
 if sys.version < '3':
     from sets import Set as set
 
@@ -79,19 +80,21 @@ class HtmlValidationFailed(Exception):
     def __init__(self, out):
         self.errors = out
 
+def _is_html_error(line):
+    ret = True
+    for p in [r'.*Warning: trimming empty <span>.*']:
+        if re.match(p, line):
+            ret = False
+    return ret
+
 def htmlvalidator(page):
+    document, errors = tidy_document(page,
+        options={'numeric-entities':1})
     f = tempfile()
-    stdout = tempfile()
-    f.write(page.encode("utf-8"))
-    f.seek(0)
-    ret = subprocess.call(['./tests/html_validator.py', '-h', f.name], stdout=stdout)
-    stdout.seek(0)
-    out = stdout.read()
-    f.close()
-    stdout.close()
-    print(out)
-    if not re.search(r'Error:.*', str(out)) is None:
-        raise HtmlValidationFailed(out)
+    for line in errors.splitlines():
+        if _is_html_error(line):
+            print(line)
+            raise HtmlValidationFailed(line)
 
 class BadModule():
     pass
@@ -361,4 +364,3 @@ def testLogger(self):
         get_loglevel('alert') is logging.CRITICAL and \
         get_loglevel('emergency') is logging.CRITICAL and \
         get_loglevel('notalevel') is logging.INFO
-

From f13961790f27a2dab07a0d8da344e6c20f4803de Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 18:40:48 +0100
Subject: [PATCH 47/82] adding exception for <nav> tags in html validation

---
 tests/test_LdapCherry.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index ff94bda..7d73183 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -82,7 +82,10 @@ def __init__(self, out):
 
 def _is_html_error(line):
     ret = True
-    for p in [r'.*Warning: trimming empty <span>.*']:
+    for p in [
+                r'.*Warning: trimming empty <span>.*',
+                r'.*Error: <nav> is not recognized!.*'
+             ]:
         if re.match(p, line):
             ret = False
     return ret

From 98fca30fba88dc07860df7ae8552e857fcbb7914 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 18:44:51 +0100
Subject: [PATCH 48/82] ignoring another nav error

---
 tests/test_LdapCherry.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 7d73183..5301ffd 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -84,7 +84,8 @@ def _is_html_error(line):
     ret = True
     for p in [
                 r'.*Warning: trimming empty <span>.*',
-                r'.*Error: <nav> is not recognized!.*'
+                r'.*Error: <nav> is not recognized!.*',
+                r'.*Warning: discarding unexpected <nav>.*',
              ]:
         if re.match(p, line):
             ret = False

From 046afbbe29413378bd802f0b59f2624de0c1ad66 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 18:54:35 +0100
Subject: [PATCH 49/82] html_tidy cleanup

---
 tests/test_LdapCherry.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 5301ffd..abe6942 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -81,15 +81,15 @@ def __init__(self, out):
         self.errors = out
 
 def _is_html_error(line):
-    ret = True
     for p in [
                 r'.*Warning: trimming empty <span>.*',
                 r'.*Error: <nav> is not recognized!.*',
                 r'.*Warning: discarding unexpected <nav>.*',
+                r'.*testing: good.*',
              ]:
         if re.match(p, line):
-            ret = False
-    return ret
+            return False
+    return True
 
 def htmlvalidator(page):
     document, errors = tidy_document(page,
@@ -97,7 +97,11 @@ def htmlvalidator(page):
     f = tempfile()
     for line in errors.splitlines():
         if _is_html_error(line):
-            print(line)
+            print("################")
+            print("Blocking error: '%s'" % line)
+            print("all tidy_document errors:"
+            print(errors)
+            print("################")
             raise HtmlValidationFailed(line)
 
 class BadModule():

From abfce4803a98cc2b936fb8978f090a17702a41df Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 18:57:50 +0100
Subject: [PATCH 50/82] fix typo

---
 tests/test_LdapCherry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index abe6942..c240d8f 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -99,7 +99,7 @@ def htmlvalidator(page):
         if _is_html_error(line):
             print("################")
             print("Blocking error: '%s'" % line)
-            print("all tidy_document errors:"
+            print("all tidy_document errors:")
             print(errors)
             print("################")
             raise HtmlValidationFailed(line)

From c5536bdc56396b68728712a72d17eb16cb7c37ca Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 19:04:40 +0100
Subject: [PATCH 51/82] adding a fffew other exception in tidylib

---
 tests/test_LdapCherry.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index c240d8f..d4aa450 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -85,6 +85,14 @@ def _is_html_error(line):
                 r'.*Warning: trimming empty <span>.*',
                 r'.*Error: <nav> is not recognized!.*',
                 r'.*Warning: discarding unexpected <nav>.*',
+                r'.*Warning: discarding unexpected </nav>.*',
+                r'.*Warning: <meta> proprietary attribute "charset".*',
+                r'.*Warning: <meta> lacks "content" attribute.*',
+                r'.*Warning: <link> inserting "type" attribute.*',
+                r'.*Warning: <link> proprietary attribute.*',
+                r'.*Warning: <button> proprietary attribute.*',
+                r'.*Warning: <form> proprietary attribute.*',
+                r'.*Warning: <table> lacks "summary" attribute.*',
                 r'.*testing: good.*',
              ]:
         if re.match(p, line):

From 9d0d321e9ba1471e509d48cd54c14d99b1c4de81 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 19:09:32 +0100
Subject: [PATCH 52/82] another ignore for tidylib

---
 tests/test_LdapCherry.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index d4aa450..071a0d9 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -93,6 +93,7 @@ def _is_html_error(line):
                 r'.*Warning: <button> proprietary attribute.*',
                 r'.*Warning: <form> proprietary attribute.*',
                 r'.*Warning: <table> lacks "summary" attribute.*',
+                r'.*Warning: <script> inserting "type" attribute.*',
                 r'.*testing: good.*',
              ]:
         if re.match(p, line):

From 7a8468f8b10eea6d131789b68f5c53782633ed0d Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 19:15:31 +0100
Subject: [PATCH 53/82] adding another ignore

---
 tests/test_LdapCherry.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 071a0d9..56cb304 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -90,6 +90,7 @@ def _is_html_error(line):
                 r'.*Warning: <meta> lacks "content" attribute.*',
                 r'.*Warning: <link> inserting "type" attribute.*',
                 r'.*Warning: <link> proprietary attribute.*',
+                r'.*Warning: <input> proprietary attribute.*',
                 r'.*Warning: <button> proprietary attribute.*',
                 r'.*Warning: <form> proprietary attribute.*',
                 r'.*Warning: <table> lacks "summary" attribute.*',

From fba2d32b440d9d8bd8fc1bf930de7624eac9582a Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 19:21:41 +0100
Subject: [PATCH 54/82] another exception for todylib

---
 tests/test_LdapCherry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 56cb304..c05a0e2 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -95,7 +95,7 @@ def _is_html_error(line):
                 r'.*Warning: <form> proprietary attribute.*',
                 r'.*Warning: <table> lacks "summary" attribute.*',
                 r'.*Warning: <script> inserting "type" attribute.*',
-                r'.*testing: good.*',
+                r'.*Warning: <input> attribute "id" has invalid value "#password2".*',
              ]:
         if re.match(p, line):
             return False

From e50df5dde31ea97b2217b84aff8288ea186612f4 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 19:26:31 +0100
Subject: [PATCH 55/82] wider exception for <input> attribute "type" has
 invalid value

---
 tests/test_LdapCherry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index c05a0e2..e03c24a 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -95,7 +95,7 @@ def _is_html_error(line):
                 r'.*Warning: <form> proprietary attribute.*',
                 r'.*Warning: <table> lacks "summary" attribute.*',
                 r'.*Warning: <script> inserting "type" attribute.*',
-                r'.*Warning: <input> attribute "id" has invalid value "#password2".*',
+                r'.*Warning: <input> attribute "id" has invalid value.*',
              ]:
         if re.match(p, line):
             return False

From 4a8aa1c655b842a77ac08c5d94610e2bed68dba8 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 19:32:50 +0100
Subject: [PATCH 56/82] another exception

---
 tests/test_LdapCherry.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index e03c24a..2fc18c9 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -96,6 +96,7 @@ def _is_html_error(line):
                 r'.*Warning: <table> lacks "summary" attribute.*',
                 r'.*Warning: <script> inserting "type" attribute.*',
                 r'.*Warning: <input> attribute "id" has invalid value.*',
+                r'.*Warning: <a> proprietary attribute.*',
              ]:
         if re.match(p, line):
             return False

From 7390c931b90d72d30aa1dca54f1194ce1497164a Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 19:37:52 +0100
Subject: [PATCH 57/82] another exception

---
 tests/test_LdapCherry.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 2fc18c9..1c5066a 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -97,6 +97,7 @@ def _is_html_error(line):
                 r'.*Warning: <script> inserting "type" attribute.*',
                 r'.*Warning: <input> attribute "id" has invalid value.*',
                 r'.*Warning: <a> proprietary attribute.*',
+                r'.*Warning: <input> attribute "type" has invalid value.*',
              ]:
         if re.match(p, line):
             return False

From f824790849dc015dc1c005cff46b27a34e6e7e66 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 19:42:48 +0100
Subject: [PATCH 58/82] another exception

---
 tests/test_LdapCherry.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 1c5066a..69445bc 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -98,6 +98,7 @@ def _is_html_error(line):
                 r'.*Warning: <input> attribute "id" has invalid value.*',
                 r'.*Warning: <a> proprietary attribute.*',
                 r'.*Warning: <input> attribute "type" has invalid value.*',
+                r'.*Warning: <span> proprietary attribute.*',
              ]:
         if re.match(p, line):
             return False

From 0793361d90692cdfec5c8ca38eec101761646a6a Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 20:19:57 +0100
Subject: [PATCH 59/82] switch to "stable" in setup.py troves

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 55d0383..b703e8b 100755
--- a/setup.py
+++ b/setup.py
@@ -135,7 +135,7 @@ def get_list_files(basedir, targetdir):
     tests_require=['pytest', 'pep8', 'pytidylib'],
     cmdclass={'test': PyTest},
     classifiers=[
-        'Development Status :: 3 - Alpha',
+        'Development Status :: 5 - Production/Stable',
         'Environment :: Web Environment',
         'Framework :: CherryPy',
         'Intended Audience :: System Administrators',

From abf145427814f040c88ef0e104e441ecb0a29eb2 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 20:21:29 +0100
Subject: [PATCH 60/82] changelog+version bump

---
 ChangeLog.rst         | 8 ++++++++
 ldapcherry/version.py | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 12f087d..3ea6e85 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,10 +1,18 @@
 Dev
 ***
 
+Version 1.0.0
+*************
+
 * [sec ] fix XSS injection in the url redirect in the login page (thanks to jthiltges)
 * [fix ] fix configuration consistency check for attribute file (error if a given backend is not declared in main .ini file but in attributes)
+* [fix ] remove a few deprecation warnings
+* [fix ] fix potential issue with group names containing non-ascii characters
+* [feat] support for python 3
+* [feat] support for python-ldap 3.X.X
 * [impr] better log error message if inconsistency between role, attribute and main .ini file for backends
 * [impr] more systematic use of html and url escaping in the html rendering to prevent against content injection (thanks to jthiltges)
+* [impr] more testing for various versions of python and python-ldap
 
 Version 0.5.2
 *************
diff --git a/ldapcherry/version.py b/ldapcherry/version.py
index 75f8af2..1547dbb 100644
--- a/ldapcherry/version.py
+++ b/ldapcherry/version.py
@@ -5,4 +5,4 @@
 # ldapCherry
 # Copyright (c) 2014 Carpentier Pierre-Francois
 
-version = '0.5.2'
+version = '1.0.0'

From 932e7a8b40c2e2160ca7484da2ad32757dd1d4c0 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 20:24:32 +0100
Subject: [PATCH 61/82] adding (mostly) working configuration example

---
 tests/test_env/etc/ldapcherry/attributes.yml | 132 +++++++++++++++++++
 tests/test_env/etc/ldapcherry/ldapcherry.ini | 125 ++++++++++++++++++
 tests/test_env/etc/ldapcherry/roles.yml      |  36 +++++
 3 files changed, 293 insertions(+)
 create mode 100644 tests/test_env/etc/ldapcherry/attributes.yml
 create mode 100644 tests/test_env/etc/ldapcherry/ldapcherry.ini
 create mode 100644 tests/test_env/etc/ldapcherry/roles.yml

diff --git a/tests/test_env/etc/ldapcherry/attributes.yml b/tests/test_env/etc/ldapcherry/attributes.yml
new file mode 100644
index 0000000..47dd9a4
--- /dev/null
+++ b/tests/test_env/etc/ldapcherry/attributes.yml
@@ -0,0 +1,132 @@
+cn:
+    description: "First Name and Display Name"
+    display_name: "Display Name"
+    type: string
+    weight: 30
+    autofill: 
+        function: lcDisplayName
+        args:
+            - $first-name
+            - $name
+    backends:
+        ldap: cn
+        ad: cn
+first-name:
+    description: "First name of the user"
+    display_name: "First Name"
+    search_displayed: True
+    type: string
+    weight: 20
+    backends:
+        ldap: givenName
+        ad: givenName
+name:
+    description: "Family name of the user"
+    display_name: "Name"
+    search_displayed: True
+    weight: 10
+    type: string
+    backends:
+        ldap: sn
+        ad: sn
+email:
+    description: "Email of the user"
+    display_name: "Email"
+    search_displayed: True
+    type: email
+    weight: 40
+    autofill: 
+        function: lcMail
+        args:
+            - $first-name
+            - $name
+            - '@example.com'
+    backends:
+        ldap: mail
+        ad: mail
+uid:
+    description: "UID of the user"
+    display_name: "UID"
+    search_displayed: True
+    key: True
+    type: string
+    weight: 50
+    autofill: 
+        function: lcUid
+        args:
+            - $first-name
+            - $name
+            - '10000'
+            - '40000'
+    backends:
+        ldap: uid
+        ad: sAMAccountName
+uidNumber:
+    description: "User ID Number of the user"
+    display_name: "UID Number"
+    weight: 60
+    type: int
+    autofill: 
+        function: lcUidNumber
+        args:
+            - $first-name
+            - $name
+            - '10000'
+            - '40000'
+    backends:
+        ldap: uidNumber
+        ad: uidNumber
+gidNumber:
+    description: "Group ID Number of the user"
+    display_name: "GID Number"
+    weight: 70
+    type: int
+    default: '10000'
+    backends:
+        ldap: gidNumber
+        ad: gidNumber
+shell:
+    description: "Shell of the user"
+    display_name: "Shell"
+    weight: 80
+    self: True
+    type: stringlist
+    values:
+        - /bin/bash
+        - /bin/zsh
+        - /bin/sh
+    backends:
+        ldap: loginShell
+        ad: loginShell 
+home:
+    description: "Home user path"
+    display_name: "Home"
+    weight: 90
+    type: string
+    autofill: 
+        function: lcHomeDir
+        args:
+            - $first-name
+            - $name
+            - /home/
+    backends:
+        ldap: homeDirectory
+        ad: homeDirectory
+password:
+    description: "Password of the user"
+    display_name: "Password"
+    weight: 31
+    self: True
+    type: password
+    backends:
+        ldap: userPassword
+        ad: unicodePwd
+
+#logscript:
+#    description: "Windows login script"
+#    display_name: "Login script"
+#    weight: 100
+#    type: fix
+#    value: login1.bat
+#    backends:
+#        ad: scriptPath
diff --git a/tests/test_env/etc/ldapcherry/ldapcherry.ini b/tests/test_env/etc/ldapcherry/ldapcherry.ini
new file mode 100644
index 0000000..d41f53d
--- /dev/null
+++ b/tests/test_env/etc/ldapcherry/ldapcherry.ini
@@ -0,0 +1,125 @@
+# global parameters
+[global]
+
+# listing interface
+server.socket_host = '127.0.0.1'
+# port
+server.socket_port = 8080
+# number of threads
+server.thread_pool = 8
+#don't show traceback on error
+request.show_tracebacks = False
+
+# log configuration
+# /!\ you can't have multiple log handlers
+#####################################
+#   configuration to log in files   #
+#####################################
+## logger 'file' for access log 
+#log.access_handler = 'file'
+## logger syslog for error and ldapcherry log 
+#log.error_handler = 'file'
+## access log file
+#log.access_file = '/tmp/ldapcherry_access.log'
+## error and ldapcherry log file
+#log.error_file = '/tmp/ldapcherry_error.log'
+
+#####################################
+#  configuration to log in syslog   #
+#####################################
+# logger syslog for access log 
+#log.access_handler = 'syslog'
+## logger syslog for error and ldapcherry log 
+log.error_handler = 'syslog'
+
+#####################################
+#  configuration to not log at all  #
+#####################################
+# logger none for access log 
+log.access_handler = 'syslog'
+# logger none for error and ldapcherry log 
+#log.error_handler = 'none'
+
+# log level
+log.level = 'debug'
+
+# session configuration
+# activate session
+tools.sessions.on = True
+# session timeout
+tools.sessions.timeout = 10
+# file session storage(to use if multiple processes, 
+# default is in RAM and per process)
+#tools.sessions.storage_type = "file"
+# session 
+#tools.sessions.storage_path = "/var/lib/ldapcherry/sessions"
+
+[attributes]
+
+# file discribing form content
+attributes.file = '/etc/ldapcherry/attributes.yml'
+
+[roles]
+
+# file listing roles
+roles.file = '/etc/ldapcherry/roles.yml'
+
+[backends]
+
+ldap.module = 'ldapcherry.backend.backendLdap'
+ldap.groupdn = 'ou=Group,dc=example,dc=org'
+ldap.userdn = 'ou=people,dc=example,dc=org'
+ldap.binddn = 'cn=dnscherry,dc=example,dc=org'
+ldap.password = 'password'
+ldap.uri = 'ldap://ldap.ldapcherry.org:390'
+ldap.ca = '/etc/dnscherry/TEST-cacert.pem'
+ldap.starttls = 'off'
+ldap.checkcert = 'off'
+ldap.user_filter_tmpl = '(uid=%(username)s)'
+ldap.group_filter_tmpl = '(member=%(userdn)s)'
+ldap.search_filter_tmpl = '(|(uid=%(searchstring)s*)(sn=%(searchstring)s*))'
+ldap.group_attr.member = "%(dn)s"
+
+#ldap.objectclasses = 'top, person, organizationalPerson, user'
+ldap.objectclasses = 'top, person, posixAccount, inetOrgPerson'
+ldap.dn_user_attr = 'uid'
+ldap.timeout = 1
+
+ad.module = 'ldapcherry.backend.backendAD'
+ad.domain = 'dc.ldapcherry.org'
+ad.login  = 'administrator'
+ad.password = 'qwertyP455'
+ad.uri = 'ldaps://ldap.ldapcherry.org:636'
+ad.checkcert = 'off'
+
+# authentification parameters
+[auth]
+
+# Auth mode
+# * and: user must authenticate on all backends
+# * or:  user must authenticate on one of the backend
+# * none: disable authentification
+# * custom: custom authentification module (need auth.module param)
+auth.mode = 'none'
+
+# custom auth module to load
+#auth.module = 'ldapcherry.auth.modNone'
+
+[ppolicy]
+
+# password policy module
+ppolicy.module = 'ldapcherry.ppolicy.simple'
+
+# parameters of the module
+min_length = 2
+min_upper = 0
+min_digit = 0
+
+# resources parameters
+[resources]
+# templates directory
+templates.dir = './resources/templates/'
+
+[/static]
+tools.staticdir.on = True
+tools.staticdir.dir = './resources/static/'
diff --git a/tests/test_env/etc/ldapcherry/roles.yml b/tests/test_env/etc/ldapcherry/roles.yml
new file mode 100644
index 0000000..114359e
--- /dev/null
+++ b/tests/test_env/etc/ldapcherry/roles.yml
@@ -0,0 +1,36 @@
+admin-lv3:
+    display_name: Administrators Level 3
+    description: description
+    backends_groups:
+        ldap:
+            - cn=dns admins,ou=Group,dc=example,dc=org
+            - cn=nagios admins,ou=Group,dc=example,dc=org
+            - cn=puppet admins,ou=Group,dc=example,dc=org
+            - cn=users,ou=Group,dc=example,dc=org
+        ad:
+            - Administrators
+            - Domain Controllers 
+
+admin-lv2:
+    display_name: Administrators Level 2
+    description: description
+    LC_admins: True
+    backends_groups:
+        ldap:
+            - cn=nagios admins,ou=Group,dc=example,dc=org
+            - cn=users,ou=Group,dc=example,dc=org
+
+developpers:
+    display_name: Developpers
+    description: description
+    backends_groups:
+        ldap:
+            - cn=developpers,ou=Group,dc=example,dc=org
+            - cn=users,ou=Group,dc=example,dc=org
+
+users:
+    display_name: Simple Users
+    description: description
+    backends_groups:
+        ldap:
+            - cn=users,ou=Group,dc=example,dc=org

From b68214022c403faa8f09b9bab47f6a9861a41a30 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 20:47:34 +0100
Subject: [PATCH 62/82] fix error handling when adding user that already exists

---
 ldapcherry/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index 0acb0e0..4e61fc9 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -678,6 +678,7 @@ def _adduser(self, params):
                 self._add_notification(
                     'User already exists in backend "' + b + '"'
                     )
+                return
         if not added:
             raise e
 

From 57bcaaed66cc8469bf60f48445232a848ea9adb1 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sat, 9 Feb 2019 20:49:23 +0100
Subject: [PATCH 63/82] changelog and version bump

---
 ChangeLog.rst         | 5 +++++
 ldapcherry/version.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 3ea6e85..2e5b4b0 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,11 @@
 Dev
 ***
 
+Version 1.0.1
+*************
+
+* [fix ] fix error handling when adding user that already exists
+
 Version 1.0.0
 *************
 
diff --git a/ldapcherry/version.py b/ldapcherry/version.py
index 1547dbb..c2fbedb 100644
--- a/ldapcherry/version.py
+++ b/ldapcherry/version.py
@@ -5,4 +5,4 @@
 # ldapCherry
 # Copyright (c) 2014 Carpentier Pierre-Francois
 
-version = '1.0.0'
+version = '1.0.1'

From e6bcf9d97d787893906b957927602f3cfb862a4a Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sun, 10 Feb 2019 18:12:45 +0100
Subject: [PATCH 64/82] adding the possibility to log to stdout

---
 conf/ldapcherry.ini      |  8 ++++++++
 docs/deploy.rst          |  6 +++---
 ldapcherry/__init__.py   | 18 ++++++++++++++++++
 tests/test_LdapCherry.py |  2 +-
 4 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/conf/ldapcherry.ini b/conf/ldapcherry.ini
index 12c12fa..d6f4c6f 100644
--- a/conf/ldapcherry.ini
+++ b/conf/ldapcherry.ini
@@ -28,6 +28,14 @@ request.show_tracebacks = False
 ## error and ldapcherry log file
 #log.error_file = '/tmp/ldapcherry_error.log'
 
+#####################################
+#  configuration to log to stdout   #
+#####################################
+## logger stdout for access log
+#log.access_handler = 'stdout'
+## logger stdout for error and ldapcherry log
+#log.error_handler = 'stdout'
+
 #####################################
 #  configuration to log in syslog   #
 #####################################
diff --git a/docs/deploy.rst b/docs/deploy.rst
index 1c8a4c0..1a541ee 100644
--- a/docs/deploy.rst
+++ b/docs/deploy.rst
@@ -445,16 +445,16 @@ Logging
 
 LdapCherry has two loggers, one for errors and applicative actions (login, del/add, logout...) and one for access logs.
 
-Each logger can be configured to log to syslog, file or be disabled. 
+Each logger can be configured to log to **syslog**, **file**, **stdout** or be disabled.
 
 Logging parameters:
 
 +--------------------+---------+---------------------------------+-------------------------------------------------+----------------------------------------+
 |      Parameter     | Section |           Description           |                      Values                     |                 Comment                |
 +====================+=========+=================================+=================================================+========================================+
-| log.access_handler |  global |    Logger type for access log   |            'syslog', 'file', 'none'             |                                        |
+| log.access_handler |  global |    Logger type for access log   |      'syslog', 'file', 'stdout', 'none'         |                                        |
 +--------------------+---------+---------------------------------+-------------------------------------------------+----------------------------------------+
-|  log.error_handler |  global | Logger type for applicative log |             'syslog', 'file', 'none'            |                                        |
+|  log.error_handler |  global | Logger type for applicative log |      'syslog', 'file', 'stdout', 'none'         |                                        |
 +--------------------+---------+---------------------------------+-------------------------------------------------+----------------------------------------+
 |   log.access_file  |  global |     log file for access log     |                 path to log file                | only used if log.access_handler='file' |
 +--------------------+---------+---------------------------------+-------------------------------------------------+----------------------------------------+
diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index 4e61fc9..963b051 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -285,6 +285,15 @@ def _set_access_log(self, config, level):
             handler.setFormatter(syslog_formatter)
             cherrypy.log.access_log.addHandler(handler)
 
+        # if stdout, open a logger on stdout
+        elif access_handler == 'stdout':
+            cherrypy.log.access_log.handlers = []
+            handler = logging.StreamHandler(sys.stdout)
+            formatter = logging.Formatter(
+                'ldapcherry.access - %(levelname)s - %(message)s'
+            )
+            handler.setFormatter(formatter)
+            cherrypy.log.access_log.addHandler(handler)
         # if file, we keep the default
         elif access_handler == 'file':
             pass
@@ -330,6 +339,15 @@ def _set_error_log(self, config, level, debug=False):
             handler.setFormatter(syslog_formatter)
             cherrypy.log.error_log.addHandler(handler)
 
+        # if stdout, open a logger on stdout
+        elif error_handler == 'stdout':
+            cherrypy.log.error_log.handlers = []
+            handler = logging.StreamHandler(sys.stdout)
+            formatter = logging.Formatter(
+                'ldapcherry.app    - %(levelname)s - %(message)s'
+            )
+            handler.setFormatter(formatter)
+            cherrypy.log.error_log.addHandler(handler)
         # if file, we keep the default
         elif error_handler == 'file':
             pass
diff --git a/tests/test_LdapCherry.py b/tests/test_LdapCherry.py
index 69445bc..d2a598f 100644
--- a/tests/test_LdapCherry.py
+++ b/tests/test_LdapCherry.py
@@ -152,7 +152,7 @@ def testInitgBackendModuleFail(self):
     def testLog(self):
         app = LdapCherry()
         cfg = { 'global' : {}}
-        for t in ['none', 'file', 'syslog']:
+        for t in ['none', 'file', 'syslog', 'stdout']:
             cfg['global']['log.access_handler']=t
             cfg['global']['log.error_handler']=t
             app._set_access_log(cfg, logging.DEBUG)

From df2746b9969aa2577092d4dd1fa0d87f36bfffd4 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sun, 10 Feb 2019 18:15:07 +0100
Subject: [PATCH 65/82] version bump + changelog

---
 ChangeLog.rst         | 5 +++++
 ldapcherry/version.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 2e5b4b0..b16233d 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,11 @@
 Dev
 ***
 
+Version 1.1.0
+*************
+
+* [feat] add stdout as a valid log method (useful when running with docker)
+
 Version 1.0.1
 *************
 
diff --git a/ldapcherry/version.py b/ldapcherry/version.py
index c2fbedb..8925fda 100644
--- a/ldapcherry/version.py
+++ b/ldapcherry/version.py
@@ -5,4 +5,4 @@
 # ldapCherry
 # Copyright (c) 2014 Carpentier Pierre-Francois
 
-version = '1.0.1'
+version = '1.1.0'

From 0cf548378576a7acf253d75c7efeb26b1cc61c49 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Sun, 10 Feb 2019 18:19:55 +0100
Subject: [PATCH 66/82] add warning in documentation for log level 'debug'

---
 docs/deploy.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/deploy.rst b/docs/deploy.rst
index 1a541ee..2475b41 100644
--- a/docs/deploy.rst
+++ b/docs/deploy.rst
@@ -477,6 +477,14 @@ Example:
     log.level = 'info'
 
 
+.. warning::
+
+   'debug' should not be used in production.
+
+   It tends to log a lot.
+   More significantly can represent a security issue,
+   as things like passwords will be logged 'clear text'.
+
 Custom javascript
 ~~~~~~~~~~~~~~~~~
 

From bbafafae60f4573364192201dc64d218034487c1 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Tue, 12 Feb 2019 21:18:45 +0100
Subject: [PATCH 67/82] remove the double escaping.

Now the escaping is done by in the templates.
We need to remove the previous escaping done by hand in the code.
Otherwise, we end-up with double escaping and funky displaying of
fields.
---
 ldapcherry/__init__.py | 47 ++++++------------------------------------
 1 file changed, 6 insertions(+), 41 deletions(-)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index 963b051..72fbc2e 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -34,10 +34,8 @@
 if sys.version < '3':
     from sets import Set as set
     from urllib import quote_plus
-    from cgi import escape as html_escape
 else:
     from urllib.parse import quote_plus
-    from html import escape as html_escape
 
 SESSION_KEY = '_cp_username'
 
@@ -62,36 +60,6 @@ def _handle_exception(self, e):
             traceback=True
             )
 
-    def _escape_list(self, data):
-        ret = []
-        for i in data:
-            ret.append(html_escape(i, True))
-        return ret
-
-    def _escape_dict(self, data):
-        for d in data:
-            if isinstance(data[d], list):
-                data[d] = self._escape_list(data[d])
-            elif isinstance(data[d], dict):
-                data[d] = self._escape_dict(data[d])
-            elif isinstance(data[d], set):
-                data[d] = set(self._escape_list(data[d]))
-            else:
-                data[d] = html_escape(data[d], True)
-        return data
-
-    def _escape(self, data, dtype):
-        if data is None:
-            return None
-        elif dtype == 'search_list':
-            for d in data:
-                data[d] = self._escape_dict(data[d])
-        elif dtype == 'attr_list':
-            data = self._escape_dict(data)
-        elif dtype == 'lonely_groups':
-            data = self._escape_dict(data)
-        return data
-
     def _get_param(self, section, key, config, default=None):
         """ Get configuration parameter "key" from config
         @str section: the section of the config file
@@ -995,7 +963,7 @@ def index(self):
         return self.temp['index.tmpl'].render(
             is_admin=is_admin,
             attrs_list=attrs_list,
-            searchresult=self._escape(user_attrs, 'attr_list'),
+            searchresult=user_attrs,
             notifications=self._empty_notification(),
             )
 
@@ -1011,7 +979,7 @@ def searchuser(self, searchstring=None):
             res = None
         attrs_list = self.attributes.get_search_attributes()
         return self.temp['searchuser.tmpl'].render(
-            searchresult=self._escape(res, 'search_list'),
+            searchresult=res,
             attrs_list=attrs_list,
             is_admin=is_admin,
             custom_js=self.custom_js,
@@ -1048,7 +1016,7 @@ def searchadmin(self, searchstring=None):
             res = None
         attrs_list = self.attributes.get_search_attributes()
         return self.temp['searchadmin.tmpl'].render(
-            searchresult=self._escape(res, 'search_list'),
+            searchresult=res,
             attrs_list=attrs_list,
             is_admin=is_admin,
             custom_js=self.custom_js,
@@ -1169,7 +1137,7 @@ def modify(self, user=None, **params):
         try:
             form = self.temp['form.tmpl'].render(
                 attributes=self.attributes.attributes,
-                values=self._escape(user_attrs, 'attr_list'),
+                values=user_attrs,
                 modify=True,
                 keyattr=key,
                 autofill=False
@@ -1187,10 +1155,7 @@ def modify(self, user=None, **params):
                 form=form,
                 roles=roles,
                 is_admin=is_admin,
-                standalone_groups=self._escape(
-                    standalone_groups,
-                    'lonely_groups'
-                    ),
+                standalone_groups=standalone_groups,
                 backends_display_names=self.backends_display_names,
                 custom_js=self.custom_js,
                 notifications=self._empty_notification(),
@@ -1245,7 +1210,7 @@ def selfmodify(self, **params):
 
             form = self.temp['form.tmpl'].render(
                 attributes=self.attributes.get_selfattributes(),
-                values=self._escape(user_attrs, 'attr_list'),
+                values=user_attrs,
                 modify=True,
                 autofill=False
                 )

From 799ca2403f480d59c3a628dfb152c6292293409e Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Tue, 12 Feb 2019 21:24:24 +0100
Subject: [PATCH 68/82] fix the urls for modify and delete

The id of the user is passed through the querystring in this page.
But the id was not properly escaped to be included as a querystring
parameter leading to weird issues like.
---
 resources/templates/searchadmin.tmpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/resources/templates/searchadmin.tmpl b/resources/templates/searchadmin.tmpl
index 7b39ab2..e682166 100644
--- a/resources/templates/searchadmin.tmpl
+++ b/resources/templates/searchadmin.tmpl
@@ -52,10 +52,10 @@
                                 </td>
                             % endfor
                                 <td>
-                                    <a href="/modify?user=${user}" class="btn btn-xs blue pad" ><span class="glyphicon glyphicon-cog"></span> Modify</a>
+                                    <a href="/modify?user=${user | n,u}" class="btn btn-xs blue pad" ><span class="glyphicon glyphicon-cog"></span> Modify</a>
                                 </td>
                                 <td>
-                                    <a href="/delete?user=${user}" data-toggle='confirmation-delete' class="btn btn-xs red pad"><span class="glyphicon glyphicon-remove-sign"></span> Delete</a>
+                                    <a href="/delete?user=${user | n,u}" data-toggle='confirmation-delete' class="btn btn-xs red pad"><span class="glyphicon glyphicon-remove-sign"></span> Delete</a>
                                 </td>
                             </tr>
                             % endfor

From 73c02ccff4158ecc0f01a7ef49b5b701663f4673 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Tue, 12 Feb 2019 22:36:16 +0100
Subject: [PATCH 69/82] disable default logger if running in debug mode (-D)

Debug mode can lead to log sensitive data (like password changes).
Having -D to log exclusively to stderr is a safer option.
---
 ldapcherry/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index 72fbc2e..6c1b277 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -330,6 +330,7 @@ def _set_error_log(self, config, level, debug=False):
         cherrypy.log.error_log.setLevel(level)
 
         if debug:
+            cherrypy.log.error_log.handlers = []
             handler = logging.StreamHandler(sys.stderr)
             handler.setLevel(logging.DEBUG)
             cherrypy.log.error_log.addHandler(handler)

From d690bbdc4137441220ca713daba1a78d6c4ea232 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Tue, 12 Feb 2019 22:40:42 +0100
Subject: [PATCH 70/82] passing the correct logger to the backend

previously, the default logger was passed, this logger was using the
default configuration and log level, not honoring log level in
particular.
As a consequence, it was impossible to get debug logs from the backend.
This is now working as expected.
---
 ldapcherry/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index 6c1b277..a948574 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -161,7 +161,7 @@ def _init_backends(self, config):
                 key = self.attributes.get_backend_key(backend)
                 self.backends[backend] = bc.Backend(
                     params,
-                    cherrypy.log,
+                    cherrypy.log.error,
                     backend,
                     attrslist,
                     key,

From 7ac7118c9a10e3f0cc5021bfbc3050051b368380 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Tue, 12 Feb 2019 22:43:03 +0100
Subject: [PATCH 71/82] adding a debug log to help figure out issues with
 filters.

---
 ldapcherry/backend/backendLdap.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index 83fb857..fb3529a 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -261,6 +261,16 @@ def _search(self, searchfilter, attrs, basedn):
         else:
             attrlist = None
 
+        self._logger(
+            severity=logging.DEBUG,
+            msg="%(backend)s: executing search "
+                "with filter '%(filter)s' in DN '%(dn)s'" % {
+                    'backend': self.backend_name,
+                    'dn': basedn,
+                    'filter': searchfilter
+                }
+        )
+
         # bind and search the ldap
         ldap_client = self._bind()
         try:

From d831b0929350a8667843e6061ae1fc1ade87d995 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Tue, 12 Feb 2019 23:06:42 +0100
Subject: [PATCH 72/82] improve documentation

* improve documentation for key: True flag in attributes.yml
* improve documentation for the ldap filters and their templating
* improve comment in the .ini file
---
 conf/ldapcherry.ini | 10 +++++++++-
 docs/backends.rst   |  8 ++++++--
 docs/deploy.rst     |  7 ++++++-
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/conf/ldapcherry.ini b/conf/ldapcherry.ini
index d6f4c6f..dfbe87a 100644
--- a/conf/ldapcherry.ini
+++ b/conf/ldapcherry.ini
@@ -106,16 +106,24 @@ ldap.timeout = 1
 ldap.groupdn = 'ou=group,dc=example,dc=org'
 # users dn
 ldap.userdn = 'ou=people,dc=example,dc=org'
-# ldapsearch filter to get a user
+
+# ldapsearch filter to get a specific user
+# %(username)s is content of the attribute marked 'key: True' in the attributes.file conf
 ldap.user_filter_tmpl = '(uid=%(username)s)'
 # ldapsearch filter to get groups of a user
+# %(username)s is content of the attribute marked 'key: True' in the attributes.file conf
 ldap.group_filter_tmpl = '(member=uid=%(username)s,ou=People,dc=example,dc=org)'
 # filter to search users
+# %(searchstring)s is content passed through the search box
 ldap.search_filter_tmpl = '(|(uid=%(searchstring)s*)(sn=%(searchstring)s*))'
 
 # ldap group attributes and how to fill them
+# 'member' is the name of the attribute
+# for the template, any of the user's ldap attributes can be user
 ldap.group_attr.member = "%(dn)s"
+# same with memverUid and the uid user's attribute
 #ldap.group_attr.memberUid = "%(uid)s"
+
 # object classes of a user entry
 ldap.objectclasses = 'top, person, posixAccount, inetOrgPerson'
 # dn entry attribute for an ldap user
diff --git a/docs/backends.rst b/docs/backends.rst
index 878fa7a..8b23a8b 100644
--- a/docs/backends.rst
+++ b/docs/backends.rst
@@ -73,7 +73,10 @@ The ldap backend exposes the following parameters:
 | userdn                   | backends | The ldap dn where users are        | ldap dn                  |                                            |
 +--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
 | user_filter_tmpl         | backends | The search filter template         | ldap search filter       | The user identifier is passed through      |
-|                          |          | to recover a given user            | template                 | the **username** variable (*%(username)s*).|
+|                          |          | to recover a given user            | template                 | the **username** variable (*%(username)s*) |
+|                          |          |                                    |                          |                                            |
+|                          |          |                                    |                          | **username** is the attribute marked by    |
+|                          |          |                                    |                          | **key: True** in the **attribute.yml** file|
 +--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
 | group_filter_tmpl        | backends | The search filter template to      | ldap search filter       | The following variables are usable:        |
 |                          |          | recover the groups of a given user | template                 | * **username**: the user key attribute     |
@@ -83,7 +86,8 @@ The ldap backend exposes the following parameters:
 |                          |          |                                    |                          |   in groups dn entries                     |
 |                          |          |                                    |                          | * every user attributes are exposed        |
 |                          |          |                                    |                          |   in the template                          |
-|                          |          |                                    |                          | * multiple attributes can be set           |
+|                          |          |                                    |                          | * multiple <memver attr> attributes        |
+|                          |          |                                    |                          |   can be set                               |
 +--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
 | objectclasses            | backends | list of object classes for users   | comma separated list     |                                            |
 +--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
diff --git a/docs/deploy.rst b/docs/deploy.rst
index 2475b41..5859d0c 100644
--- a/docs/deploy.rst
+++ b/docs/deploy.rst
@@ -112,7 +112,12 @@ If **type** is set to **stringlist** the parameter **values** must be filled wit
 Key attribute:
 ^^^^^^^^^^^^^^
 
-One attribute must be used as a unique key across all backends:
+One attribute must be used as a unique key across all backends.
+
+It acts as a reconciliation key.
+
+It also marks which attribute must be used within ldapcherry (ex: querysting parameter in links)
+to point to one given user.
 
 To set the key attribute, you must set **key** to **True** on this attribute.
 

From 882a303474eea610c83cbed5f0d86fc370e401be Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Tue, 12 Feb 2019 23:27:30 +0100
Subject: [PATCH 73/82] fix crash due to encoding in python 2

---
 ldapcherry/backend/backendLdap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ldapcherry/backend/backendLdap.py b/ldapcherry/backend/backendLdap.py
index fb3529a..d6215cc 100644
--- a/ldapcherry/backend/backendLdap.py
+++ b/ldapcherry/backend/backendLdap.py
@@ -267,7 +267,7 @@ def _search(self, searchfilter, attrs, basedn):
                 "with filter '%(filter)s' in DN '%(dn)s'" % {
                     'backend': self.backend_name,
                     'dn': basedn,
-                    'filter': searchfilter
+                    'filter': self._uni(searchfilter)
                 }
         )
 

From dc60300a29dbeca629262bbd864a1c6512a5c223 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Tue, 12 Feb 2019 23:28:41 +0100
Subject: [PATCH 74/82] version + changelog

---
 ChangeLog.rst         | 10 ++++++++++
 ldapcherry/version.py |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index b16233d..883f28a 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,16 @@
 Dev
 ***
 
+Version 1.1.1
+*************
+
+* [fix ] fix double escaping issues introduced in 1.0.0
+* [fix ] fix missing url escaping in links with querystring parameters (delete and modify page mostly)
+* [fix ] fix log level not being honored in the backends
+* [impr] clarify the role of 'key: True' of attributes.yml in the documentation
+* [impr] add a few more comments in the .ini file to explain better the \*_filter_tmpl and group_attr parameters
+* [impr] add debug log to help debug ldap search filters
+
 Version 1.1.0
 *************
 
diff --git a/ldapcherry/version.py b/ldapcherry/version.py
index 8925fda..4ce55d1 100644
--- a/ldapcherry/version.py
+++ b/ldapcherry/version.py
@@ -5,4 +5,4 @@
 # ldapCherry
 # Copyright (c) 2014 Carpentier Pierre-Francois
 
-version = '1.1.0'
+version = '1.1.1'

From 30c28c5febd819c78767a6aceeb75ac273ad1c2b Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Tue, 12 Feb 2019 23:28:52 +0100
Subject: [PATCH 75/82] slightly more robust unit tests

pre-cleanup to avoid contamination if other tests fail
---
 tests/test_BackendAD.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/test_BackendAD.py b/tests/test_BackendAD.py
index 416dde6..26ab8e5 100644
--- a/tests/test_BackendAD.py
+++ b/tests/test_BackendAD.py
@@ -119,10 +119,15 @@ def testGetGroups(self):
     @travis_disabled
     def testSearchUser(self):
         inv = Backend(cfg, cherrypy.log, u'test☭', attr, 'sAMAccountName')
+        try:
+            inv.del_user(u'☭default_user')
+        except: pass
+        try:
+            inv.del_user(u'☭default_user2')
+        except: pass
         try:
             inv.add_user(default_user.copy())
-        except:
-            pass
+        except: pass
         inv.add_user(default_user2.copy())
         ret = inv.search(u'test☭')
         expected = [u'☭default_user', u'☭default_user2']

From 96acda7aa68deadcdd6d84b0dc14ddc536b2ac9f Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 13 Feb 2019 09:19:08 +0100
Subject: [PATCH 76/82] fix formatting

---
 docs/backends.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/backends.rst b/docs/backends.rst
index 8b23a8b..9eb30d3 100644
--- a/docs/backends.rst
+++ b/docs/backends.rst
@@ -79,8 +79,9 @@ The ldap backend exposes the following parameters:
 |                          |          |                                    |                          | **key: True** in the **attribute.yml** file|
 +--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
 | group_filter_tmpl        | backends | The search filter template to      | ldap search filter       | The following variables are usable:        |
-|                          |          | recover the groups of a given user | template                 | * **username**: the user key attribute     |
-|                          |          |                                    |                          | * **userdn**: the user ldap dn             |
+|                          |          | recover the groups of a given user | template                 |                                            |
+|                          |          | recover the groups of a given user | template                 | * **username**: the user's key attribute   |
+|                          |          |                                    |                          | * **userdn**: the user's ldap dn           |
 +--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
 | group_attr.<member attr> | backends | Member attribute template value    | template                 | * <member attr> is the member attribute    |
 |                          |          |                                    |                          |   in groups dn entries                     |

From 5ee8a7404022c33a9a8b3eea9d670fd45eea8646 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 13 Feb 2019 09:41:17 +0100
Subject: [PATCH 77/82] update documentation

---
 docs/backends.rst | 186 +++++++++++++++++++++++++---------------------
 1 file changed, 100 insertions(+), 86 deletions(-)

diff --git a/docs/backends.rst b/docs/backends.rst
index 9eb30d3..ec5994b 100644
--- a/docs/backends.rst
+++ b/docs/backends.rst
@@ -47,53 +47,55 @@ Configuration
 
 The ldap backend exposes the following parameters:
 
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-|      Parameter           | Section  |            Description             |           Values         |                Comment                     |
-+==========================+==========+====================================+==========================+============================================+
-| uri                      | backends | The ldap uri to access             | ldap uri                 | * use ldap:// for clear/starttls           |
-|                          |          |                                    |                          | * use ldaps:// for ssl                     |
-|                          |          |                                    |                          | * custom port: ldap://<host>:<port>        |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| ca                       | backends | Path to the CA file                | file path                | optional                                   |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| starttls                 | backends | Use starttls                       | 'on' or 'off'            | optional                                   |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| checkcert                | backends | Check the server certificat        | 'on' or 'off'            | optional                                   |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| binddn                   | backends | The bind dn to use                 | ldap dn                  | This dn must have read/write permissions   |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| password                 | backends | The password of the bind dn        | password                 |                                            |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| timeout                  | backends | Ldap connexion timeout             | integer (second)         |                                            |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| password                 | backends | The password of the bind dn        | password                 |                                            |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| groupdn                  | backends | The ldap dn where groups are       | ldap dn                  |                                            |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| userdn                   | backends | The ldap dn where users are        | ldap dn                  |                                            |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| user_filter_tmpl         | backends | The search filter template         | ldap search filter       | The user identifier is passed through      |
-|                          |          | to recover a given user            | template                 | the **username** variable (*%(username)s*) |
-|                          |          |                                    |                          |                                            |
-|                          |          |                                    |                          | **username** is the attribute marked by    |
-|                          |          |                                    |                          | **key: True** in the **attribute.yml** file|
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| group_filter_tmpl        | backends | The search filter template to      | ldap search filter       | The following variables are usable:        |
-|                          |          | recover the groups of a given user | template                 |                                            |
-|                          |          | recover the groups of a given user | template                 | * **username**: the user's key attribute   |
-|                          |          |                                    |                          | * **userdn**: the user's ldap dn           |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| group_attr.<member attr> | backends | Member attribute template value    | template                 | * <member attr> is the member attribute    |
-|                          |          |                                    |                          |   in groups dn entries                     |
-|                          |          |                                    |                          | * every user attributes are exposed        |
-|                          |          |                                    |                          |   in the template                          |
-|                          |          |                                    |                          | * multiple <memver attr> attributes        |
-|                          |          |                                    |                          |   can be set                               |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| objectclasses            | backends | list of object classes for users   | comma separated list     |                                            |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
-| dn_user_attr             | backends | attribute used in users dn         | dn attribute             |                                            |
-+--------------------------+----------+------------------------------------+--------------------------+--------------------------------------------+
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+|      Parameter           | Section  |            Description             |           Values         |                Comment                         |
++==========================+==========+====================================+==========================+================================================+
+| uri                      | backends | The ldap uri to access             | ldap uri                 | * use ldap:// for clear/starttls               |
+|                          |          |                                    |                          | * use ldaps:// for ssl                         |
+|                          |          |                                    |                          | * custom port: ldap://<host>:<port>            |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| ca                       | backends | Path to the CA file                | file path                | optional                                       |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| starttls                 | backends | Use starttls                       | 'on' or 'off'            | optional                                       |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| checkcert                | backends | Check the server certificat        | 'on' or 'off'            | optional                                       |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| binddn                   | backends | The bind dn to use                 | ldap dn                  | This dn must have read/write permissions       |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| password                 | backends | The password of the bind dn        | password                 |                                                |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| timeout                  | backends | Ldap connexion timeout             | integer (second)         |                                                |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| password                 | backends | The password of the bind dn        | password                 |                                                |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| groupdn                  | backends | The ldap dn where groups are       | ldap dn                  |                                                |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| userdn                   | backends | The ldap dn where users are        | ldap dn                  |                                                |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| user_filter_tmpl         | backends | The search filter template         | ldap search filter       | The user identifier is passed through          |
+|                          |          | to recover a given user            | template                 | the **username** variable (*%(username)s*)     |
+|                          |          |                                    |                          |                                                |
+|                          |          |                                    |                          | **username** is the content of the             |
+|                          |          |                                    |                          | the attribute marked by '**key: Truee**'       |
+|                          |          |                                    |                          | in the **attributes.yml** file                 |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| group_filter_tmpl        | backends | The search filter template to      | ldap search filter       | The following variables are usable:            |
+|                          |          | recover the groups of a given user | template                 |                                                |
+|                          |          | recover the groups of a given user | template                 | * **username**: the user's key attribute       |
+|                          |          |                                    |                          | * **userdn**: the user's ldap dn               |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| group_attr.<member attr> | backends | Member attribute template value    | template                 | * <member attr> is the member attribute        |
+|                          |          |                                    |                          |   in groups dn entries                         |
+|                          |          |                                    |                          | * every user attributes are exposed            |
+|                          |          |                                    |                          |   in the template                              |
+|                          |          |                                    |                          | * multiple <memver attr> attributes            |
+|                          |          |                                    |                          |   can be set (ex: group_attr.member            |
+|                          |          |                                    |                          |   (ex: group_attr.member, group_attr.usermemb) |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| objectclasses            | backends | list of object classes for users   | comma separated list     |                                                |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
+| dn_user_attr             | backends | attribute used in users dn         | dn attribute             |                                                |
++--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
 
 
 Example
@@ -101,46 +103,58 @@ Example
 
 .. sourcecode:: ini
 
-    [backends]
-    
-    # name of the module
-    ldap.module = 'ldapcherry.backend.backendLdap'
-    # display name of the ldap
-    ldap.display_name = 'My Ldap Directory'
-    
-    # uri of the ldap directory
-    ldap.uri = 'ldap://ldap.ldapcherry.org'
-    # ca to use for ssl/tls connexion
-    #ldap.ca = '/etc/dnscherry/TEST-cacert.pem'
-    # use start tls
-    #ldap.starttls = 'off'
-    # check server certificate (for tls)
-    #ldap.checkcert = 'off'
-    # bind dn to the ldap
-    ldap.binddn = 'cn=dnscherry,dc=example,dc=org'
-    # password of the bind dn
-    ldap.password = 'password'
-    # timeout of ldap connexion (in second)
-    ldap.timeout = 1
-    
-    # groups dn
-    ldap.groupdn = 'ou=group,dc=example,dc=org'
-    # users dn
-    ldap.userdn = 'ou=people,dc=example,dc=org'
-    # ldapsearch filter to get a user
-    ldap.user_filter_tmpl = '(uid=%(username)s)'
-    # ldapsearch filter to get groups of a user
-    ldap.group_filter_tmpl = '(member=uid=%(username)s,ou=People,dc=example,dc=org)'
-    # filter to search users
-    ldap.search_filter_tmpl = '(|(uid=%(searchstring)s*)(sn=%(searchstring)s*))'
+   [backends]
     
-    # ldap group attributes and how to fill them
-    ldap.group_attr.member = "%(dn)s"
-    #ldap.group_attr.memberUid = "%(uid)s"
-    # object classes of a user entry
-    ldap.objectclasses = 'top, person, posixAccount, inetOrgPerson'
-    # dn entry attribute for an ldap user
-    ldap.dn_user_attr = 'uid'
+   #####################################
+   #   configuration of ldap backend   #
+   #####################################
+   
+   # name of the module
+   ldap.module = 'ldapcherry.backend.backendLdap'
+   # display name of the ldap
+   ldap.display_name = 'My Ldap Directory'
+   
+   # uri of the ldap directory
+   ldap.uri = 'ldap://ldap.ldapcherry.org'
+   # ca to use for ssl/tls connexion
+   #ldap.ca = '/etc/dnscherry/TEST-cacert.pem'
+   # use start tls
+   #ldap.starttls = 'off'
+   # check server certificate (for tls)
+   #ldap.checkcert = 'off'
+   # bind dn to the ldap
+   ldap.binddn = 'cn=dnscherry,dc=example,dc=org'
+   # password of the bind dn
+   ldap.password = 'password'
+   # timeout of ldap connexion (in second)
+   ldap.timeout = 1
+   
+   # groups dn
+   ldap.groupdn = 'ou=group,dc=example,dc=org'
+   # users dn
+   ldap.userdn = 'ou=people,dc=example,dc=org'
+   
+   # ldapsearch filter to get one specific user
+   # %(username)s is content of the attribute marked 'key: True' in the attributes.file config file
+   ldap.user_filter_tmpl = '(uid=%(username)s)'
+   # ldapsearch filter to get groups of a user
+   # %(username)s is content of the attribute marked 'key: True' in the attributes.file config file
+   ldap.group_filter_tmpl = '(member=uid=%(username)s,ou=People,dc=example,dc=org)'
+   # filter to search users
+   # %(searchstring)s is the content passed through the search box
+   ldap.search_filter_tmpl = '(|(uid=%(searchstring)s*)(sn=%(searchstring)s*))'
+   
+   # ldap group attributes and how to fill them
+   # 'member' is the name of the attribute
+   # for the template, any of the user's ldap attributes can be user
+   ldap.group_attr.member = "%(dn)s"
+   # same with memverUid and the uid user's attribute
+   #ldap.group_attr.memberUid = "%(uid)s"
+   
+   # object classes of a user entry
+   ldap.objectclasses = 'top, person, posixAccount, inetOrgPerson'
+   # dn entry attribute for an ldap user
+   ldap.dn_user_attr = 'uid'
 
 
 Active Directory Backend

From 245bafb01c4b7f7302667577a7f4f0390f53a9d1 Mon Sep 17 00:00:00 2001
From: kakwa <carpentier.pf@gmail.com>
Date: Wed, 13 Feb 2019 09:44:47 +0100
Subject: [PATCH 78/82] typo

---
 conf/ldapcherry.ini | 8 ++++----
 docs/backends.rst   | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/conf/ldapcherry.ini b/conf/ldapcherry.ini
index dfbe87a..c5de286 100644
--- a/conf/ldapcherry.ini
+++ b/conf/ldapcherry.ini
@@ -107,14 +107,14 @@ ldap.groupdn = 'ou=group,dc=example,dc=org'
 # users dn
 ldap.userdn = 'ou=people,dc=example,dc=org'
 
-# ldapsearch filter to get a specific user
-# %(username)s is content of the attribute marked 'key: True' in the attributes.file conf
+# ldapsearch filter to get one specific user
+# %(username)s is content of the attribute marked 'key: True' in the attributes.file config file
 ldap.user_filter_tmpl = '(uid=%(username)s)'
 # ldapsearch filter to get groups of a user
-# %(username)s is content of the attribute marked 'key: True' in the attributes.file conf
+# %(username)s is content of the attribute marked 'key: True' in the attributes.file config file
 ldap.group_filter_tmpl = '(member=uid=%(username)s,ou=People,dc=example,dc=org)'
 # filter to search users
-# %(searchstring)s is content passed through the search box
+# %(searchstring)s is the content passed through the search box
 ldap.search_filter_tmpl = '(|(uid=%(searchstring)s*)(sn=%(searchstring)s*))'
 
 # ldap group attributes and how to fill them
diff --git a/docs/backends.rst b/docs/backends.rst
index ec5994b..7679597 100644
--- a/docs/backends.rst
+++ b/docs/backends.rst
@@ -76,7 +76,7 @@ The ldap backend exposes the following parameters:
 |                          |          | to recover a given user            | template                 | the **username** variable (*%(username)s*)     |
 |                          |          |                                    |                          |                                                |
 |                          |          |                                    |                          | **username** is the content of the             |
-|                          |          |                                    |                          | the attribute marked by '**key: Truee**'       |
+|                          |          |                                    |                          | the attribute marked by '**key: True**'        |
 |                          |          |                                    |                          | in the **attributes.yml** file                 |
 +--------------------------+----------+------------------------------------+--------------------------+------------------------------------------------+
 | group_filter_tmpl        | backends | The search filter template to      | ldap search filter       | The following variables are usable:            |

From 50c6259035113d6f91a9fe5f7fe0664f3aa095a4 Mon Sep 17 00:00:00 2001
From: Johann Queuniet <sub_code.git@queuniet.fr>
Date: Wed, 20 Mar 2019 15:57:45 +0100
Subject: [PATCH 79/82] Fix email regexp

---
 ldapcherry/attributes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ldapcherry/attributes.py b/ldapcherry/attributes.py
index 5ddea82..a544569 100644
--- a/ldapcherry/attributes.py
+++ b/ldapcherry/attributes.py
@@ -71,7 +71,7 @@ def __init__(self, attributes_file):
             raise MissingUserKey()
 
     def _is_email(self, email):
-        pattern = r'[\.\w]{1,}[@]\w+[.]\w+'
+        pattern = r'[\+\.\w]+@[-\.\w]+\.\w+'
         if re.match(pattern, email):
             return True
         else:

From 0a96ca61d541cf2cd76c29a811b82914764e43fc Mon Sep 17 00:00:00 2001
From: Johann Queuniet <sub_code.git@queuniet.fr>
Date: Tue, 26 Mar 2019 11:05:39 +0100
Subject: [PATCH 80/82] Fix default handler arguments

---
 ldapcherry/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ldapcherry/__init__.py b/ldapcherry/__init__.py
index a948574..4a3b584 100644
--- a/ldapcherry/__init__.py
+++ b/ldapcherry/__init__.py
@@ -1170,7 +1170,7 @@ def modify(self, user=None, **params):
 
     @cherrypy.expose
     @exception_decorator
-    def default(self, attr='', **params):
+    def default(self, attr='', *args, **params):
         cherrypy.response.status = 404
         self._check_auth(must_admin=False)
         is_admin = self._check_admin()

From d61c89460e4e61e5872e168943666b653e581d4b Mon Sep 17 00:00:00 2001
From: smacz <andrewcz@neomailbox.net>
Date: Tue, 23 Apr 2019 11:24:17 -0400
Subject: [PATCH 81/82] Add install command to setup in README

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 330eb88..e7c407c 100644
--- a/README.rst
+++ b/README.rst
@@ -90,7 +90,7 @@ The default backend plugins permit to manage Ldap and Active Directory.
     $ export DATAROOTDIR=/usr/share/
     
     # install ldapcherry
-    $ python setup.py
+    $ python setup.py install
 
     # edit configuration files
     $ vi /etc/ldapcherry/ldapcherry.ini

From b9743ef9d04ee7147b285bfef1b7570340377c9a Mon Sep 17 00:00:00 2001
From: AndrewCz <andrewcz@neomailbox.net>
Date: Sun, 21 Jul 2019 18:55:02 -0400
Subject: [PATCH 82/82] Add documentation for installing on Debian/Ubuntu and
 CentOS/RHEL

---
 docs/install.rst | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/docs/install.rst b/docs/install.rst
index 448644d..896a576 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -1,9 +1,35 @@
 Install
 =======
 
+Using pip
+---------
+
+Install the dependencies from the operating system's repos, and then install ldapcherry using `pip`:
+
+.. sourcecode:: bash
+
+    # RHEL/CentOS
+    $ sudo yum install epel-release && sudo yum install python-pip python-devel gcc openldap-devel
+
+    # Debian/Ubuntu
+    $ sudo apt install python-dev python-pip libldap2-dev libsasl2-dev libssl-dev
+
+    $ pip install ldapcherry
+
 From the sources
 ----------------
 
+Install the dependencies from the operating system's repos:
+
+.. sourcecode:: bash
+
+    # RHEL/CentOS
+    $ sudo yum install epel-release && sudo yum install python36-pip python36-devel python36-PyYAML gcc openldap-devel
+
+    # Debian/Ubuntu
+    $ sudo apt install python-dev python-pip libldap2-dev libsasl2-dev libssl-dev
+
+
 Download the latest release from `GitHub <https://github.com/kakwa/ldapcherry/releases>`_.
 
 .. sourcecode:: bash