Merge remote-tracking branch 'refs/remotes/origin/master'

README.md

@@ -20,32 +20,12 @@ Install the requirements.
 **Required** minimal packages:
 
 ```
-apt install python3 python3-dev python3-setuptools python3-pip
-```
-
-**Optional** packages (these packages will probably be outdated in debian/ubuntu, and may break scripts):
-
-```
-apt install python3-ldap3 python3-gssapi python3-tabulate
+apt install python3 python3-dev python3-setuptools python3-pip libacl1-dev libkrb5-dev
 ```
 
 ### git / pip
 
-**Required** for modules installation via `pip`:
-
-```
-apt install libkrb5-dev                      # required for `gssapi` module
-```
-
-**Required** modules in local lib directory:
-
-```
-/usr/bin/pip3 install --upgrade pip
-/usr/bin/pip3 install -r requirements-git.txt -t lib/git
-/usr/bin/pip3 install -r requirements-pip.txt -t lib/pip
-```
-
-**Alternative** install from a clean venv (may be needed if you have any conflicting outdated modules installed as packages):
+**Recommended** install from a clean venv (may be needed if you have any conflicting outdated modules installed as packages):
 
 ```
 python3 -m venv venv
@@ -57,6 +37,14 @@ deactivate
 rm -r venv
 ```
 
+**Alternative** modules in local lib directory (this will probably not work):
+
+```
+/usr/bin/pip3 install -r requirements-git.txt -t lib/git
+/usr/bin/pip3 install -r requirements-pip.txt -t lib/pip
+```
+
+
 ## modules
 
 Store or load modules in these directories:

bin/slaptail.py

+#!/usr/bin/env python3
+
+import sys
+import time
+import subprocess
+import select
+import re
+import argparse
+
+update_every = 0.001
+log_path = '/var/log/'
+log_file = 'slapd'
+log_file_path = log_path + log_file
+rgx = dict()
+connections = dict()
+rgx['conn'] = re.compile(r'conn=(?P<conn>[0-9]+)(?P<closed> .*?=.*? closed)?')
+
+
+def sleep(start_time):
+    """After hard work, take a nap"""
+    current_time = time.perf_counter()
+    elapsed_time = current_time - start_time
+    sleep_time = update_every - elapsed_time
+
+    if sleep_time > 0:
+        time.sleep(sleep_time)
+
+
+def parse(line):
+    """Parse line and print"""
+    connect = rgx['connect'].search(line)
+
+    if(connect):
+        ip = connect.group('ip')
+        port = connect.group('port')
+        connection_id = connect.group('conn')
+
+        if connection_id not in connections:
+            connections[connection_id] = ip, port
+
+        print(len(connections), ip, port, line, end='')
+
+    else:
+        conn = rgx['conn'].search(line)
+
+        if(conn):
+            connection_id = conn.group('conn')
+
+            if connection_id in connections:
+                ip, port = connections[connection_id]
+
+                print(len(connections), ip, port, line, end='')
+
+                if conn.group('closed'):
+                    del connections[connection_id]
+
+
+def tail_input(fin):
+    """Open input file in tail mode"""
+    poll = select.poll()
+    poll.register(fin.stdout)
+
+    try:
+        while True:
+            start_time = time.perf_counter()
+
+            if poll.poll(1):
+                line = fin.stdout.readline().decode('utf-8')
+
+                parse(line)
+
+            sleep(start_time)
+
+    except KeyboardInterrupt:
+        sys.stdout.flush()
+        pass
+
+
+def read_input(fin):
+    """Read from input file"""
+    try:
+        for line in fin:
+            parse(line)
+    except KeyboardInterrupt:
+        sys.stdout.flush()
+        pass
+
+
+def main():
+    """Open file in non blocking mode and parse it"""
+    parser = argparse.ArgumentParser(
+        add_help=False,
+        description='Parse from file or stdin'
+    )
+    parser.add_argument(
+        '-f', '--follow',
+        dest='follow', action='store_const', const=True,
+        help='Output appended data as the file grows'
+    )
+    parser.add_argument(
+        'pattern',
+        type=str,
+        help='Regex pattern'
+    )
+    parser.add_argument(
+        'file',
+        nargs='?', type=str,
+        help='File to open'
+    )
+    parser.add_argument(
+        '-h', '--help',
+        action='help',
+        help='Show this help message and exit'
+    )
+    arg = vars(parser.parse_args())
+
+    rgx['connect'] = re.compile(''.join([r'conn=(?P<conn>[0-9]+).+?ACCEPT from IP=(?P<ip>',
+                                arg['pattern'], r'):[0-9]+ \(IP=0\.0\.0\.0:(?P<port>[0-9]+)\)']))
+
+    if arg['follow']:
+        if arg['file']:
+            path = arg['file']
+        else:
+            path = log_file_path
+
+        fin = subprocess.Popen(['tail', '-F', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        tail_input(fin)
+
+    else:
+        if arg['file']:
+            with open(arg['file'], 'r') as fin:
+                read_input(fin)
+        else:
+            fin = sys.stdin
+            read_input(fin)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()
+    sys.exit(0)

bin/sync-winhosts.py

+#!/usr/bin/env python3
+
+import lib_path
+import lib
+#import dphysldap
+import ssl
+import os
+
+from datetime import datetime
+from itertools import count, filterfalse
+from ldap3 import Server, Connection, ALL, Tls, SASL, GSSAPI, ServerPool, RANDOM
+
+# ldap                  = dphysldap.Ldap(ca_certs_file=".\\ca-certificates.crt")
+main_windows_netgroup = "windows"
+windows_log_path      = "c:\\scratch\\"
+unix_log_path         = "/var/log/usradm/"
+log_file              = "sync-winhosts.log"
+ca_certs_file         = "/etc/ssl/certs/ca-certificates.crt"
+
+def connect_ldap(server_names, base):
+    """Connet to the ldap server"""
+    server_names = server_names
+    base = base
+    tls = Tls(
+        validate=ssl.CERT_REQUIRED,
+        version=ssl.PROTOCOL_TLSv1_2,
+        ca_certs_file=ca_certs_file)
+    servers = [Server(s, tls=tls, get_info=ALL) for s in server_names]
+    server_pool = ServerPool(
+        servers,
+        pool_strategy=RANDOM,
+        active=True,
+        exhaust=False)
+    connection = Connection(
+        server_pool,
+        user='ldapadmin/phd-systemxen.ethz.ch',
+        authentication=SASL,
+        sasl_mechanism=GSSAPI,
+        auto_bind='NONE',
+        version=3,
+        client_strategy='SYNC')
+    connection.open()
+    connection.start_tls()
+    connection.bind()
+    user_classes = ['posixAccount', 'dphysUser', 'inetOrgPerson', 'shadowAccount']
+    group_classes = ['posixGroup', 'dphysGroup']
+    obj_user = None
+    obj_group = None
+
+    return connection
+
+def get_netgroups(cn_name):
+    """Get the nisnetgroups"""
+    windows_netgroups = []
+
+    search_filter = "(&(objectClass=nisNetgroup)(cn="+ cn_name +"))"
+    ldap.search(search_base="ou=netgroup,dc=phys,dc=ethz,dc=ch", search_filter=search_filter, attributes=['memberNisNetgroup'])
+    found_groups = ldap.entries
+
+    for found_group in found_groups:
+        windows_netgroups += found_group['memberNisNetgroup']
+
+    return windows_netgroups
+
+def get_netgroup_members(group_name):
+    """Get windows clients of specific nisnetgroup"""
+    windows_clients = []
+
+    search_filter = "(&(objectClass=nisNetgroup)(cn="+ group_name +"))"
+    ldap.search(search_base="ou=netgroup,dc=phys,dc=ethz,dc=ch", search_filter=search_filter, attributes=['nisNetgroupTriple'])
+    found_groups = ldap.entries
+
+    for found_group in found_groups:
+        for client_fqdn in found_group['nisNetgroupTriple']:
+            client_temp = client_fqdn.replace('(','')
+            client_ldap = client_temp.replace('.ethz.ch,-,-)','') + '$'
+            windows_clients.append(client_ldap)
+
+    return windows_clients
+
+def get_netgroup_clients(main_windows_netgroup):
+    """Get all windows clients from nisnetgroup"""
+    clients_netgroup = []
+
+    windows_netgroups = get_netgroups(main_windows_netgroup)
+
+    for windows_netgroup in windows_netgroups:
+        clients_netgroup += get_netgroup_members(windows_netgroup)
+
+    return clients_netgroup
+
+
+def get_ldap_clients():
+    """Get Windows clients in LDAP"""
+    windows_clients = []
+
+    search_filter = "(objectClass=device)"
+    ldap.search(search_base="ou=ad,dc=phys,dc=ethz,dc=ch", search_filter=search_filter, attributes=['cn'])
+    found_clients = ldap.entries
+
+    for client in found_clients:
+        windows_clients.append(str(client['cn']))
+
+    return windows_clients
+
+def sync_clients(clients_netgroup, clients_ldap):
+    """Synchronisation der windows clients im ldap zur master db nisnetgroup"""
+    remove_clients_from_ldap(list(set(clients_ldap) - set(clients_netgroup)))
+    add_clients_to_ldap(list(set(clients_netgroup) - set(clients_ldap)))
+
+def remove_clients_from_ldap(clients):
+    """Remove old obsolet Computers"""
+    write_log("Remove " + str(len(clients)) + " clients from ldap.", True)
+    for client in clients:
+        write_output("remove client "+ client +" from ldap.....", False)
+        delete_windows_host(client)
+
+def add_clients_to_ldap(clients):
+    """Add new Windows Computers to Ldap"""
+    write_log("Add " + str(len(clients)) + " clients to ldap.", True)
+    for client in clients:
+        write_output("add client "+ client + " to ldap.....", False)
+        create_windows_host(client)
+
+def create_windows_host(computername):
+    """create windows host in ldap"""
+    hostname         = computername
+    host_dn          = "uid="+ hostname +",ou=ad,dc=phys,dc=ethz,dc=ch"
+    host_objectClass = ["device","posixAccount"]
+    host_gidNumber   = "60000"
+    host_home        = "/home/" + hostname
+    host_cn          = hostname
+    host_uidNumber   = new_uidNumber()
+    host_attrib      = {'gidNumber':host_gidNumber,'homeDirectory':host_home, 'cn':host_cn, 'uidNumber':host_uidNumber}
+
+    ldap.add(host_dn, host_objectClass, host_attrib)
+    write_output(str(ldap.result['description']), True, False)
+
+def new_uidNumber():
+    """check next free uidNumber and return it"""
+    used_nr = []
+
+    ldap.search("ou=ad,dc=phys,dc=ethz,dc=ch", search_filter="(objectClass=device)", attributes=['uid' ,'uidNumber'])
+    entries = ldap.entries
+
+    for entrie in entries:
+        nr = str(entrie['uidNumber'])
+        used_nr.append(int(nr))
+
+    free_nr = next(filterfalse(set(used_nr).__contains__, count(60001)))
+
+    return str(free_nr)
+
+def delete_windows_host(computername):
+    """remove windows host in ldap"""
+    hostname = computername
+    host_dn  = "uid="+ hostname +",ou=ad,dc=phys,dc=ethz,dc=ch"
+
+    ldap.delete(host_dn)
+    write_output(str(ldap.result['description']), True, False)
+
+def write_log(message,new_line,with_timestamp=True):
+    """write output to logfile"""
+    if os.name == "nt":
+        logfile_path = windows_log_path + log_file
+    else:
+        logfile_path = unix_log_path + log_file
+
+    if with_timestamp:
+        log_message = datetime.now().strftime('%d-%m-%Y %H:%M:%S') + ": " + message
+    else:
+        log_message = message
+
+    if new_line:
+        log_message += "\n"
+
+    f = open(logfile_path, "a+")
+    f.write(log_message)
+
+def write_output(message, new_line,with_timestamp=True):
+    """wirte output to standardout and logfile"""
+    write_log(message, new_line, with_timestamp)
+    if new_line:
+        message += "\n"
+    print(message, end='')
+
+###############
+#
+# Start Script
+#
+###############
+write_log("", True)
+write_log("Start Sync-Windowshost",True)
+write_log("",True)
+
+# LDAP Connection
+write_log("Start connection to LDAP",True)
+ldap = connect_ldap(["phd-aa1.ethz.ch","phd-aa2.ethz.ch","phd-aa3.ethz.ch"],'ou=ldap,dc=phys,dc=ethz,dc=ch')
+
+# Get all Clients from nis Netgroup
+write_log("Get from Netgroup all clients",True)
+clients_netgroup = get_netgroup_clients(main_windows_netgroup)
+#clients_netgroup = ['peter$', 'heidi$', 'geisse$']
+write_log("found "+ str(len(clients_netgroup))+ " clients in netgroup",True)
+
+# Get all Clients for LDAP
+write_log ("Get from LDAP all clients",True)
+clients_ldap = get_ldap_clients()
+write_log("found "+ str(len(clients_ldap))+ " clients in ldap",True)
+
+# Synchrnisation von netgroup auf ldap
+write_log("Start Sync",True)
+sync_clients(clients_netgroup, clients_ldap)
+
+write_log("",True)
+write_log("End Sync-Windowshost",True)
+write_log("",True)