skript na sync uživatelů z gimliho

README.md

@@ -0,0 +1,25 @@
+## Skripty a configy pro KSP VM
+
+### Hadí skripty na synchronizaci uživatelů z gimliho: [./user-sync](./user-sync)
+
+[`collect_keys.py`](./user-sync/collect_keys.py) vykrade .ssh/authorized_keys z home adresářů specifikovaných uživatelů.
+
+[`copy_users.py`](./user-sync/copy_users.py) je zkopíruje na zadaný server, případně na něm pak spustí nějaký příkaz.
+
+[`accept-users.py`](./user-sync/accept-users.py) vyrobí, případně modifikuje uživatele na cílovém stroji.
+
+Zamýšlené schéma použití
+```bash
+sudo ./collect-keys.py --output ./ssh_keys.json --groups ksp --users případně nějací další uživatelé --verbose
+
+./copy_users.py --user_file ./ssh_keys.json --server root@ksp-vm --server-transfer-file /var/db/gimliusersync/new-users.json --execute 'skript.sh'
+```
+na ksp-vm skript.sh
+```
+./accept-users.py --new-file /var/db/gimliusersync/new-users.json \
+                  --old-file /var/db/gimliusersync/current-users.json \
+                  --ssh-keys-location /etc/ssh/gimli-authorized_keys.d \
+                  --marker-group gimli-user
+                  --extra-groups ksp-org cokoli-dalsiho
+                  --removed-group gimli-removed-user
+```

user-sync/accept_users.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+
+import os, sys, argparse, json, datetime, tempfile, pwd, grp, subprocess
+from dataclasses import dataclass
+from typing import Optional, Any
+
+def report_persistent_problem(*msg):
+    print(*msg, file=sys.stderr)
+    with open("./errors", "a") as f:
+        print(*msg, file=f)
+
+@dataclass(frozen=True)
+class UserData:
+    uid: int
+    name: str
+    homedir: str
+    full_name: str
+    shell: str
+    ssh_keys: list[str]
+
+def read_file(path: str) -> Optional[dict[str, Any]]:
+    if not os.path.exists(path):
+        return None
+    with open(path) as f:
+        return json.load(f)
+    
+def validate_shell(user: UserData) -> str:
+    if not os.path.exists(user.shell):
+        report_persistent_problem(f"Shell {user.shell} is not installed on this system (user {user.name})!")
+        return "/bin/bash"
+    return user.shell
+
+def execute_maybe(cmd: list[str], dry_run: bool, verbose: bool):
+    if verbose or dry_run:
+        print(f"Executing", *cmd)
+    if not dry_run:
+        subprocess.check_call(cmd)
+
+def create_user(user: UserData, marker_group: str, extra_groups: list[str], dry_run: bool, verbose: bool):
+    try:
+        pwd.getpwnam(user.name)
+        report_persistent_problem(f"Cannot create {user.name}:{user.uid} - username already taken!")
+        return
+    except KeyError:
+        pass
+    try:
+        pwd.getpwuid(user.uid)
+        report_persistent_problem(f"Cannot create {user.name}:{user.uid} - uid already taken!")
+        return
+    except KeyError:
+        pass
+
+
+    cmd = [ "useradd",
+           "--uid", str(user.uid),
+           "--user-group",
+           "--create-home",
+           "--comment", user.full_name,
+           "--groups", ",".join([marker_group, *extra_groups]),
+           "--shell", validate_shell(user),
+            # TODO: subuid and subgid are handled?
+    ]
+    execute_maybe(cmd, dry_run, verbose)
+
+def modify_user(old: UserData, new: UserData, marker_group: str, dry_run: bool, verbose: bool):
+    if old.uid != new.uid or old.name != new.name:
+        report_persistent_problem(f"Cannot modify {old.name}:{old.uid} to {new.name}:{new.uid} - uid or name changed!")
+        return
+    
+    if new.name not in grp.getgrnam(marker_group).gr_mem:
+        if verbose:
+            print(f"Skipping {new.name}:{new.uid} - not in {marker_group} group")
+        return
+
+    cmd = [ "usermod" ]
+    if old.full_name != new.full_name:
+        cmd.extend([ "--comment", new.full_name ])
+    if old.shell != new.shell:
+        cmd.extend([ "--shell", validate_shell(new) ])
+
+    if cmd == [ "usermod" ]:
+        return
+    execute_maybe(cmd, dry_run, verbose)
+
+def add_group(user_name: str, group: str, dry_run: bool, verbose: bool):
+    if user_name not in grp.getgrnam(group).gr_mem:
+        cmd = [ "usermod", "--append", "--groups", group, user_name ]
+        execute_maybe(cmd, dry_run, verbose)
+
+
+def save_keys(users: list[UserData], ssh_keys_location: str, dry_run: bool, verbose: bool):
+    remove_files = set(os.listdir(ssh_keys_location)).difference(u.name for u in users if len(u.ssh_keys) > 0)
+    if remove_files:
+        if verbose or dry_run:
+            print(f"Removing SSH keys: {remove_files}")
+        if not dry_run:
+            for f in remove_files:
+                os.remove(os.path.join(ssh_keys_location, f))
+
+    for user in users:
+        # doesn't matter if it changed on source, always overwrite if different
+        # users should not overwrite these files
+        if len(user.ssh_keys) == 0:
+            continue
+        file = os.path.join(ssh_keys_location, user.name)
+        new_content = "".join(line.rstrip() + "\n" for line in  user.ssh_keys)
+        if os.path.exists(file):
+            with open(file) as f:
+                old_content = f.read()
+            if old_content == new_content:
+                continue
+
+        if verbose or dry_run:
+            print(f"Saving {len(user.ssh_keys)} SSH keys for {user.name}:{user.uid}")
+        if not dry_run:
+            with open(file, "w") as f:
+                f.write(new_content)
+
+def main_args(new_file_path: str, old_file_path: str, ssh_keys_location, marker_group: str, extra_groups: list[str], removed_group: Optional[str], dry_run: bool, verbose: bool):
+    if os.getuid() != 0:
+        print("WARNING: This script should be executed as root", file=sys.stderr)
+
+    new_file = read_file(new_file_path)
+    if new_file is None:
+        raise Exception(f"File {new_file_path} does not exist")
+    old_file = read_file(old_file_path)
+
+    if old_file and datetime.datetime.fromisoformat(old_file["timestamp"]) > datetime.datetime.fromisoformat(new_file["timestamp"]):
+        raise Exception(f"Old file {old_file_path} is newer than {new_file_path}")
+
+    new_users = [UserData(**u) for u in new_file["users"] ]
+    old_users = {u["name"]: UserData(**u) for u in old_file["users"] } if old_file else {}
+
+    created_users = [u for u in new_users if u.name not in old_users]
+    modified_users = [u for u in new_users if u.name in old_users and u != old_users[u.name]]
+    removed_users = set(u.name for u in old_users.values() if u.name).difference(u.name for u in new_users)
+
+    allowed_users = set(grp.getgrnam(marker_group).gr_mem).union(u.name for u in created_users)
+
+    for u in created_users:
+        create_user(u, marker_group, extra_groups, dry_run, verbose)
+    for u in modified_users:
+        if u.name in allowed_users:
+            modify_user(old_users[u.name], u, marker_group, dry_run, verbose)
+    for u in removed_users:
+        if u in allowed_users and removed_group is not None:
+            add_group(u, removed_group, dry_run, verbose)
+
+    save_keys(new_users, ssh_keys_location, dry_run, verbose)
+
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Creates or modifies users on this machine based on data created by copy_users.py')
+    parser.add_argument("--new-file", help="The file generated using collect-users.py script", required=True)
+    parser.add_argument("--old-file", help="Files used to compare differences, when script finishes this file is overwritten with new_file", required=True)
+    parser.add_argument("--ssh-keys-location", help="Directory where all ssh keys are stored", required=True)
+    parser.add_argument("--marker-group", help="Group name to add to users created by this script. Only users with this group will be updated", required=True)
+    parser.add_argument("--extra-groups", nargs="+", help="Additional groups")
+    parser.add_argument("--removed-group", help="Add this group to users which disappeared from source")
+    parser.add_argument("--dry_run", action="store_true", help="Only print which commands would be executed")
+    parser.add_argument("--verbose", action="store_true", help="Print debug messages")
+    args = parser.parse_args()
+    main_args(args.new_file, args.old_file, args.ssh_keys_location, args.marker_group, args.extra_groups, args.removed_group, args.dry_run, args.verbose)
+
+main()

user-sync/collect_keys.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+import os, sys, argparse, json, datetime, pwd, grp
+from dataclasses import dataclass
+from typing import Optional, Any
+
+@dataclass(frozen=True, order=True)
+class UserInfo:
+    uid: int
+    name: str
+    homedir: str
+
+    def __repr__(self) -> str:
+        return f"{self.name}:{self.uid}"
+    
+    @staticmethod
+    def create(struct: pwd.struct_passwd) -> 'UserInfo':
+        return UserInfo(struct.pw_uid, struct.pw_name, struct.pw_dir)
+
+def get_users(users: list[str], groups: list[str], verbose: bool) -> list[UserInfo]:
+    result = []
+
+    for user in users:
+        try:
+            if user.isdigit():
+                result.append(UserInfo.create(pwd.getpwuid(int(user))))
+            else:
+                result.append(UserInfo.create(pwd.getpwnam(user)))
+        except KeyError:
+            print(f"User {user} does not exist", file=sys.stderr)
+    if verbose:
+        print(f"Added hardcoded users: {result}")
+
+    for group in groups:
+        try:
+            if group.isdigit():
+                g = grp.getgrgid(int(group))
+            else:
+                g = grp.getgrnam(group)
+        except KeyError:
+            print(f"Group {group} does not exist", file=sys.stderr)
+            continue
+        
+        # gp_mem contains only those members which don't have gid == g.gr_gid
+        members = [ UserInfo.create(pwd.getpwnam(member)) for member in g.gr_mem ]
+        members.extend(UserInfo.create(p) for p in pwd.getpwall() if p.pw_gid == g.gr_gid)
+        if verbose:
+            print(f"Added all group '{group}' members: {members}")
+        result.extend(members)
+
+    return list(sorted(set(result)))
+
+def read_authorized_keys(user: UserInfo) -> list[str]:
+    original_uid = os.geteuid()
+    try:
+        os.seteuid(user.uid)
+        keys_file = os.path.join(user.homedir, ".ssh", "authorized_keys")
+        if not os.path.exists(keys_file):
+            return []
+        with open(keys_file) as f:
+            return f.readlines()
+    finally:
+        os.seteuid(original_uid)
+
+def main_args(user_names: list[str], group_names: list[str], output_file: str, verbose: bool):
+    if os.getuid() != 0:
+        print("WARNING: This script should be executed as root", file=sys.stderr)
+    
+    users = get_users(user_names, group_names, verbose)
+
+    result = []
+    for user in users:
+        user_row: dict[str, Any] = {
+            "name": user.name,
+            "uid": user.uid,
+        }
+        try:
+            keys = read_authorized_keys(user)
+            user_row["keys"] = keys
+            if verbose:
+                print(f"Read {len(keys)} keys for {user}")
+        except Exception as e:
+            print(f"Error reading keys for {user}: {e}", file=sys.stderr)
+            user_row["failure"] = True
+        result.append(user_row)
+
+    with open(output_file + ".part", "w") as f:
+        json.dump({
+            "users": result,
+            "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        }, f, indent=4)
+    os.rename(output_file + ".part", output_file)
+
+def main():
+    parser = argparse.ArgumentParser(description='Collect .ssh/authorized_keys files from a specified list of users')
+    parser.add_argument("--users", nargs="+", default=[], help="Collect keys from these users (username or uid)")
+    parser.add_argument("--groups", nargs="+", default=[], help="Collect keys from all users in these groups (group name or gid)")
+    parser.add_argument("--output", help="Output file to write authorized_keys to", required=True)
+    parser.add_argument("--verbose", action="store_true", help="Print debug messages")
+    args = parser.parse_args()
+    main_args(args.users, args.groups, args.output, args.verbose)
+
+main()

user-sync/copy_users.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+import os, sys, argparse, json, datetime, tempfile, pwd, grp, subprocess
+from dataclasses import dataclass, asdict
+from typing import Optional, Any
+
+
+@dataclass(frozen=True)
+class UserData:
+    uid: int
+    name: str
+    homedir: str
+    full_name: str
+    shell: str
+    ssh_keys: list[str]
+
+    @staticmethod
+    def create(struct: pwd.struct_passwd, ssh_keys: list[str]) -> 'UserData':
+        return UserData(struct.pw_uid, struct.pw_name, struct.pw_gecos, struct.pw_shell, struct.pw_dir, [])
+
+def get_userdata(user_file: dict[str, Any], verbose: bool) -> tuple[datetime.datetime, list[UserData]]:
+    users: list[dict[str, Any]] = user_file["users"]
+    created_at = datetime.datetime.fromisoformat(user_file["timestamp"])
+    now = datetime.datetime.now(datetime.timezone.utc)
+
+    if (now - created_at) > datetime.timedelta(days=1):
+        print(f"WARNING: The user file is older than 1 day ({created_at=})", file=sys.stderr)
+    
+    if verbose:
+        print(f"Loaded {len(users)} users, datafile age = {now - created_at}")
+
+    result = []
+    for user in users:
+        uid = int(user["uid"])
+        keys = user["ssh_keys"]
+        try:
+            struct = pwd.getpwuid(uid)
+        except KeyError:
+            print(f"User {user['name']}:{uid} does not exist", file=sys.stderr)
+            continue
+        result.append(UserData.create(struct, keys))
+    return created_at, result
+
+def copy_file(local_path, server, remote_path, verbose):
+    cmd = [ "rsync", local_path, f"{server}:{remote_path}" ]
+    if verbose:
+        cmd.append("--verbose")
+        print(f"Executing", *cmd)
+    subprocess.check_call(cmd)
+
+def execute_command(server, command, verbose):
+    cmd = [ "ssh", server, command ]
+    if verbose:
+        print(f"Executing", *cmd)
+    subprocess.check_call(cmd)
+
+def main_args(keys_file: str, server: str, command: Optional[str], transfer_file: Optional[str], server_transfer_file: str, verbose: bool):
+    if os.getuid() == 0:
+        print("WARNING: This script should be not executed as root, only collect_keys.py requires that", file=sys.stderr)
+
+    with open(keys_file, "r") as f:
+        user_list = json.load(f)
+    keys_ts, users = get_userdata(user_list, verbose)
+    transfer_json = {
+        "keys_timestamp": keys_ts.isoformat(),
+        "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        "users": [ asdict(user) for user in users ]
+    }
+
+    if transfer_file is None:
+        with tempfile.NamedTemporaryFile("w") as f:
+            json.dump(transfer_json, f)
+            f.flush()
+            copy_file(f.name, server, server_transfer_file, verbose)
+    else:
+        with open(transfer_file, "w") as f:
+            json.dump(transfer_json, f)
+        copy_file(transfer_file, server, server_transfer_file, verbose)
+
+    if command is not None:
+        execute_command(server, command, verbose)
+
+def main():
+    parser = argparse.ArgumentParser(description='Copies list of users to a remote server')
+    parser.add_argument("--keys-file", help="The file generated using collect_keys.py script", required=True)
+    parser.add_argument("--server", help="SSH server where to copy the file and execute the command", required=True)
+    parser.add_argument("--command", default=None, help="Command to execute on the remote server. Will be executed using default shell of ssh")
+    parser.add_argument("--transfer-file", default=None, help="Write the transfer JSON file to this path. Otherwise, temporary file is used.")
+    parser.add_argument("--server-transfer-file", help="The file generated using collect_keys.py script", default="/var/ksp-user-sync/new-users.json")
+    parser.add_argument("--verbose", action="store_true", help="Print debug messages")
+    args = parser.parse_args()
+    main_args(args.keys_file, args.server, args.command, args.transfer_file, args.server_transfer_file, args.verbose)
+
+main()