SATELLITE BACKUP

#!/usr/bin/env python3
# satellite_backup_main.py -- puts your copy of a piece of software on your server, over ssh.
# One PROFILE a software. Written for satellite; works for anything.
#
#     cd <the folder you keep its files in>             (any folder; the script can be anywhere)
#     python3 satellite_backup_main.py                    -> the menu: which software, then which repo
#     python3 satellite_backup_main.py satellite main     -> that software, the main folder on the server
#     python3 satellite_backup_main.py satellite 7        -> that software, repo-7 under its repo folder
#     python3 satellite_backup_main.py new                -> a new software profile
#     python3 satellite_backup_main.py satellite setup    -> that profile's setup again
#     python3 satellite_backup_main.py satellite create upload   -> writes a fresh satellite.upload.txt.c4
#     python3 satellite_backup_main.py satellite create ignore   -> writes a fresh satellite.ignore.txt.c4
#
# A PROFILE IS THREE FILES, named after the software, in the folder you run the script from:
#     <name>.settings.txt.c4   the setup's answers: user, server, main, repos, folder
#     <name>.upload.txt.c4     what goes: the folders and files at the top of the software
#     <name>.ignore.txt.c4     what never goes, whatever the upload list says
# The menu lists every profile found there, the last used first, and "new software profile":
#     select software to back up:
#       1. satellite
#       2. new software profile
#     which choice? [1]
# With no profile at all, the first run is the setup. A list that is missing is offered: for a
# software called satellite the standard satellite lists (at the end of this file); for any
# other, an upload list made from the top of its folder, and a short ignore list of build output.
#
# THE SETUP asks the software's name, your user name on the server, the server, the main
# folder on the server [/home/<user>/<name>/main], the folder for the numbered repos
# [/home/<user>/<name>], and where the software is on this machine; Enter keeps what is in
# brackets. The password is never written: it goes in PASSWORD below, or is asked for each run.
#
# NEEDS: Linux or macOS with python3, ssh and rsync; on the server a password login, rsync
# and GNU find (any Linux). The server's key is trusted on first contact, as ssh does it.
#
# The server keeps "main" and a hundred numbered copies, repo-1 to repo-100; one run updates
# one of them, whole. With no repo given the script asks which.
#
# What it does, in order:
#   1. reads the profile's upload list (what goes) and ignore list (what never goes), and
#      walks this machine's copy of the software for the files on the list;
#   2. logs in to the server once, with the password below (asked for if it is empty);
#   3. scans the chosen repo on the server for the files it has -- sizes and times;
#   4. compares: a file goes up when the server does not have it, when ours is newer, or
#      when the two are the same age and their sizes differ. A file that is NEWER on the
#      server is left alone and named. Nothing is ever deleted on the server;
#   5. shows the plan and asks, then uploads the files that go (rsync keeps their times,
#      so the next run sees them as the same), three lines a file as each one starts;
#   6. signs the date, "10/7/2026", into ~/Documents/last_<name>_update.txt on the server,
#      and the same file here: one line a repo, main first, so the file says when each repo
#      was last updated.
#
# PATHS IN BOTH LISTS start with ../ (or ./, the same) and are counted from the top of the
# software's folder: ../build/ is its build folder. A trailing / means a folder and
# everything in it.
# In the ignore list an entry written as a BARE NAME -- no ./ or ../ in front and no slash
# inside -- is matched everywhere in the tree: *.o is every object file wherever it is,
# __pycache__/ every such folder. ../tests/ is only the tests folder at the top.
#
# THE SCRIPT'S OWN FOLDER NEVER GOES UP, whatever the lists say: it may hold the password.

PASSWORD = ""            # <- type the password here before a run, or leave it empty and the script asks
REPO_COUNT = 100
STAMP = "Documents/last_{name}_update.txt"          # under the home folder, there and here
SSH_OPTIONS = []         # anything extra for ssh, e.g. ["-p", "2222"] for another port

SETTINGS_FILE = "{name}.settings.txt.c4"            # a profile's three files, in the folder the script is run from
UPLOAD_FILE = "{name}.upload.txt.c4"
IGNORE_FILE = "{name}.ignore.txt.c4"

import os
import pty
import re
import select
import shlex
import shutil
import subprocess
import sys
import tempfile
import threading
import time
import getpass

HERE = os.path.dirname(os.path.abspath(__file__))
OWN_FOLDER = os.path.basename(HERE)                # never uploaded, wherever it is
HOME = os.path.expanduser("~")
WORK = os.getcwd()                                 # where the profiles live


# ---------------------------------------------------------------- the profiles

SETTING_KEYS = ("name", "user", "server", "main", "repos", "folder")
SETTINGS_SUFFIX = SETTINGS_FILE.format(name="")    # .settings.txt.c4


def profile_path(name, pattern):
    return os.path.join(WORK, pattern.format(name=name))


def find_profiles():
    """The software names with a settings file here, the last used first."""
    found = []
    for f in os.listdir(WORK):
        if f.endswith(SETTINGS_SUFFIX) and len(f) > len(SETTINGS_SUFFIX) and not f.startswith("."):
            found.append((os.path.getmtime(os.path.join(WORK, f)), f[:-len(SETTINGS_SUFFIX)]))
    return [name for _, name in sorted(found, reverse=True)]


def load_profile(name):
    """A profile's settings, or None when its file is missing or short."""
    path = profile_path(name, SETTINGS_FILE)
    if not os.path.exists(path):
        return None
    settings = {"name": name}
    with open(path, encoding="utf-8") as f:
        for raw in f:
            line = raw.strip()
            if not line or line.startswith("#") or "=" not in line:
                continue
            key, value = line.split("=", 1)
            if key.strip() != "name":
                settings[key.strip()] = value.strip()
    if not all(settings.get(k) for k in SETTING_KEYS):
        return None
    for k in ("main", "repos", "folder"):
        settings[k] = os.path.expanduser(settings[k]).rstrip("/") or "/"
    return settings


def profile(name):
    """A profile that must be there and whole."""
    settings = load_profile(name)
    if settings is None:
        sys.exit(f"the {name} profile is missing or short -- `python3 {os.path.basename(__file__)} {name} setup`")
    return settings


def good_name(name):
    """A software's name is one word: letters, digits, - _ . and nothing else."""
    return re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name or "") is not None and name not in (".", "..")


def guess_folder(name):
    """Where the software probably is on this machine: the folder above this one when that is
    its name, else ~/code/cxx/<name>, ~/code/<name> or ~/<name> when one exists, else nothing."""
    parent = os.path.dirname(HERE)
    if os.path.basename(parent) == name:
        return parent
    for candidate in (os.path.join(HOME, "code", "cxx", name), os.path.join(HOME, "code", name), os.path.join(HOME, name)):
        if os.path.isdir(candidate):
            return candidate
    return ""


def ask(what, default=""):
    """Asks until there is an answer; Enter takes the default when there is one."""
    while True:
        answer = input(f"  {what} [{default}]: " if default else f"  {what}: ").strip() or default
        if answer:
            return answer


def setup(old=None, name=""):
    """One profile's setup: asks the six things and writes <name>.settings.txt.c4."""
    old = old or {}
    me = os.path.basename(__file__)
    if not sys.stdin.isatty():
        sys.exit(f"the setup needs a terminal: run `python3 {me} new` there once, or write a profile by hand,"
                 f" <name>{SETTINGS_SUFFIX} in this folder:\n"
                 "    user = yourname\n    server = your.server\n"
                 "    main = /home/yourname/<name>/main\n    repos = /home/yourname/<name>\n"
                 "    folder = /home/yourname/code/<name>")
    print(("the setup of " + old["name"] if old else "a new software profile") + " -- Enter keeps what is in brackets")
    while True:
        name = ask("the software's name, one word", old.get("name") or name)
        if good_name(name):
            break
        print("  one word: letters, digits, - _ . and nothing else")
    user = ask("your user name on the server", old.get("user") or getpass.getuser())
    server = ask("the server", old.get("server"))
    main = ask("the main folder on the server", old.get("main") or f"/home/{user}/{name}/main").rstrip("/") or "/"
    repos = ask(f"the folder for repo-1 to repo-{REPO_COUNT} on the server",
                old.get("repos") or f"/home/{user}/{name}").rstrip("/") or "/"
    guess = old.get("folder") or guess_folder(name)
    while True:
        folder = os.path.expanduser(ask(f"the {name} folder on this machine", guess)).rstrip("/") or "/"
        if os.path.isdir(folder):
            break
        print(f"  there is no folder {folder}")
    settings = {"name": name, "user": user, "server": server, "main": main, "repos": repos, "folder": folder}
    path = profile_path(name, SETTINGS_FILE)
    with open(path, "w", encoding="utf-8") as f:
        f.write(f"# {me}: the {name} profile, from its setup; `python3 {me} {name} setup` asks again."
                " The password is never here.\n")
        for k in SETTING_KEYS[1:]:
            f.write(f"{k} = {settings[k]}\n")
    print(f"  written: {path}")
    return settings


def choose_profile(word):
    """The software to back up: the one named, the only one, or the menu's choice.
    Its settings, or None for a new profile."""
    profiles = find_profiles()
    if word and word != "new":
        return profile(word) if word in profiles else sys.exit(
            f"there is no {word} profile here" + (f" -- there are: {', '.join(profiles)}" if profiles else "")
            + f"; `python3 {os.path.basename(__file__)} new` makes one")
    if word == "new" or not profiles:
        return None
    if not sys.stdin.isatty():
        if len(profiles) == 1:
            return profile(profiles[0])
        sys.exit("which software? one of: " + ", ".join(profiles))
    print("select software to back up:")
    for i, name in enumerate(profiles, 1):
        print(f"  {i}. {name}")
    print(f"  {len(profiles) + 1}. new software profile")
    while True:
        answer = input("which choice? [1] ").strip() or "1"
        if answer.isdigit() and 1 <= int(answer) <= len(profiles):
            return profile(profiles[int(answer) - 1])
        if answer == "new" or (answer.isdigit() and int(answer) == len(profiles) + 1):
            return None
        if answer in profiles:
            return profile(answer)
        print(f"  that is not a choice: 1 to {len(profiles) + 1}")


# ---------------------------------------------------------------- the two lists

GENERIC_IGNORE = """\
# What the server copy never gets. One entry a line; # starts a comment.
# A path starts with ../ and is counted from the top of the software's folder: ../build/ is
# its build folder, and nothing else called build. A trailing / means a folder and everything
# in it; an entry without one is a file. * and ? match inside one name.
# An entry with no slash is a NAME, matched everywhere in the tree: *.o is every object file.
../.git/
../build/
*.o
*.so
*.a
*.d
*.pyc
__pycache__/
"""


def generated_upload(folder):
    """An upload list of everything at the top of the folder (hidden names left out), folders first."""
    now = time.localtime()
    names = [n for n in sorted(os.listdir(folder), key=str.lower) if not n.startswith(".")]
    lines = [f"# What goes to the server: every folder and file at the top of {folder}, as found on"
             f" {now.tm_mon}/{now.tm_mday}/{now.tm_year}. One entry a line; a trailing / is a folder."
             " Remove what should not go.\n"]
    lines += [f"./{n}/\n" for n in names if os.path.isdir(os.path.join(folder, n))]
    lines += [f"{n}\n" for n in names if not os.path.isdir(os.path.join(folder, n))]
    return "".join(lines)


def fresh_list(settings, kind):
    """What a missing list is offered: the satellite standard for satellite, else a made one."""
    if settings["name"] == "satellite":
        return STANDARD_UPLOAD if kind == "upload" else STANDARD_IGNORE
    return generated_upload(settings["folder"]) if kind == "upload" else GENERIC_IGNORE


def list_path(settings, kind):
    return profile_path(settings["name"], UPLOAD_FILE if kind == "upload" else IGNORE_FILE)


def create_list(settings, kind):
    """Writes a fresh list into this folder, asking first when one is already here."""
    path = list_path(settings, kind)
    name = os.path.basename(path)
    if os.path.exists(path):
        if not sys.stdin.isatty():
            sys.exit(f"{name} is already here; remove it first to write a fresh one")
        if not input(f"  {name} is already here -- write a fresh one over it? [y/N] ").strip().lower().startswith("y"):
            print(f"  {name} left as it is")
            return False
    with open(path, "w", encoding="utf-8") as f:
        f.write(fresh_list(settings, kind))
    print(f"  written: {path}")
    return True


def read_list(settings, kind):
    """The entries of the profile's upload or ignore list: (path, is_folder, is_path), comments
    and blanks dropped. is_path: written as a path (./, ../ or a slash inside); otherwise a
    bare name. A list that is not here is offered: a fresh one, written here."""
    entries = []
    path = list_path(settings, kind)
    name = os.path.basename(path)
    if not os.path.exists(path):
        me = os.path.basename(__file__)
        if not sys.stdin.isatty():
            sys.exit(f"there is no {name} in {WORK} -- `python3 {me} {settings['name']} create {kind}` writes a fresh one")
        if not input(f"  there is no {name} in {WORK} -- write a fresh one here? [Y/n] ").strip().lower().startswith("n"):
            create_list(settings, kind)
        else:
            sys.exit(f"nothing to do without {name}")
    with open(path, encoding="utf-8") as f:
        for raw in f:
            line = "" if raw.lstrip().startswith("#") else raw.strip()
            if not line:
                continue
            folder = line.endswith("/")
            path = line.rstrip("/")
            is_path = path.startswith("./") or path.startswith("../") or path.startswith("/")
            while path.startswith("./") or path.startswith("../"):
                path = path[2:] if path.startswith("./") else path[3:]
            path = path.strip("/")
            if not path or path in (".", ".."):
                sys.exit(f"{name}: the entry {raw.strip()!r} names no file")
            entries.append((path, folder, is_path or "/" in path))
    return entries


def pattern_to_regex(pattern):
    """A shell-style pattern for a path: * and ? stay inside one name."""
    out = ""
    for ch in pattern:
        if ch == "*":
            out += "[^/]*"
        elif ch == "?":
            out += "[^/]"
        else:
            out += re.escape(ch)
    return re.compile(out + r"\Z")


class Ignore:
    """The ignore list: exact paths from the top of satellite, and names matched anywhere."""

    def __init__(self, entries):
        self.folders = set()        # exact folder paths
        self.files = set()          # exact file (or folder) paths
        self.path_patterns = []     # (regex, folder_only) for exact paths with * or ?
        self.name_patterns = []     # (regex, folder_only) for bare names
        for path, folder, is_path in entries:
            if is_path:
                if "*" in path or "?" in path:
                    self.path_patterns.append((pattern_to_regex(path), folder))
                elif folder:
                    self.folders.add(path)
                else:
                    self.files.add(path)
            else:
                self.name_patterns.append((pattern_to_regex(path), folder))

    def folder_ignored(self, rel):
        """Only an entry with a trailing slash takes a folder; a file entry never prunes one."""
        name = rel.rsplit("/", 1)[-1]
        if rel in self.folders:
            return True
        return (any(r.match(rel) for r, folder in self.path_patterns if folder)
                or any(r.match(name) for r, folder in self.name_patterns if folder))

    def file_ignored(self, rel):
        name = rel.rsplit("/", 1)[-1]
        if rel in self.files:
            return True
        return (any(r.match(rel) for r, folder in self.path_patterns if not folder)
                or any(r.match(name) for r, folder in self.name_patterns if not folder))

    def ignored(self, rel, is_folder):
        """Is this path, or any folder above it, on the list?"""
        parts = rel.split("/")
        for i in range(1, len(parts)):
            if self.folder_ignored("/".join(parts[:i])):
                return True
        return self.folder_ignored(rel) if is_folder else self.file_ignored(rel)


# ---------------------------------------------------------------- this machine's files

def collect_local(root, upload, ignore):
    """Every file that goes: {path from the top of satellite: (size, mtime)}.
    Also the top-level entries that are on the ignore list, and the ones not here."""
    files = {}
    tops, ignored_tops, missing = [], [], []
    for path, folder, _ in upload:
        full = os.path.join(root, path)
        if not os.path.lexists(full):
            missing.append(path + ("/" if folder else ""))
            continue
        is_dir = os.path.isdir(full)
        if ignore.ignored(path, is_dir):
            ignored_tops.append(path + ("/" if is_dir else ""))
            continue
        tops.append(path)
        if not is_dir:
            st = os.stat(full)
            files[path] = (st.st_size, st.st_mtime)
            continue
        for dirpath, dirnames, filenames in os.walk(full):
            rel_dir = os.path.relpath(dirpath, root).replace(os.sep, "/")
            dirnames[:] = sorted(d for d in dirnames
                                 if not ignore.folder_ignored(rel_dir + "/" + d)
                                 and not os.path.islink(os.path.join(dirpath, d)))
            for f in filenames:
                rel = rel_dir + "/" + f
                if "\t" in f or "\n" in f:
                    print(f"  skipped, its name holds a tab or a newline: {rel}")
                    continue
                if ignore.file_ignored(rel):
                    continue
                full_f = os.path.join(dirpath, f)
                if os.path.islink(full_f):
                    continue                  # links are not copied; nothing on the lists is one
                try:
                    st = os.stat(full_f)
                except FileNotFoundError:
                    continue                  # gone between the listing and now
                files[rel] = (st.st_size, st.st_mtime)
    return files, tops, ignored_tops, missing


# ---------------------------------------------------------------- the server

class Server:
    """One ssh login, kept open on a control socket; every command and the rsync reuse it."""

    def __init__(self, user, host, password, options=()):
        self.user, self.host, self.password = user, host, password
        self.target = f"{user}@{host}"
        self.options = list(options)
        self.dir = None
        self.sock = None
        self.pid = None
        self.fd = None

    def open(self, timeout=90.0):
        # /tmp, not $TMPDIR: a socket path longer than 107 bytes cannot be made
        self.dir = tempfile.mkdtemp(prefix="satellite-update-", dir="/tmp" if os.path.isdir("/tmp") else None)
        self.sock = os.path.join(self.dir, "ssh")
        command = ["ssh", "-M", "-S", self.sock, "-N"] + self.options + [
                   "-o", "ControlPersist=no", "-o", "NumberOfPasswordPrompts=1",
                   "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=20",
                   self.target]
        self.pid, self.fd = pty.fork()
        if self.pid == 0:                       # the child: ssh, with the pty as its terminal
            try:
                os.execvp(command[0], command)
            finally:
                os._exit(127)
        transcript = b""
        window = b""
        sent = False
        started = time.monotonic()
        while True:
            ready, _, _ = select.select([self.fd], [], [], 0.2)
            if ready:
                try:
                    data = os.read(self.fd, 65536)
                except OSError:
                    data = b""
                if data:
                    transcript += data
                    window = (window + data)[-4096:]
                    if not sent and b"assword:" in window.lower():
                        os.write(self.fd, self.password.encode() + b"\n")
                        sent = True
                        window = b""
                    if b"permission denied" in transcript.lower():
                        self._fail("the server refused the password" if sent
                                   else "the server refused the login", transcript)
            if os.path.exists(self.sock) and self._check():
                self._keep_reading()
                return
            done, status = os.waitpid(self.pid, os.WNOHANG)
            if done == self.pid:
                self.pid = None
                self._fail("ssh stopped before the login was done", transcript)
            if time.monotonic() - started > timeout:
                self._fail(f"no login after {int(timeout)} seconds", transcript)

    def _keep_reading(self):
        """After the login the master says nothing worth reading -- but it is read all the same,
        so a full terminal buffer can never block it (and with it every command and the rsync)."""
        def drain():
            while True:
                try:
                    ready, _, _ = select.select([self.fd], [], [], 0.5)
                    if ready and not os.read(self.fd, 65536):
                        return
                except (OSError, ValueError, TypeError):
                    return                      # the terminal was closed: close() ran
        threading.Thread(target=drain, daemon=True).start()

    def _control(self, what):
        """ssh -O check / exit on the control socket: True when the master answered."""
        r = subprocess.run(["ssh", "-S", self.sock] + self.options + ["-O", what, self.target],
                           stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        return r.returncode == 0

    def _check(self):
        return self._control("check")

    def _fail(self, why, transcript):
        text = transcript.decode("utf-8", "replace").replace("\r", "").strip()
        self.close()
        lines = [l for l in text.splitlines() if l.strip()]
        sys.exit(f"{why}\n" + ("\n".join("  ssh: " + l for l in lines[-6:]) if lines else ""))

    def _ssh(self):
        return ["ssh", "-S", self.sock] + self.options + ["-o", "BatchMode=yes", self.target]

    def run(self, command, stdin=None):
        """Run one shell command on the server: (exit code, stdout bytes, stderr bytes)."""
        feed = {"input": stdin} if stdin is not None else {"stdin": subprocess.DEVNULL}
        r = subprocess.run(self._ssh() + [command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, **feed)
        return r.returncode, r.stdout, r.stderr

    def rsync_transport(self):
        return ["-e", " ".join(shlex.quote(a) for a in ["ssh", "-S", self.sock] + self.options
                                                        + ["-o", "BatchMode=yes"])]

    def rsync_destination(self, remote_root):
        return f"{self.target}:{remote_root}/"

    def close(self):
        if self.sock and os.path.exists(self.sock):
            self._control("exit")
        if self.pid:
            try:
                os.kill(self.pid, 15)
            except OSError:
                pass
            try:
                os.waitpid(self.pid, 0)
            except OSError:
                pass
            self.pid = None
        if self.fd is not None:
            try:
                os.close(self.fd)
            except OSError:
                pass
            self.fd = None
        if self.dir:
            shutil.rmtree(self.dir, ignore_errors=True)
            self.dir = None


def scan_server(server, remote_root, tops, ignore):
    """The files the server has under the uploaded entries: {path: (size, mtime)}."""
    command = ("mkdir -p " + shlex.quote(remote_root) + " && cd " + shlex.quote(remote_root)
               + " && echo SCAN-START && find " + " ".join(shlex.quote(t) for t in tops)
               + " -type f -printf '%p\\t%s\\t%T@\\n' 2>/dev/null; echo SCAN-END")
    rc, out, err = server.run(command)
    text = out.decode("utf-8", "surrogateescape")
    if "SCAN-START\n" not in text or not text.rstrip("\n").endswith("SCAN-END"):
        sys.exit("the scan of the server failed (exit " + str(rc) + ")\n  "
                 + err.decode("utf-8", "replace").strip())
    files = {}
    body = text.split("SCAN-START\n", 1)[1].rsplit("SCAN-END", 1)[0]
    for line in body.splitlines():
        try:
            path, size, mtime = line.rsplit("\t", 2)
        except ValueError:
            continue
        if ignore.ignored(path, False):
            continue
        files[path] = (int(size), float(mtime))
    return files


# ---------------------------------------------------------------- the comparison

def compare(local, remote):
    """What goes up and why. Times are compared to the second, which is what rsync keeps."""
    new, changed, same, server_newer = [], [], [], []
    for path in sorted(local):
        size, mtime = local[path]
        if path not in remote:
            new.append(path)
            continue
        rsize, rmtime = remote[path]
        ours, theirs = int(mtime), int(rmtime)
        if ours > theirs:
            changed.append(path)
        elif ours == theirs and size != rsize:
            changed.append(path)
        elif ours < theirs:
            server_newer.append(path)
        else:
            same.append(path)
    only_there = sorted(set(remote) - set(local))
    return new, changed, same, server_newer, only_there


def by_top(paths):
    counts = {}
    for p in paths:
        top = p.split("/", 1)[0]
        counts[top] = counts.get(top, 0) + 1
    return ", ".join(f"{t} {n:,}" for t, n in sorted(counts.items(), key=lambda kv: -kv[1]))


def show(title, paths, limit=40):
    if not paths:
        return
    print(f"  {title}: {len(paths):,}")
    if len(paths) <= limit:
        for p in paths:
            print(f"      {p}")
    else:
        print(f"      {by_top(paths)}")


def megabytes(n):
    return f"{n / 1e6:,.1f} MB"


# ---------------------------------------------------------------- the upload and the stamp

def kb(n):
    return f"{n / 1024:,.1f} kb"


def upload(server, paths, remote_root, local, remote, root):
    """rsync the named files, times and permissions kept. As each file starts, three lines
    (the author's format): a blank line, the sizes -- the server's now, the change, the size
    after -- and the path from the root folder. Returns rsync's exit code."""
    root_name = os.path.basename(remote_root.rstrip("/")) or remote_root
    with tempfile.NamedTemporaryFile("wb", prefix="upload-", suffix=".list", delete=False) as f:
        f.write(b"\0".join(p.encode("utf-8", "surrogateescape") for p in paths) + b"\0")
        list_path = f.name
    try:
        command = (["rsync", "-tp", "--from0", "--files-from=" + list_path,
                    "--timeout=120", "--out-format=%n"]            # %n: each file's path as it is sent
                   + server.rsync_transport()
                   + [root + "/", server.rsync_destination(remote_root)])
        rsync = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL)
        for raw in rsync.stdout:
            name = raw.decode("utf-8", "surrogateescape").rstrip("\r\n")
            if name not in local:
                continue                                           # a folder being made
            after = local[name][0]
            before = remote[name][0] if name in remote else 0
            print(f"\nprev-size: {kb(before)}   change: {kb(after - before)}   total: {kb(after)}"
                  f"\n{root_name}/{name}", flush=True)
        return rsync.wait()
    finally:
        os.unlink(list_path)


def repo_order(name):
    return (0, 0) if name == "main" else (1, int(name.split("-")[-1]) if name.split("-")[-1].isdigit() else 0)


def merge_stamp(existing, name, line):
    """The stamp file with this repo's line replaced or added: one line a repo, main first.
    A line that is not a repo's is dropped."""
    rows = {}
    for old in existing.splitlines():
        key = old.split()[0] if old.split() else ""
        if key == "main" or re.fullmatch(r"repo-[0-9]+", key):
            rows[key] = old.rstrip("\n")
    rows[name] = line
    return "".join(rows[k] + "\n" for k in sorted(rows, key=repo_order))


def stamp(server, software, name, line):
    """This repo's line in the software's stamp file, under the home folder on the server and here."""
    path = STAMP.format(name=software)
    there = "~/" + shlex.quote(path)                        # the shell expands ~ to that user's home
    rc, old, _ = server.run("cat " + there + " 2>/dev/null; true")
    text = merge_stamp(old.decode("utf-8", "replace") if rc == 0 else "", name, line)
    command = "mkdir -p ~/" + shlex.quote(os.path.dirname(path)) + " && cat > " + there
    rc, _, err = server.run(command, stdin=text.encode())
    if rc != 0:
        print("  could not write the stamp on the server: " + err.decode("utf-8", "replace").strip())
    local = os.path.join(HOME, path)
    try:
        here = ""
        if os.path.exists(local):
            with open(local, encoding="utf-8", errors="replace") as f:
                here = f.read()
        os.makedirs(os.path.dirname(local), exist_ok=True)
        with open(local, "w", encoding="utf-8") as f:
            f.write(merge_stamp(here, name, line))
    except OSError as e:
        print(f"  could not write the stamp here: {e}")


# ---------------------------------------------------------------- the run

def is_repo_word(word):
    number = word[5:] if word.startswith("repo-") else word
    return word == "main" or (number.isdigit() and 1 <= int(number) <= REPO_COUNT)


def choose_repo(argv, settings):
    """main, or 1 to REPO_COUNT, from the command line or asked for: (name, folder on the server)."""
    word = argv[0].strip() if argv else ""
    if not word and sys.stdin.isatty():
        word = input(f"  which repo? main, or 1 to {REPO_COUNT}: ").strip()
    number = word[5:] if word.startswith("repo-") else word
    if word == "main":
        return "main", settings["main"]
    if number.isdigit() and 1 <= int(number) <= REPO_COUNT:
        return f"repo-{int(number)}", f"{settings['repos']}/repo-{int(number)}"
    me = os.path.basename(__file__)
    sys.exit(f"which repo? main, or a number from 1 to {REPO_COUNT}:\n"
             f"    python3 {me} {settings['name']} main        -> {settings['main']}\n"
             f"    python3 {me} {settings['name']} 7           -> {settings['repos']}/repo-7")


def main(argv=None):
    argv = list(sys.argv[1:] if argv is None else argv)
    me = os.path.basename(__file__)
    word = ""                                      # the software, unless the first word is a repo or a command
    if argv and not is_repo_word(argv[0]) and argv[0] not in ("setup", "create"):
        word = argv.pop(0)
    settings = choose_profile(word)
    if settings is None:
        settings = setup()
    if argv[:1] == ["setup"]:
        settings = setup(settings)
        argv = argv[1:]
    if argv[:1] == ["create"]:
        which = argv[1] if len(argv) > 1 else ""
        kinds = {"upload": ["upload"], "ignore": ["ignore"], "": ["upload", "ignore"]}
        if which not in kinds:
            sys.exit(f"create what? `create upload` writes {os.path.basename(list_path(settings, 'upload'))},"
                     f" `create ignore` writes {os.path.basename(list_path(settings, 'ignore'))}")
        for kind in kinds[which]:
            create_list(settings, kind)
        return 0
    software = settings["name"]
    name, remote_root = choose_repo(argv, settings)
    root = settings["folder"]
    if not os.path.isdir(root):
        sys.exit(f"there is no folder {root} -- run `python3 {me} {software} setup`")
    upload_entries = read_list(settings, "upload")
    ignore = Ignore(read_list(settings, "ignore") + [(OWN_FOLDER, True, False)])
    if not upload_entries:
        sys.exit(f"{os.path.basename(list_path(settings, 'upload'))} names nothing to upload")

    user, host = settings["user"], settings["server"]
    print(f"{software}: {root} -> {user}@{host}:{remote_root}  ({name})")
    local, tops, ignored_tops, missing = collect_local(root, upload_entries, ignore)
    folders = sum(1 for p, _, _ in upload_entries if os.path.isdir(os.path.join(root, p)))
    print(f"  upload list: {folders} folders, {len(upload_entries) - folders} files")
    if ignored_tops:
        print(f"  on the list but ignored: {' '.join(ignored_tops)}")
    if missing:
        print(f"  on the list but not here: {' '.join(missing)}")
    if not local:
        sys.exit("nothing to upload: every entry is ignored or missing")
    total = sum(size for size, _ in local.values())
    print(f"  this machine: {len(local):,} files, {megabytes(total)}")

    password = PASSWORD or getpass.getpass(f"  password for {user}@{host}: ")
    print(f"  logging in to {user}@{host} ...")
    server = Server(user, host, password, SSH_OPTIONS)
    try:
        server.open()
        remote = scan_server(server, remote_root, tops, ignore)
        print(f"  the server: {len(remote):,} files")
        new, changed, same, server_newer, only_there = compare(local, remote)
        show("new", new)
        show("changed", changed)
        show("newer on the server, left alone", server_newer)
        show("only on the server, left alone", only_there, limit=20)
        print(f"  same: {len(same):,}")
        going = new + changed
        if server_newer and sys.stdin.isatty():
            answer = input(f"  {len(server_newer):,} files are newer on the server -- send ours over them too? [y/N] ")
            if answer.strip().lower().startswith("y"):
                going = going + server_newer
        if not going:
            print("nothing to upload: the server is up to date")
        else:
            size = sum(local[p][0] for p in going)
            if sys.stdin.isatty():
                answer = input(f"upload {len(going):,} files, {megabytes(size)}? [Y/n] ").strip().lower()
                if answer.startswith("n"):
                    print("nothing uploaded")
                    return 0
            started = time.monotonic()
            rc = upload(server, going, remote_root, local, remote, root)
            if rc != 0:
                sys.exit(f"rsync stopped with exit code {rc}; the stamp was not written")
            print(f"uploaded {len(going):,} files, {megabytes(size)}, in {time.monotonic() - started:.1f} s")
        now = time.localtime()
        date = f"{now.tm_mon}/{now.tm_mday}/{now.tm_year}"
        line = (f"{name:<9} {date:>10} {time.strftime('%I:%M %p', now).lstrip('0'):>8}  "
                f"{len(going):,} files uploaded, {len(local):,} checked, from {os.uname().nodename}")
        stamp(server, software, name, line)
        print(f"signed {date} for {name} in ~/{STAMP.format(name=software)}, there and here")
        os.utime(profile_path(software, SETTINGS_FILE))       # last used: first in the menu
        return 0
    finally:
        server.close()
        
if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        print("\nstopped")
        sys.exit(130)