#! /usr/bin/env python
#
# fmdr --- Manage Debian package repositories
#
# Copyright (c) 2006, 2007, 2009, 2010 Florent Rougon
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 dated June, 1991.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; see the file COPYING. If not, write to the
# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
# Boston, MA  02110-1301 USA.

from __future__ import nested_scopes
import sys, os, errno, getopt, re, random, shutil

progname = os.path.basename(sys.argv[0])
progversion = "1.6"
version_blurb = """Written by Florent Rougon.

Copyright (c) 2006, 2007, 2009 Florent Rougon
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."""


usage = """Usage: %(progname)s [option ...]
Manage Debian package repositories.

The archive to act on must be specified with --label and an action, among
those listed below, must also be provided.

The process of finalizing a distribution comprehends the following steps:
  - make sure there are Packages[.gz,.bz2], Sources[.gz,.bz2] and legacy
    Release files (in dists/<dist>/<section>/<arch>/) for every section and
    supported architecture;
  - compact the Sources and Packages files;
  - create the master Release file in dists/<dist> (read by APT >= 0.6);
  - optionally sign the master Release file.

Notes:
  1. With --upload, all symbolic links in the master repository are removed
     during the upload and restored afterwards. This is a necessary workaround
     because scp follows symbolic links instead of copying them verbatim.

  2. Removals performed through the management interface only affect the
     Packages[.gz,.bz2] and Sources[.gz,.bz2] files, the master Release file
     of the distribution (when the distribution is finalized) and the
     corresponding Release.gpg (unless --no-sign is given). The removal of
     source and binary package files (.dsc, *.tar.(gz|bz2|lzma), .diff.gz,
     .deb and .udeb files) has to be done manually in the package pool.


Actions:
  -a, --add=PACKAGE.CHANGES    add PACKAGE to the archive and finalize the
                               distribution in which it is installed. This
                               action may be given several times.
  -d, --dist=DISTRIBUTION      override the target distribution when adding
                               packages with --add
  -f, --finalize=DISTRIBUTION  finalize DISTRIBUTION. This action may be
                               given several times.
  -m, --manage=DISTRIBUTION    manage DISTRIBUTION
  -u, --upload                 upload the archive to one or more remote hosts
                               with ssh and scp
  -r, --replicate              replicate the master repository to one or more
                               local directories

Options:
  -l, --label=LABEL            label of the archive to act on
  -n, --no-sign                don't sign the master Release file when
                               finalizing
  -C, --config-file=FILE       use FILE instead of the default configuration
                               file, ~/.%(progname)s/config.py
      --help                   display this message and exit
      --version                output version information and exit""" \
  % {"progname": progname}

codenames = None
codename_to_dist = {}

upstream_version_re = r"[A-Za-z0-9.+-~]+?"
version_re = (r"((?P<epoch>\d*):)?(?P<upstream_version>%s)" + \
              r"(-(?P<debian_revision>[A-Za-z0-9.+~]+))?") % upstream_version_re
dsc_cre = re.compile(r"^(?P<srcpkg>[^_]+)_(?P<version>%s)\.dsc$" % version_re)
deb_cre = re.compile(
    r"^(?P<binpkg>[a-z0-9+-.]+)_(?P<version>%s)_(?P<arch>.+)\.u?deb$"
    % version_re)

# Exceptions raised by this module
class error(Exception):
    """Base class for exceptions in %s.""" % progname
    def __init__(self, message=None):
        self.message = message
    def __str__(self):
        return "<%s: %s>" % (self.__class__.__name__, self.message)
    def complete_message(self):
        if self.message:
            return "%s: %s" % (self.ExceptionShortDescription, self.message)
        else:
            return "%s" % self.ExceptionShortDescription
    ExceptionShortDescription = "%s generic exception" % progname

# This one is best derived directly from 'Exception' instead of 'error', so
# that we get a proper traceback when it is raised.
class ProgramError(Exception):
    """Exception raised when the program finds that it has a bug."""
    ExceptionShortDescription = "Bug in %s" % progname

class UnableToFindConfigFile(error):
    """Exception raised when we cannot find the configuration file."""
    ExceptionShortDescription = "Unable to find the configuration file"

class ExternalProgramError(error):
    """Exception raised when an external program fails where it shouldn't."""
    ExceptionShortDescription = "External program error"

class UserError(error):
    """Exception raised when the program is used incorrectly."""
    ExceptionShortDescription = "Error"

class InvalidSyntaxForChangesFile(error):
    """Exception raised when a .changes file has an invalid syntax."""
    ExceptionShortDescription = "Invalid syntax for .changes file"

class UnableToCreateTemporaryDirError(error):
    """Exception raised when we cannot create a temporary directory."""
    ExceptionShortDescription = "Unable to create a temporary directory"

class SourcePackageSpreadOverSeveralSections(error):
    """Exception raised when a source package is spread over several sections."""
    ExceptionShortDescription = "Source package files spread over several \
sections"

class IncompleteSourcePackage(error):
    """Exception raised when a source package is incomplete."""
    ExceptionShortDescription = "Incomplete source package"

class ProbablyPythonBug(error):
    """Exception raised when %s behaves in a way that seems to \
indicate a Python bug.""" % progname
    ExceptionShortDescription = "Bug in python, probably"


def run_program_with_stdout_redirected(program, arguments, destfile):
    (rfd, wfd) = os.pipe()
    child_pid = os.fork()
    if child_pid == 0:
        # We are in the child process. We MUST NOT raise any exception.
        try:
            # The child process doesn't need this file descriptor
            os.close(rfd)
            os.dup2(wfd, 1)             # 1 = STDOUT_FILENO
            os.execvp(program, [program] + arguments)
        except:
            os._exit(127)
        
        # Should not happen unless there is a bug in Python
        os._exit(126)

    # We are in the father process.
    #
    # It is essential to close wfd, otherwise we will never see EOF while
    # reading on rfd and the parent process will block forever on the read()
    # call.
    # [ after the fork(), the "reference count" of wfd from the operating
    #   system's point of view is 2; after the child exits, it is 1 until the
    #   father closes it itself; then it is 0 and a read on rfd encounters EOF
    #   once all the remaining data in the pipe has been read. ]
    os.close(wfd)

    # Read the program's output on its stdout
    data = os.fdopen(rfd, "rb").read()
    file(destfile, "wb").write(data)
    
    exit_info = os.waitpid(child_pid, 0)[1]
    if os.WIFEXITED(exit_info):
        exit_code = os.WEXITSTATUS(exit_info)
    # As we wait()ed for the child process to terminate, there is no
    # need to call os.WIFSTOPPED()
    elif os.WIFSIGNALED(exit_info):
        raise ExternalProgramError("%s was terminated by signal %u" %
                                   (program, os.WTERMSIG(exit_info)))
    else:
        raise ProgramError()

    if exit_code == 0:
        return 0
    elif exit_code == 127:
        raise ExternalProgramError("unable to execute '%s'" % program)
    elif exit_code == 126:
        raise ProbablyPythonBug(
            "a child process returned with exit status 126; this might "
            "be the exit status of '%s'; otherwise, we have probably found "
            "a python bug" % program)
    else:
        raise ExternalProgramError("'%s' returned exist status %u"
                                   % (program, exit_code))


def run_program_with_stdin_redirected(program, arguments, srcfile):
    (rfd, wfd) = os.pipe()
    child_pid = os.fork()
    if child_pid == 0:
        # We are in the child process. We MUST NOT raise any exception.
        try:
            # The child process doesn't need this file descriptor
            os.close(wfd)
            os.dup2(rfd, 0)             # 0 = STDIN_FILENO
            os.execvp(program, [program] + arguments)
        except:
            os._exit(127)
        
        # Should not happen unless there is a bug in Python
        os._exit(126)

    # We are in the father process.
    os.close(rfd)                       # Not needed in the father process

    # Pipe srcfile's contents into wfd
    f = os.fdopen(wfd, "wb")
    f.write(file(srcfile, "rb").read())
    f.close()
    
    exit_info = os.waitpid(child_pid, 0)[1]
    if os.WIFEXITED(exit_info):
        exit_code = os.WEXITSTATUS(exit_info)
    # As we wait()ed for the child process to terminate, there is no
    # need to call os.WIFSTOPPED()
    elif os.WIFSIGNALED(exit_info):
        raise ExternalProgramError("%s was terminated by signal %u" %
                                   (program, os.WTERMSIG(exit_info)))
    else:
        raise ProgramError()

    if exit_code == 0:
        return 0
    elif exit_code == 127:
        raise ExternalProgramError("unable to execute '%s'" % program)
    elif exit_code == 126:
        raise ProbablyPythonBug(
            "a child process returned with exit status 126; this might "
            "be the exit status of '%s'; otherwise, we have probably found "
            "a python bug" % program)
    else:
        raise ExternalProgramError("'%s' returned exist status %u"
                                   % (program, exit_code))


def gzip(f):
    run_program_with_stdout_redirected("gzip", ["-9c", f], "%s.gz" % f)

def bzip2(f):
    run_program_with_stdout_redirected("bzip2", ["-9c", f], "%s.bz2" % f)

def gunzip(f):
    assert f.endswith(".gz"), "'%s' should end in '.gz'" % f
    run_program_with_stdout_redirected("gzip", ["-cd", f], f[:-3])

def bunzip2(f):
    assert f.endswith(".bz2"), "'%s' should end in '.bz2'" % f
    run_program_with_stdout_redirected("bzip2", ["-cd", f], f[:-4])


def make_sure_directory_exists(dir):
    if os.path.exists(dir):
        if os.path.isdir(dir):
            return 0
        else:
            raise UserError("'%s' should be a directory but is an existing "
                            "file" % dir)
    else:
        os.makedirs(dir)


def create_temporary_directory(base=None, find_temporary_nb_attempts=5):
    """Create a local temporary directory (securely)."""
    if base is None:
        try:
            base = os.environ["TMPDIR"]
        except KeyError:
            base = "/tmp"
    
    tmp_dir_base = os.path.join(base, os.path.basename(sys.argv[0]))

    for i in range(find_temporary_nb_attempts):
        # Note: one has to use 2**30-1 (well, anything < 2**31)
        # with Python 2.2...
        tmp_dir = "%s-%u-%u" % (tmp_dir_base,
                                os.getpid(),
                                random.randint(0, 2**32-1))
        try:
            os.mkdir(tmp_dir, 0700)
        except OSError:
            continue
        else:
            break
    else:
        raise UnableToCreateTemporaryDirError(
            "somebody is probably trying to attack us")

    return tmp_dir


def normalize_as_distribution(s):
    """If d is a valid distribution, return d. Otherwise, try to see if it is
    a valid codename and return the corresponding distribution name."""
    try:
        codename = codenames[s]
    except KeyError:
        # dist is not a valid distribution
        try:
            s = codename_to_dist[s]
        except KeyError:
            raise UserError("unknown distribution: '%s'" % s)

    return s


def add_srcpkg_file(source_pkg, sections_found, elem):
    section = elem["section"]
    if section not in sections_found:
        sections_found[section] = []

    sections_found[section].append(elem["file"])
    source_pkg["files"].append(elem["file"])


def parse_changes_file(changes_filepath):
    path = os.path.dirname(changes_filepath)
    changes_file = file(changes_filepath, "rb")

    files = []
    dist_found = False

    dist_cre = re.compile(r"^Distribution: (?P<dist>[^ \t\n]+) *$")
    file_cre = re.compile(r"^ +(?P<md5sum>[^ \t\n]+) +(?P<size>\d+) +"
                          r"(?P<sec>[^ \t\n]+) +(?P<priority>[^ \t\n]+) +"
                          r"(?P<file>[^ \t\n]+) *$")
    contrib_cre = re.compile(r"^contrib/([^ \t\n]+)$")
    nonfree_cre = re.compile(r"^non-free/([^ \t\n]+)$")
    # Files belonging to a source package other than the .dsc
    nondsc_spkg_files_cre = re.compile(r".*\.(diff\.gz|tar\.(gz|bz2|lzma))$")

    while True:
        line = changes_file.readline()
        if not line:
            break

        if not dist_found:
            mo = dist_cre.match(line)
            if mo:
                d = mo.group("dist")
                dist = normalize_as_distribution(d)
                dist_found = True

        if line.startswith("Files:"):
            while True:
                l = changes_file.readline()
                if not l.startswith(" "):
                    break
                mo2 = file_cre.match(l)
                if not mo2:
                    raise InvalidSyntaxForChangesFile(
                        "malformed line in section Files: \n\n%s" % l)
                f = {}
                sec, f["file"] = mo2.group("sec", "file")

                mo3 = contrib_cre.match(sec)
                if mo3:
                    f["section"] = "contrib"
                    f["subsection"] = mo3.group(1)
                else:
                    mo3 = nonfree_cre.match(sec)
                    if mo3:
                        f["section"] = "non-free"
                        f["subsection"] = mo3.group(1)
                    else:
                        f["section"] = "main"
                        f["subsection"] = sec
                files.append(f)

    if not dist_found:
        raise InvalidSyntaxForChangesFile("no distribution found")

    source_pkg = {"files": []}
    # Record the section to which each source package file is supposed to
    # belong according to the .changes file, because having the source package
    # spread over several sections doesn't make sense IMO.
    source_sections_found = {}
    bin_pkgs = []

    for f in files:
        if f["file"].endswith(".dsc"):
            mo = dsc_cre.match(f["file"])
            assert mo is not None, f["file"]
            source_pkg["name"] = mo.group("srcpkg")
            source_pkg["version"] = mo.group("version")
            # Take the "source section" from the .dsc file
            source_pkg["section"] = f["section"]
            source_pkg["dsc"] = f["file"]
            add_srcpkg_file(source_pkg, source_sections_found, f)            
        elif nondsc_spkg_files_cre.match(f["file"]):
            add_srcpkg_file(source_pkg, source_sections_found, f)            
        elif f["file"].endswith(".deb") or f["file"].endswith(".udeb"):
            mo = deb_cre.match(f["file"])
            assert mo is not None, f["file"]
            f["name"] = mo.group("binpkg")
            f["version"] = mo.group("version")
            f["architecture"] = mo.group("arch")
            bin_pkgs.append(f)
        else:
            raise UserError("unexpected file in .changes: '%s'" % f["file"])

    if len(source_sections_found.keys()) > 1:
        raise SourcePackageSpreadOverSeveralSections(
            str(source_sections_found))
        
    if not source_pkg.has_key("dsc"):
        raise IncompleteSourcePackage("the .dsc file is missing")

    return path, dist, source_pkg, bin_pkgs


def create_directory_tree_in_repository(archive, dist):
    repo_root = archive["master repository"]
    make_sure_directory_exists(
        os.path.join(repo_root, "dists", codenames[dist]))
    if not os.path.islink(os.path.join(repo_root, "dists", dist)):
        os.symlink(codenames[dist], os.path.join(repo_root, "dists", dist))

    for section in ("main", "contrib", "non-free"):
        for arch in archive["architectures"]:
            make_sure_directory_exists(os.path.join(
                repo_root, "dists", dist, section, "binary-%s" % arch))
        make_sure_directory_exists(os.path.join(
                repo_root, "dists", dist, section, "source"))
        make_sure_directory_exists(os.path.join(repo_root, "pool", section))


def merge_stanza(archive, packagename, destfile, stanza):
    # If destfile doesn't exist but either destfile.gz or destfile.bz2 does,
    # create destfile from a compressed version.
    if not os.path.exists(destfile):
        if os.path.exists("%s.gz" % destfile):
            gunzip("%s.gz" % destfile)
        elif os.path.exists("%s.bz2" % destfile):
            bunzip2("%s.bz2" % destfile)

    if os.path.exists(destfile):
        pkg_header_cre = re.compile(r"^Package: ([^ \t\n]+) *$")
        f = file(destfile, "rb")
        nf = file("%s.fmdr-new" % destfile, "wb")

        found = False
        while True:
            line = f.readline()
            if not line:
                break
            mo = pkg_header_cre.match(line)
            if mo:
                # The same package already has a stanza in destfile
                if mo.group(1) == packagename:
                    found = True
                    # Skip until the next stanza
                    while True:
                        l = f.readline()
                        if not l:
                            nf.write("\n")
                            break
                        elif l == "\n":
                            break
                    nf.write(stanza)
                    continue
            nf.write(line)

        # If we didn't find a stanza for the same package, write the new
        # stanza at the end of destfile.
        if not found:
            nf.write(stanza)

        nf.close()
        os.rename("%s.fmdr-new" % destfile, destfile)
    else:
        if not os.path.isdir(os.path.dirname(destfile)):
            os.makedirs(os.path.dirname(destfile))
        file(destfile, "wb").write(stanza)


def install_source_stuff(archive, dist, path, src_pkg):
    repo_root = archive["master repository"]
    # We previously checked that all source package files are declared to
    # belong to the same section in the .changes file.
    section = src_pkg["section"]
    tmp_dir = create_temporary_directory()
    tmp_pool_dir = os.path.join(tmp_dir, "pool", section, src_pkg["name"])
    pool_dir = os.path.join(repo_root, "pool", section, src_pkg["name"])
    
    os.makedirs(tmp_pool_dir)
    make_sure_directory_exists(pool_dir)

    for f in src_pkg["files"]:
        shutil.copyfile(os.path.join(path, f),
                        os.path.join(tmp_pool_dir, f))
        shutil.copyfile(os.path.join(path, f),
                        os.path.join(pool_dir, f))

    cur_dir = os.getcwd()
    os.chdir(tmp_dir)

    f = os.popen("apt-ftparchive sources %s" % os.path.join("pool", section),
                 'r')

    os.chdir(cur_dir)
    stanza_to_merge = f.read()
    if f.close() is not None:
        raise ExternalProgramError("error when generating the Sources file")

    # Cleanup the temporary directory
    for f in src_pkg["files"]:
        os.unlink(os.path.join(tmp_pool_dir, f))
    os.rmdir(tmp_pool_dir)
    os.rmdir(os.path.join(tmp_dir, "pool", section))
    os.rmdir(os.path.join(tmp_dir, "pool"))
    os.rmdir(tmp_dir)
              
    Sources = os.path.join(repo_root, "dists", dist, section, "source",
                           "Sources")
    merge_stanza(archive, src_pkg["name"], Sources, stanza_to_merge)


def install_binary_stuff(archive, dist, path, bin_pkgs):
    repo_root = archive["master repository"]
    for pkg in bin_pkgs:
        section = pkg["section"]
        tmp_dir = create_temporary_directory()
        tmp_pool_dir = os.path.join(tmp_dir, "pool", section, pkg["name"])
        pool_dir = os.path.join(repo_root, "pool", section, pkg["name"])

        os.makedirs(tmp_pool_dir)
        make_sure_directory_exists(pool_dir)

        shutil.copyfile(os.path.join(path, pkg["file"]),
                        os.path.join(tmp_pool_dir, pkg["file"]))
        shutil.copyfile(os.path.join(path, pkg["file"]),
                        os.path.join(pool_dir, pkg["file"]))

        cur_dir = os.getcwd()
        os.chdir(tmp_dir)

        f = os.popen("apt-ftparchive packages %s"
                     % os.path.join("pool", section), 'r')

        os.chdir(cur_dir)
        stanza_to_merge = f.read()
        if f.close() is not None:
            raise ExternalProgramError("error when generating the Packages "
                                       "file")

        # Cleanup the temporary directory
        os.unlink(os.path.join(tmp_pool_dir, pkg["file"]))
        os.rmdir(tmp_pool_dir)
        os.rmdir(os.path.join(tmp_dir, "pool", section))
        os.rmdir(os.path.join(tmp_dir, "pool"))
        os.rmdir(tmp_dir)

        if pkg["architecture"] == "all":
            # Arch all packages must end up in the Packages files for
            # every supported architecture.
            for arch in archive["architectures"]:
                Packages = os.path.join(repo_root, "dists", dist, section,
                                        "binary-%s" % arch, "Packages")
                merge_stanza(archive, pkg["name"], Packages, stanza_to_merge)
        else:
            Packages = os.path.join(repo_root, "dists", dist, section,
                                    "binary-%s" % pkg["architecture"],
                                    "Packages")
            merge_stanza(archive, pkg["name"], Packages, stanza_to_merge)


def add_package(archive, changes_filepath, overridden_dist=None):
    path, dist, src_pkg, bin_pkgs = parse_changes_file(changes_filepath)
    dist = overridden_dist or dist
    dist = normalize_as_distribution(dist)

    create_directory_tree_in_repository(archive, dist)
    install_source_stuff(archive, dist, path, src_pkg)
    install_binary_stuff(archive, dist, path, bin_pkgs)
    return dist


def make_sure_every_supported_arch_has_Packages_and_Release(archive, dist):
    repo_root = archive["master repository"]
    for section in os.listdir(os.path.join(repo_root, "dists", dist)):
        if not os.path.isdir(os.path.join(repo_root, "dists", dist,
                                          section)):
            continue

        for arch in archive["architectures"]:
            path = os.path.join(repo_root, "dists", dist, section,
                                "binary-%s" % arch)
            Packages = os.path.join(path, "Packages")
            Release = os.path.join(path, "Release")

            if (not os.path.isfile(Packages)) \
                   and (not os.path.isfile("%s.gz" % Packages)) \
                   and (not os.path.isfile("%s.bz2" % Packages)):
                if os.path.isdir(Packages):
                    raise UserError("'%s' should not be a directory"
                                    % Packages)
                # Create an empty Packages file
                file(Packages, "wb").write("")

            if not os.path.isfile(Release):
                if os.path.isdir(Release):
                    raise UserError("'%s' should not be a directory"
                                    % Release)
                # Create a Release file
                r = archive["release file"]
                f = file(Release, "wb")
                f.write("""\
Archive: %s
Component: %s
Origin: %s
Label: %s
Architecture: %s""" % (dist, section, r["origin"], r["label"], arch))
                f.close()


def make_sure_source_pseudoarch_has_Sources_and_Release(archive, dist):
    repo_root = archive["master repository"]
    r = archive["release file"]
    for section in os.listdir(os.path.join(repo_root, "dists", dist)):
        if not os.path.isdir(os.path.join(repo_root, "dists", dist,
                                          section)):
            continue

        path = os.path.join(repo_root, "dists", dist, section, "source")
        Sources = os.path.join(path, "Sources")
        Release = os.path.join(path, "Release")

        if (not os.path.isfile(Sources)) \
               and (not os.path.isfile("%s.gz" % Sources)) \
               and (not os.path.isfile("%s.bz2" % Sources)):
            if os.path.isdir(Sources):
                raise UserError("'%s' should not be a directory"
                                % Sources)
            # Create an empty Sources file
            file(Sources, "wb").write("")

        if not os.path.isfile(Release):
            if os.path.isdir(Release):
                raise UserError("'%s' should not be a directory"
                                % Release)
            # Create a Release file
            f = file(Release, "wb")
            f.write("""\
Archive: %s
Component: %s
Origin: %s
Label: %s
Architecture: source""" % (dist, section, r["origin"], r["label"]))
            f.close()


def compact_Sources_and_Packages(repo_root, dist):
    for section in os.listdir(os.path.join(repo_root, "dists", dist)):
        if not os.path.isdir(os.path.join(repo_root, "dists", dist, section)):
            continue

        for arch in os.listdir(os.path.join(repo_root, "dists", dist,
                                            section)):
            path = os.path.join(repo_root, "dists", dist, section, arch)
            for f in ("Sources", "Packages"):
                if os.path.isfile(os.path.join(path, f)):
                    gzip(os.path.join(path, f))
                    bzip2(os.path.join(path, f))


def create_master_Release_file(archive, dist):
    repo_root = archive["master repository"]
    r = archive["release file"]
    path = os.path.join(repo_root, "dists", dist)
    Release = os.path.join(path, "Release")
    if os.path.isdir(Release):
        raise UserError("'%s' should not be a directory" % Release)

    # Build the argument list for apt-ftparchive
    if r["origin"]:
        args = ["-o", "APT::FTPArchive::Release::Origin=%s" % r["origin"]]
    if r["label"]:
        args.extend(["-o", "APT::FTPArchive::Release::Label=%s" % r["label"]])
    args.extend(["-o", "APT::FTPArchive::Release::Suite=%s" % dist])
    args.extend(["-o", "APT::FTPArchive::Release::Codename=%s" %
                 codenames[dist]])
    args.extend(["-o", "APT::FTPArchive::Release::Architectures=%s"
                 % " ".join(archive["architectures"])])

    components = []
    for section in os.listdir(path):
        if os.path.isdir(os.path.join(path, section)):
            components.append(section)
    args.extend(["-o", "APT::FTPArchive::Release::Components=%s"
                 % ' '.join(components)])

    if r["description"]:
        args.extend(["-o", "APT::FTPArchive::Release::Description=%s"
                     % r["description"]])

    args.append("release")
    args.append(path)

    # We cannot create the Release file directly in its final location,
    # otherwise it will be listed in itself with size 0 (being truncated
    # by the redirection of apt-ftparchive's stdout).
    tmp_dir = create_temporary_directory()
    run_program_with_stdout_redirected("apt-ftparchive", args,
                                       os.path.join(tmp_dir, "Release"))

    tmp_Release_file = os.path.join(tmp_dir, "Release")
    try:
        os.rename(tmp_Release_file, Release)
    except os.error, e:
        # Maybe tmp_dir and the destination Release file are not on the
        # same filesystem. In this case, try to copy and remove instead.
        if errno.errorcode[e.errno] == "EXDEV":
            shutil.copyfile(tmp_Release_file, Release)
            os.unlink(tmp_Release_file)
        else:
            raise
    os.rmdir(tmp_dir)

    # The accompanying Release.gpg, if it exists, is now irrelevant.
    if os.path.exists("%s.gpg" % Release):
        os.unlink("%s.gpg" % Release)


def sign_master_Release_file(archive, dist):
    repo_root = archive["master repository"]
    Release = os.path.join(repo_root, "dists", dist, "Release")
    if not os.path.isfile(Release):
        raise UserError("'%s' should is not a regular file" % Release)

    # Build the argument list for gpg
    args = ["--sign", "--detach-sign", "--armor", "--output"]
    args.append("%s.gpg" % Release)
    args.append(Release)

    # To avoid gpg prompting before overwriting the file
    if os.path.exists("%s.gpg" % Release):
        os.unlink("%s.gpg" % Release)

    exit_code = os.spawnvp(os.P_WAIT, "gpg", ["gpg"] + args)
    if exit_code:
        raise ExternalProgramError(
            "'gpg' failed to sign file '%s' (returned exit code %u)"
            % (Release, exit_code))


def finalize(archive, dist, sign=True):
    repo_root = archive["master repository"]

    if not os.path.isdir(os.path.join(repo_root, "dists")):
        raise UserError("the repository is not well-formed (no 'dists' "
                        "directory)")
    make_sure_every_supported_arch_has_Packages_and_Release(archive, dist)
    make_sure_source_pseudoarch_has_Sources_and_Release(archive, dist)
    compact_Sources_and_Packages(repo_root, dist)
    create_master_Release_file(archive, dist)
    if sign:
        sign_master_Release_file(archive, dist)


def packages_listed_in_Sources_file(Sources):
    pkg_cre = re.compile(r"^Package: ([^ \t\n]+) *$")
    binary_cre = re.compile(r"^Binary: ([^\t\n]+) *$")
    version_cre = re.compile(r"^Version: ([^ \t\n]+) *$")

    f = file(Sources, "rb")

    res = []
    pkg = {}
    while True:
        line = f.readline()
        if not line:
            if pkg:
                res.append(pkg)
            break
        elif line == '\n':
            if pkg:
                res.append(pkg)
                pkg = {}
        else:
            mo = pkg_cre.match(line)
            if mo:
                pkg["name"] = mo.group(1)
                continue

            mo = binary_cre.match(line)
            if mo:
                bins = mo.group(1)
                l = bins.split(',')
                pkg["binary"] = map(lambda s: s.lstrip(), l)
                continue
            
            mo = version_cre.match(line)
            if mo:
                pkg["version"] = mo.group(1)
                continue
    return res


def source_packages_in_distribution(archive, dist):
    repo_root = archive["master repository"]
    res = []

    for section in os.listdir(os.path.join(repo_root, "dists", dist)):
        if not os.path.isdir(os.path.join(repo_root, "dists", dist, section)):
            continue

        Sources = os.path.join(repo_root, "dists", dist, section, "source",
                               "Sources")
        if os.path.isfile(Sources):
            res.extend(packages_listed_in_Sources_file(Sources))

    return res


def list_source_packages_in_distribution(archive, dist):
    for src_pkg in source_packages_in_distribution(archive, dist):
        sys.stdout.write("%s %s -> %s" % (src_pkg["name"], src_pkg["version"],
                                          src_pkg["binary"][0]))
        for bin_pkg in src_pkg["binary"][1:]:
            sys.stdout.write(", %s" % bin_pkg)
        sys.stdout.write("\n")


def binary_packages_for_source_package(archive, dist, spkg):
    spkg_list = source_packages_in_distribution(archive, dist)
    for p in spkg_list:
        if p["name"] == spkg:
            res = p["binary"]
            break
    else:
        raise UserError("no such source package: '%s'" % spkg)

    return res


def list_binary_packages_for_source_package(archive, dist, spkg):
    try:
        for p in binary_packages_for_source_package(archive, dist, spkg):
            print p
    except UserError, e:
        sys.stderr.write(e.message.capitalize() + '\n')
        return False

    return True


def try_to_remove_package_from_file(filepath, package):
    pkg_cre = re.compile(r"^Package: ([^ \t\n]+) *$")

    f = file(filepath, "rb")
    nf = file("%s.fmdr-new" % filepath, "wb")

    res = "not found"
    current_stanza = []
    current_pkg_name = None
    while True:
        line = f.readline()
        if line in ('', '\n'):
            if current_pkg_name and (current_pkg_name != package):
                current_stanza.append(line)
                nf.write(''.join(current_stanza))
            if current_pkg_name == package:
                res = "removed"
            if line == '':
                break
            else:
                current_stanza = []
                current_pkg_name = None
        else:
            current_stanza.append(line)
            mo = pkg_cre.match(line)
            if mo:
                current_pkg_name = mo.group(1)

    nf.close()
    os.rename("%s.fmdr-new" % filepath, filepath)

    return res


def remove_binary_package_for_arch(archive, dist, package, arch):
    if arch == "all":
        remove_binary_package_for_all_archs(archive, dist, package)
        return True
    elif not arch in archive["architectures"]:
        sys.stderr.write("Architecture not supported by this archive: %s\n"
                         % arch)
        return False

    repo_root = archive["master repository"]

    for section in os.listdir(os.path.join(repo_root, "dists", dist)):
        if not os.path.isdir(os.path.join(repo_root, "dists", dist, section)):
            continue

        Packages = os.path.join(repo_root, "dists", dist, section,
                                "binary-%s" % arch, "Packages")
        
        res = try_to_remove_package_from_file(Packages, package)
        if res == "removed":
            break
        elif res == "not found":
            pass
        else:
            raise ProgramError()
    else:
        sys.stderr.write("No binary package '%s' for architecture %s\n"
                         % (package, arch))
        return False

    return True


def remove_binary_package_for_all_archs(archive, dist, package):
    for arch in archive["architectures"]:
        remove_binary_package_for_arch(archive, dist, package, arch)


def remove_source_package(archive, dist, package):
    repo_root = archive["master repository"]

    for section in os.listdir(os.path.join(repo_root, "dists", dist)):
        if not os.path.isdir(os.path.join(repo_root, "dists", dist, section)):
            continue

        Sources = os.path.join(repo_root, "dists", dist, section, "source",
                               "Sources")
        
        res = try_to_remove_package_from_file(Sources, package)
        if res == "removed":
            break
        elif res == "not found":
            pass
        else:
            raise ProgramError()
    else:
        sys.stderr.write("No such source package: '%s'\n" % package)
        return False

    return True


def remove_everything_from_source_package(archive, dist, src_pkg):
    try:
        for bin_pkg in binary_packages_for_source_package(archive, dist,
                                                          src_pkg):
            remove_binary_package_for_all_archs(archive, dist, bin_pkg)
        return remove_source_package(archive, dist, src_pkg)
    except UserError, e:
        sys.stderr.write(e.message.capitalize() + '\n')
        return False


def manage(archive, dist, sign=True):
    print "Welcome to fmdr's management interface. Type 'h' for help.\n"
    while True:
        try:
            cmd = raw_input("fmdr> ")
        except EOFError:
            break
        if cmd == "ls":
            list_source_packages_in_distribution(archive, dist)
        elif cmd.startswith("ls "):
            l = cmd.split(' ')
            if len(l) != 2:
                sys.stderr.write("Syntax: ls <source package>\n")
                continue
            list_binary_packages_for_source_package(archive, dist, l[1])
        elif cmd.startswith("rmb "):
            l = cmd.split(' ')
            if len(l) == 2:
                remove_binary_package_for_all_archs(archive, dist, l[1])
            elif len(l) == 3:
                remove_binary_package_for_arch(archive, dist, l[1], l[2])
            else:
                sys.stderr.write(
                    "Syntax: rmb <binary package> [<architecture>]\n")
                continue
        elif cmd.startswith("rms "):
            l = cmd.split(' ')
            if len(l) != 2:
                sys.stderr.write(
                    "Syntax: rms <source package>\n")
                continue
            remove_source_package(archive, dist, l[1])
        elif cmd.startswith("rm "):
            l = cmd.split(' ')
            if len(l) != 2:
                sys.stderr.write("Syntax: rm <source package>\n")
                continue
            remove_everything_from_source_package(archive, dist, l[1])
        elif cmd in ("f", "fin", "finalize"):
            try:
                finalize(archive, dist, sign)
            except UserError, e:
                sys.stderr.write(e.message.capitalize() + '\n')
        elif cmd in ("quit", "exit"):
            break
        elif cmd in ("h", "?", "help"):
            print """
Commands:

ls                       list the source packages in the distribution, along
                         with the binary packages they generate
ls <source pkg>          list the binary packages generated by <source pkg>
rmb <binary pkg> <arch>  remove <binary pkg> from <arch>'s Packages file
rmb <binary pkg>         remove <binary pkg> from the Packages files of all
                         architectures
rms <source pkg>         remove <source pkg> from its Sources file
rm <source pkg>          remove <source pkg> from its Sources file and all the
                         binary packages it generates from the Packages files
                         of all architectures
f, fin, finalize         finalize the distribution
quit, exit               exit
h, ?, help               print this message\n"""
        elif cmd == "":
            print "Type 'h', '?' or 'help' to get the list of available " \
                  "commands."
        else:
            sys.stderr.write("Invalid command: %s\n" % cmd)
    

def upload_archive_to_host(archive, dest):
    user = dest["user"]
    host = dest["host"]

    # Clear the remote repository (beware of this one...)
    exit_code = os.spawnvp(
        os.P_WAIT, "ssh",
        ["ssh", "%s@%s" % (user, host), "rm", "-rf", "--", dest["root dir"]])
    if exit_code:
        raise ExternalProgramError(
            "'ssh' failed with exit code %u when trying to clear "
            "'%s@%s:%s'" % (exit_code, user, host, dest["root dir"]))

    # scp blindly follows symbolic links, therefore we have to remove
    # them before copying the files. Let's save them in a tarball. We'll
    # then use this tarball to restore the symlinks in the master repository
    # and copy them to the remote repository (tarball fed to ssh by stdin).
    tmp_dir = create_temporary_directory()
    tarball = os.path.join(tmp_dir, "links.tar")
    cur_dir = os.getcwd()
    os.chdir(archive["master repository"])

    exit_code = os.system("find . -type l | tar -cf %s --files-from=-"
                          % tarball)
    if exit_code:
        raise ExternalProgramError(
            "'tar' failed with exit code %u" % exit_code)

    # Erase the symlinks
    exit_code = os.system("find . -type l -print0 | xargs -0r rm --")
    if exit_code:
        raise ExternalProgramError(
            "'rm' failed with exit code %u" % exit_code)

    # Copy the files
    exit_code = os.spawnvp(
        os.P_WAIT, "scp",
        ["scp", "-r", archive["master repository"],
         "%s@%s:%s" % (user, host, dest["root dir"])])
    if exit_code:
        raise ExternalProgramError(
            "'scp' failed with exit code %u when trying to upload "
            "the files to '%s@%s:%s'" % (exit_code, user, host,
                                         dest["root dir"]))

    # Copy the symlinks on the remote host
    run_program_with_stdin_redirected("ssh",
                                      [host, "cd", dest["root dir"],
                                       "&&", "tar", "-xf", "-"],
                                      tarball)

    # Restore the symlinks in the master repository
    exit_code = os.spawnvp(os.P_WAIT, "tar", ["tar", "-xf", tarball])
    if exit_code:
        raise ExternalProgramError(
            "'tar' failed with exit code %u" % exit_code)
    
    os.chdir(cur_dir)
    os.unlink(tarball)
    os.rmdir(tmp_dir)


def upload(archive):
    for dest in archive["remote copies"]:
        upload_archive_to_host(archive, dest)


def replicate_master_repository_to_one_place(src, dest):
    # Clearing dest (beware of this one...)
    exit_code = os.spawnvp(os.P_WAIT, "rm", ["rm", "-rf", dest])
    if exit_code:
        raise ExternalProgramError(
            "'rm' failed with exit code %u when trying to clear local "
            "repository '%s'" % (exit_code, dest))

    # Copying src
    exit_code = os.spawnvp(os.P_WAIT, "cp", ["cp", "-R", src, dest])
    if exit_code:
        raise ExternalProgramError(
            "'cp' failed with exit code %u when trying to copy the "
            "master repository to '%s'" % (exit_code, dest))


def replicate(archive):
    src = archive["master repository"]
    if not os.path.isdir(src):
        raise UserError("the master repository (%s) is empty" % src)

    for d in archive["local copies"]:
        replicate_master_repository_to_one_place(src, d)


def process_command_line():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "C:l:a:d:f:m:nur",
                                   ["config-file=",
                                    "label=",
                                    "add=",
                                    "dist=",
                                    "finalize=",
                                    "manage=",
                                    "no-sign",
                                    "upload",
                                    "replicate",
                                    "help",
                                    "version"])
    except getopt.GetoptError, message:
        sys.stderr.write(usage + "\n")
        return ("exit", 1)

    params = {}
    actions = ("add", "finalize", "manage", "upload")

    # Default values for options
    params["config_file"] = None
    params["archive_label"] = None
    params["sign"] = True
    params["action"] = None
    params["overridden_dist"] = None
    params["list"] = []

    for option, value in opts:
        if option in ("-C", "--config-file"):
            params["config_file"] = value
        elif option in ("-l", "--label"):
            params["archive_label"] = value
        elif option in ("-a", "--add"):
            if (params["action"] in actions) and (params["action"] != "add"):
                sys.stderr.write("Only one type of action can be "
                                 "specified.\n")
                return ("exit", 1)
            params["action"] = "add"
            params["list"].append(value)
        elif option in ("-d", "--dist"):
            params["overridden_dist"] = value
        elif option in ("-f", "--finalize"):
            if (params["action"] in actions) \
                   and (params["action"] != "finalize"):
                sys.stderr.write("Only one type of action can be "
                                 "specified.\n")
                return ("exit", 1)
            params["action"] = "finalize"
            params["list"].append(value)
        elif option in ("-m", "--manage"):
            if params["action"] is not None:
                sys.stderr.write("The --manage action can only be specified "
                                 "once, and with no other action.\n")
                return ("exit", 1)
            params["action"] = "manage"
            params["list"].append(value)
        elif option in ("-n", "--no-sign"):
            params["sign"] = False
        elif option in ("-u", "--upload"):
            params["action"] = "upload"
        elif option in ("-r", "--replicate"):
            params["action"] = "replicate"
        elif option == "--help":
            print usage
            return ("exit", 0)
        elif option == "--version":
            print "%s %s\n%s" % (progname, progversion, version_blurb)
            return ("exit", 0)
        else:
            raise ProgramError("unexpected option received from the "
                               "getopt module: '%s'" % option)

    if params["archive_label"] is None:
        sys.stderr.write("You must specify the archive to act on "
                         "(--label option).\n")
        return ("exit", 1)

    if params["action"] is None:
        sys.stderr.write("You must specify an action to perform "
                         "(--add, --manage, etc.).\n")
        return ("exit", 1)

    # If the -C option was not supplied, use the default config file
    if params["config_file"] is None:
        try:
            home_dir = os.environ["HOME"]
        except KeyError:
            raise UnableToFindConfigFile(
                "no HOME variable in the environment")
        params["config_file"] = os.path.join(home_dir, ".%s/config.py" \
                                             % progname)

    if len(args) != 0:
        sys.stderr.write(usage + '\n')
        return ("exit", 1)

    return ("continue", params)


def main():
    global codenames, codename_to_dist

    try:
        action, p = process_command_line()
        if action == "exit":
            sys.exit(p)

        # Read the configuration file (i.e., execute it)
        l = {}
        execfile(p["config_file"], {}, l)
        archives = l["archives"]
        codenames = l["codenames"]

        # Build the reverse mapping of codenames
        for dist, codename in codenames.iteritems():
            codename_to_dist[codename] = dist

        # Check that every archive label given on the command line corresponds
        # to an archive declared in the configuration file.
        for a in archives:
            if a["label"] == p["archive_label"]:
                archive = a
                break
        else:
            raise UserError("archive '%s' is not declared in the "
                            "configuration file" % p["archive_label"])

        # Tilde expansion
        archive["master repository"] = os.path.expanduser(
            archive["master repository"])
        for i, d in enumerate(archive["local copies"]):
            archive["local copies"][i] = os.path.expanduser(d)

        # Do the actual work
        if p["action"] == "add":
            dirty_dists = []
            for changes_file in p["list"]:
                dist = add_package(archive, changes_file, p["overridden_dist"])
                if not (dist in dirty_dists):
                    dirty_dists.append(dist)
            for dist in dirty_dists:
                finalize(archive, dist, p["sign"])
        elif p["action"] == "finalize":
            for dist in p["list"]:
                finalize(archive, normalize_as_distribution(dist), p["sign"])
        elif p["action"] == "manage":
            manage(archive, normalize_as_distribution(p["list"][0]), p["sign"])
        elif p["action"] == "upload":
            upload(archive)
        elif p["action"] == "replicate":
            replicate(archive)
        else:
            raise ProgramError()

    except error, exc_instance:
        sys.stderr.write("%s\n" % exc_instance.complete_message())
        sys.exit(2)
    except (IOError, OSError), exc_instance:
        sys.stderr.write("Error: %s\n" % exc_instance)
        sys.exit(2)
    # I prefer letting other exceptions generate a traceback, because they
    # would probably reveal a bug (besides being more useful that way).

if __name__ == "__main__": main()