eos-update-flatpak-repos

#!/usr/bin/python3
#
# eos-update-flatpak-repos: adds standard flatpak repos, removes legacy remotes
# and runtimes, and migrates installed flatpaks between origins
#
# Copyright © 2017–2020 Endless OS LLC
# Authors:
#  Mario Sanchez Prada <mario@endlessm.com>
#  Philip Chimento <philip@endlessm.com>
#  Robert McQueen <rob@endlessm.com>
#  Will Thompson <wjt@endlessos.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

import argparse
import fnmatch
import logging
import os
import subprocess
import sys
from urllib.parse import urlparse, urlunparse

from systemd import journal

import gi
gi.require_version('Flatpak', '1.0')
gi.require_version('OSTree', '1.0')
from gi.repository import Flatpak, Gio, GLib, OSTree  # noqa: E402


def _flatpak_inst_get_remote(inst, name):
    try:
        remote = inst.get_remote_by_name(name)
    except GLib.Error:
        remote = None

    return remote


FLATPAK_REPO_DIR = os.getenv('EOS_FLATPAK_REPO_DIR',
                             '/usr/share/eos-boot-helper/flatpak-repos')


def _add_flatpak_repos():
    insts = Flatpak.get_system_installations()
    inst = insts[0]

    for f in os.listdir(FLATPAK_REPO_DIR):
        if not f.endswith('.flatpakrepo'):
            continue

        name = f[:-12]

        repo_file = os.path.join(FLATPAK_REPO_DIR, f)

        if not os.path.isfile(repo_file):
            continue

        remote = _flatpak_inst_get_remote(inst, name)

        if remote:
            logging.debug("Remote {} already configured in {}"
                          .format(name, inst.get_path().get_path()))
            continue

        logging.info("Adding remote {} to {} from {}"
                     .format(name, inst.get_path().get_path(), repo_file))

        subprocess.check_call(['flatpak', 'remote-add', '--system',
                               '--from', name, repo_file])


# list of ostree refs
OSTREE_REFS_TO_REMOVE = [
    # remove eos3.3 refs leaked by eos-updater checkpoint bug (T23410)
    'eos:os/eos/amd64/eos3',
    'eos:os/eos/ec100/eos3',
    'eos:os/eos/nexthw/eos3'

    # remove eos2 ref left behind after eos-upgrade-eos2-to-eos3 (T23443)
    'eos:eos2/ec100',
    'eos:eos2/i386',

    # remove "initial factory commit" ref from pre-2.3.1 (T23443)
    'local:factory',
]


def _remove_ostree_refs():
    try:
        repo = OSTree.Repo.new_default()
        repo.open()
        _, ostree_refs = repo.list_refs()  # dictionary of ref -> commit
        for ref in OSTREE_REFS_TO_REMOVE:
            if ref in ostree_refs:
                logging.info('Deleting leaked OSTree ref {}'.format(ref))
                repo.set_ref_immediate(ref.split(':')[0], ref.split(':')[1],
                                       None)
    except Exception:
        logging.exception('Failure deleting leaked OSTree refs')


# list of remote names
REMOTES_TO_REMOVE = [
]


def _remove_remotes():
    insts = Flatpak.get_system_installations()
    inst = insts[0]

    for name in REMOTES_TO_REMOVE:
        if _flatpak_inst_get_remote(inst, name):
            logging.info("Removing remote {} from {}"
                         .format(name, inst.get_path().get_path()))

            subprocess.check_call(['flatpak', 'remote-delete',
                                   '--system', '--force', name])


# dict of remote -> URLs and collection ID to migrate. If a collection
# ID is not needed, None should be used.
REMOTES_TO_MIGRATE = {
    "eos-runtimes": {
        "old-url": "https://ostree.endlessm.com/ostree/eos",
        "new-url": "https://ostree.endlessm.com/ostree/eos-runtimes",
        "new-collection-id": "com.endlessm.Runtimes",
    },
}


def _migrate_remotes():
    insts = Flatpak.get_system_installations()
    inst = insts[0]

    for name, config in REMOTES_TO_MIGRATE.items():
        remote = _flatpak_inst_get_remote(inst, name)
        if remote and remote.get_url() == config["old-url"]:
            logging.info("Migrating remote {} from {} to {} ({})"
                         .format(name, config["old-url"], config["new-url"],
                                 config["new-collection-id"]))

            remote.set_url(config["new-url"])
            remote.set_collection_id(config["new-collection-id"])
            inst.modify_remote(remote)

            # Try to update the metadata from the remote configuration,
            # but ignore failures in case we're offline.
            logging.info("Updating remote metadata for {}".format(name))
            try:
                inst.update_remote_sync(name)
            except GLib.Error as err:
                logging.warning("Could not update remote metadata for {}: {}"
                                .format(name, err))


# dict of remote -> list of runtimes
RUNTIMES_TO_REMOVE = {
    "eos-apps": [
        # remove partially-installed, empty extension (which can't currently
        # be migrated) before migrating main app (T31185)
        "com.hack_computer.Clubhouse.Locale",
    ],
}


def _remove_runtimes():
    insts = Flatpak.get_system_installations()
    inst = insts[0]

    for ref in inst.list_installed_refs_by_kind(Flatpak.RefKind.RUNTIME):
        origin = ref.get_origin()
        if ref.get_origin() not in RUNTIMES_TO_REMOVE:
            continue

        name = ref.get_name()
        if name not in RUNTIMES_TO_REMOVE[origin]:
            continue

        refspec = '{}/{}/{}'.format(name, ref.get_arch(), ref.get_branch())
        logging.info("Removing runtime {} from {}"
                     .format(refspec, inst.get_path().get_path()))
        subprocess.check_call(['flatpak', 'uninstall', '--system',
                               '--runtime', '--noninteractive', refspec])


# From 3.7.6 through 3.7.8, images were shipped with remote URLs using
# our internal ostree.endlessm-sf.com domain instead of our
# ostree.endlessm.com CDN domain.
#
# https://phabricator.endlessm.com/T29535
def _fix_url(url):
    parts = urlparse(url)
    if parts.netloc == 'ostree.endlessm-sf.com':
        parts = parts._replace(netloc='ostree.endlessm.com')
        url = urlunparse(parts)
    return url


def _fix_remote_urls():
    _, repo = OSTree.Sysroot().get_repo()
    repo.open()
    for remote in repo.remote_list():
        # Only adjust non-flatpak remotes (xa.disable = true)
        _, is_os_repo = repo.get_remote_boolean_option(remote, 'xa.disable',
                                                       False)
        if not is_os_repo:
            logging.debug('Skipping non-OSTree remote %s', remote)
            continue

        _, cur_url = repo.remote_get_url(remote)
        logging.debug('OSTree remote %s current URL %s', remote, cur_url)
        new_url = _fix_url(cur_url)
        if new_url != cur_url:
            # OSTree offers no reasonable API for this
            logging.info('Changing OSTree remote %s URL from %s to %s',
                         remote, cur_url, new_url)
            config = repo.copy_config()
            remote_group = 'remote "{}"'.format(remote)
            config.set_string(remote_group, 'url', new_url)
            repo.write_config(config)

    for inst in Flatpak.get_system_installations():
        for remote in inst.list_remotes():
            cur_url = remote.get_url()
            logging.debug('Flatpak remote %s current URL %s',
                          remote.get_name(), cur_url)
            new_url = _fix_url(cur_url)
            if new_url != cur_url:
                logging.info('Changing Flatpak remote %s URL from %s to %s',
                             remote.get_name(), cur_url, new_url)
                remote.set_url(new_url)
                inst.modify_remote(remote)


def update_deploy_file_with_origin(deploy_file_path, new_origin):
    orig_variant = load_deploy_file(deploy_file_path)

    if orig_variant[0] == new_origin:
        logging.info('Nothing to do, {} origin already set to: {}'
                     .format(deploy_file_path, new_origin))
        return

    write_deploy_file(deploy_file_path, orig_variant, 0,
                      GLib.Variant('s', new_origin))


def update_deploy_file_with_previous_ids(deploy_file_path, new_previous_ids):
    orig_variant = load_deploy_file(deploy_file_path)
    metadata = GLib.VariantDict(orig_variant.get_child_value(4))

    previous_ids_variant = metadata.lookup_value('previous-ids')
    if previous_ids_variant:
        previous_ids = set(previous_ids_variant.unpack())
    else:
        previous_ids = set()

    to_add = new_previous_ids - previous_ids
    if not to_add:
        logging.info('Nothing to do, {} already contains previous-id: {}'
                     .format(deploy_file_path, new_previous_ids))
        return

    logging.debug('Adding {} to previous-ids'.format(to_add))
    previous_ids.update(new_previous_ids)
    metadata.insert_value('previous-ids', GLib.Variant('as', list(previous_ids)))

    write_deploy_file(deploy_file_path, orig_variant, 4, metadata.end())


def load_deploy_file(deploy_file_path):
    logging.debug("Reading data from the deploy file at {}..."
                  .format(deploy_file_path))

    src_file_contents = None
    with open(deploy_file_path, 'rb') as f:
        src_file_contents = GLib.Bytes.new(f.read())

    # We need to read the GVariant in the deploy file and generate a
    # new one with all the same content but the right remote set.
    variant_type = GLib.VariantType.new('(ssasta{sv})')
    orig_variant = GLib.Variant.new_from_bytes(variant_type,
                                               src_file_contents, False)
    logging.debug("Original variant: {}".format(str(orig_variant)))

    return orig_variant


def write_deploy_file(deploy_file_path, orig_variant, idx, new_val):
    variant_type = GLib.VariantType.new('(ssasta{sv})')
    builder = GLib.VariantBuilder.new(variant_type)
    for i, val in enumerate(orig_variant):
        child = orig_variant.get_child_value(i)
        if i == idx:
            builder.add_value(new_val)
        else:
            builder.add_value(child)

    new_variant = builder.end()
    logging.debug("New variant: {}".format(str(new_variant)))

    # Replace the original deploy file with the new GVariant
    logging.info("Writing new deploy file for {}, setting [{}] to: {}"
                 .format(deploy_file_path, idx, new_val))
    GLib.file_set_contents(deploy_file_path,
                           new_variant.get_data_as_bytes().get_data())


def mangle_firefox_metadata(metadata):
    """Endless's Firefox Flatpak stored its data in the real ~/.mozilla in the user's
    home directory. Mozilla's Firefox Flatpak stores its data in what it thinks is
    ~/.mozilla, but which is actually ~/.var/app/$APP_ID/.mozilla through the magic of
    --persist=.mozilla. Monkeypatch our Flatpak to match.

    eos-migrate-firefox-profile is run via a systemd user unit to actually move the
    data."""
    filesystems = _get_with_default(
        metadata.get_string_list,
        FLATPAK_METADATA_GROUP_CONTEXT,
        FLATPAK_METADATA_FILESYSTEMS,
        [],
    )

    if "home" in filesystems:
        filesystems.remove("home")
    if "xdg-download" not in filesystems:
        filesystems.append("xdg-download")

    metadata.set_string_list(
        FLATPAK_METADATA_GROUP_CONTEXT,
        FLATPAK_METADATA_FILESYSTEMS,
        filesystems,
    )

    persistent = _get_with_default(
        metadata.get_string_list,
        FLATPAK_METADATA_GROUP_CONTEXT,
        FLATPAK_METADATA_PERSISTENT,
        [],
    )

    if ".mozilla" not in persistent:
        persistent.append(".mozilla")

    metadata.set_string_list(
        FLATPAK_METADATA_GROUP_CONTEXT,
        FLATPAK_METADATA_PERSISTENT,
        persistent,
    )


FLATPAKS_TO_MIGRATE = [
    # migrate EknServices from old eos-apps remote to eos-sdk (T17863)
    {
        'name': 'com.endlessm.EknServices',
        'kind': Flatpak.RefKind.APP,
        'old-origin': 'eos-apps',
        'new-origin': 'eos-sdk'
    },
    # fix up image-builder bug where com.endlessm.apps.* runtimes were
    # mistakenly installed with eos-runtimes origin (T18366)
    {
        'name': 'com.endlessm.apps.Platform',
        'kind': Flatpak.RefKind.RUNTIME,
        'old-origin': 'eos-runtimes',
        'new-origin': 'eos-sdk'
    },
    {
        'name': 'com.endlessm.apps.Sdk',
        'kind': Flatpak.RefKind.RUNTIME,
        'old-origin': 'eos-runtimes',
        'new-origin': 'eos-sdk'
    },
    # migrate Steam from eos-apps to flathub (T20322)
    {
        'name': 'com.valvesoftware.Steam',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate org.gnome.* from gnome-apps to flathub (T19898)
    {
        'prefix': 'org.gnome.',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'stable',
        'old-origin': 'gnome-apps',
        'new-origin': 'flathub'
    },
    # migrate Teeworlds and MegaGlest from eos-apps to flathub (T20411)
    {
        'name': 'com.teeworlds.Teeworlds',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.megaglest.MegaGlest',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Atom from eos-apps to flathub (T20301)
    {
        'name': 'io.atom.Atom',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate gnome 3.2[02468] and freedesktop 1.[46] runtimes to flathub
    # (T20443 and T25809 and T31079)
    {
        'prefix': 'org.freedesktop.',
        'kind': Flatpak.RefKind.RUNTIME,
        'old-branch': '1.4',
        'old-origin': 'gnome',
        'new-origin': 'flathub'
    },
    {
        'prefix': 'org.freedesktop.',
        'kind': Flatpak.RefKind.RUNTIME,
        'old-branch': '1.6',
        'old-origin': 'gnome',
        'new-origin': 'flathub'
    },
    {
        'prefix': 'org.gnome.',
        'kind': Flatpak.RefKind.RUNTIME,
        'old-branch': '3.20',
        'old-origin': 'gnome',
        'new-origin': 'flathub'
    },
    {
        'prefix': 'org.gnome.',
        'kind': Flatpak.RefKind.RUNTIME,
        'old-branch': '3.22',
        'old-origin': 'gnome',
        'new-origin': 'flathub'
    },
    {
        'prefix': 'org.gnome.',
        'kind': Flatpak.RefKind.RUNTIME,
        'old-branch': '3.24',
        'old-origin': 'gnome',
        'new-origin': 'flathub'
    },
    {
        'prefix': 'org.gnome.',
        'kind': Flatpak.RefKind.RUNTIME,
        'old-branch': '3.26',
        'old-origin': 'gnome',
        'new-origin': 'flathub'
    },
    {
        'prefix': 'org.gnome.',
        'kind': Flatpak.RefKind.RUNTIME,
        'old-branch': '3.28',
        'old-origin': 'gnome',
        'new-origin': 'flathub'
    },
    # migrate eos-apps to flathub where the app ID is the same (T20724)
    {
        'name': 'com.google.AndroidStudio',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'com.slack.Slack',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'com.spotify.Client',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'com.transmissionbt.Transmission',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.minetest.Minetest',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.freeciv.Freeciv',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.gnome.Builder',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.gnome.Genius',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.gnome.Gnote',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.inkscape.Inkscape',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.pitivi.Pitivi',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.tuxpaint.Tuxpaint',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.videolan.VLC',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.wesnoth.Wesnoth',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Visual Studio Code OSS from eos-apps to flathub (T20306)
    {
        'name': 'com.visualstudio.code.oss',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Sublime Text from eos-apps to flathub (T22875)
    {
        'name': 'com.sublimetext.three',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Photo Editor from eos-apps to flathub (T21891)
    {
        'name': 'com.endlessm.photos',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Dropbox from eos-apps to flathub (T23174)
    {
        'name': 'com.dropbox.Client',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Stellarium from eos-apps to flathub (T23681)
    {
        'name': 'org.stellarium.Stellarium',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate 61 legacy eos-apps to flathub (T23856)
    {
        'name': 'com.arduino.App',
        'new-name': 'cc.arduino.arduinoide',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'com.github.Slingshot',
        'new-name': 'com.github.ryanakca.slingshot',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'io.github.Supertux',
        'new-name': 'org.supertuxproject.SuperTux',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.blockout.Blockout2',
        'new-name': 'net.blockout.BlockOutII',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.gcompris.Gcompris',
        'new-name': 'org.kde.gcompris',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.olofson.Kobodeluxe',
        'new-name': 'net.olofson.KoboDeluxe',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.Atanks',
        'new-name': 'net.sourceforge.atanks',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.Audacity',
        'new-name': 'org.audacityteam.Audacity',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.Btanks',
        'new-name': 'net.sourceforge.btanks',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.ChromiumBSU',
        'new-name': 'net.sourceforge.chromium-bsu',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.Extremetuxracer',
        'new-name': 'net.sourceforge.ExtremeTuxRacer',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.Rili',
        'new-name': 'net.sourceforge.Ri-li',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.Supertuxkart',
        'new-name': 'net.supertuxkart.SuperTuxKart',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.Torcs',
        'new-name': 'net.sourceforge.torcs',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.Tuxfootball',
        'new-name': 'net.sourceforge.TuxFootball',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.sourceforge.Warmux',
        'new-name': 'org.gna.Warmux',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'net.wz2100.Warzone2100',
        'new-name': 'net.wz2100.wz2100',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.armagetronad.Armagetronad',
        'new-name': 'org.armagetronad.ArmagetronAdvanced',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.codeblocks.App',
        'new-name': 'org.codeblocks.codeblocks',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.debian.Tuxpuck',
        'new-name': 'org.debian.TuxPuck',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.debian.alioth.tux4kids.Tuxmath',
        'new-name': 'com.tux4kids.tuxmath',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.debian.alioth.tux4kids.Tuxtype',
        'new-name': 'com.tux4kids.tuxtype',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.frozenbubble.FrozenBubble',
        'new-name': 'org.frozen_bubble.frozen-bubble',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.gimp.Gimp',
        'new-name': 'org.gimp.GIMP',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.gnome.Iagno',
        'new-name': 'org.gnome.iagno',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.gnome.Quadrapassel',
        'new-name': 'org.gnome.quadrapassel',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.gnome.Tetravex',
        'new-name': 'org.gnome.tetravex',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.gnome.people.dscorgie.Labyrinth',
        'new-name': 'com.github.labyrinth_team.labyrinth',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kalzium',
        'new-name': 'org.kde.kalzium',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kapman',
        'new-name': 'org.kde.kapman',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Katomic',
        'new-name': 'org.kde.katomic',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kblocks',
        'new-name': 'org.kde.kblocks',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kbounce',
        'new-name': 'org.kde.kbounce',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kbruch',
        'new-name': 'org.kde.kbruch',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kdiamond',
        'new-name': 'org.kde.kdiamond',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kgeography',
        'new-name': 'org.kde.kgeography',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kgoldrunner',
        'new-name': 'org.kde.kgoldrunner',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Khangman',
        'new-name': 'org.kde.khangman',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kigo',
        'new-name': 'org.kde.kigo',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Killbots',
        'new-name': 'org.kde.killbots',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kjumpingcube',
        'new-name': 'org.kde.kjumpingcube',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Klines',
        'new-name': 'org.kde.klines',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Knavalbattle',
        'new-name': 'org.kde.knavalbattle',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Knetwalk',
        'new-name': 'org.kde.knetwalk',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Ksame',
        'new-name': 'org.kde.klickety',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Ksquares',
        'new-name': 'org.kde.ksquares',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Ksudoku',
        'new-name': 'org.kde.ksudoku',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Ktuberling',
        'new-name': 'org.kde.ktuberling',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kubrick',
        'new-name': 'org.kde.kubrick',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Kwordquiz',
        'new-name': 'org.kde.kwordquiz',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.kde.Palapeli',
        'new-name': 'org.kde.palapeli',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.maemo.Numptyphysics',
        'new-name': 'io.thp.numptyphysics',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.marsshooter.Marsshooter',
        'new-name': 'net.sourceforge.mars-game',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.openarena.Openarena',
        'new-name': 'ws.openarena.OpenArena',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.openscad.Openscad',
        'new-name': 'org.openscad.OpenSCAD',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.processing.App',
        'new-name': 'org.processing.processingide',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.seul.Pingus',
        'new-name': 'org.seul.pingus',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.sugarlabs.Turtleblocks',
        'new-name': 'org.laptop.TurtleArtActivity',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'org.tuxfamily.Xmoto',
        'new-name': 'org.tuxfamily.XMoto',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Skype to flathub (T24741)
    {
        'name': 'com.microsoft.Skype',
        'new-name': 'com.skype.Client',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Minecraft to flathub (T26588)
    {
        'name': 'com.mojang.Minecraft',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Scratch to flathub (T26658)
    {
        'name': 'org.squeakland.Scratch',
        'new-name': 'edu.mit.Scratch',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate endlessnetwork apps to flathub (T25928)
    {
        'name': 'com.endlessm.MidnightmareTeddy',
        'new-name': 'com.endlessnetwork.MidnightmareTeddy',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    {
        'name': 'com.endlessm.MissileMath',
        'new-name': 'com.endlessnetwork.missilemath',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate BillardGL from eos-apps to flathub (T27078)
    {
        'name': 'de.billardgl.Billardgl',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Etoys from eos-apps to flathub (T28386)
    {
        'name': 'org.squeakland.Etoys',
        'kind': Flatpak.RefKind.APP,
        'old-branch': 'eos3',
        'new-branch': 'stable',
        'old-origin': 'eos-apps',
        'new-origin': 'flathub'
    },
    # migrate Eclipse from eos-apps to flathub (T28707)
    {
        "name": "org.eclipse.Eclipse",
        "new-name": "org.eclipse.Java",
        "kind": Flatpak.RefKind.APP,
        "old-branch": "eos3",
        "new-branch": "stable",
        "old-origin": "eos-apps",
        "new-origin": "flathub",
    },
    # migrate Firefox from eos-apps to flathub (T29746)
    {
        "name": "org.mozilla.Firefox",
        "new-name": "org.mozilla.firefox",
        "kind": Flatpak.RefKind.APP,
        "old-branch": "eos3",
        "new-branch": "stable",
        "old-origin": "eos-apps",
        "new-origin": "flathub",
        "mangle-metadata": mangle_firefox_metadata,
    },
    # migrate Hack apps from eos-apps to flathub (T31133)
    {
        "name": "com.hack_computer.Clubhouse",
        "kind": Flatpak.RefKind.APP,
        "old-branch": "eos3",
        "new-branch": "stable",
        "old-origin": "eos-apps",
        "new-origin": "flathub",
    },
    {
        "name": "com.hack_computer.Clippy.Extension",
        "kind": Flatpak.RefKind.RUNTIME,
        "old-branch": "eos3",
        "new-branch": "stable",
        "old-origin": "eos-apps",
        "new-origin": "flathub",
    },
    {
        "name": "com.hack_computer.OperatingSystemApp",
        "kind": Flatpak.RefKind.APP,
        "old-branch": "eos3",
        "new-branch": "stable",
        "old-origin": "eos-apps",
        "new-origin": "flathub",
    },
    {
        "name": "com.hack_computer.Sidetrack",
        "kind": Flatpak.RefKind.APP,
        "old-branch": "eos3",
        "new-branch": "stable",
        "old-origin": "eos-apps",
        "new-origin": "flathub",
    },
    {
        "name": "org.gnome.Solitaire",
        "new-name": "org.gnome.Aisleriot",
        "kind": Flatpak.RefKind.APP,
        "old-branch": "eos3",
        "new-branch": "stable",
        "old-origin": "eos-apps",
        "new-origin": "flathub",
    },
    {
        "name": "org.gnome.Freecell",
        "new-name": "org.gnome.Aisleriot",
        "kind": Flatpak.RefKind.APP,
        "old-branch": "eos3",
        "new-branch": "stable",
        "old-origin": "eos-apps",
        "new-origin": "flathub",
    },
]


def _filter_refs(refs, name=None, prefix=None, kind=None, arch=None,
                 branch=None, origin=None):
    ret = []
    for ref in refs:
        if kind and kind != ref.get_kind():
            continue
        if name and name != ref.get_name():
            continue
        if prefix and not ref.get_name().startswith(prefix):
            continue
        if arch and arch != ref.get_arch():
            continue
        if branch and branch != ref.get_branch():
            continue
        if origin and origin != ref.get_origin():
            continue
        ret.append(ref)
    return ret


def _filter_ostree_refs(refs, origin, kind, name='*', prefix=None, arch='*',
                        branch='*'):
    assert origin
    assert kind.value_nick in {'app', 'runtime'}

    if prefix:
        name = '{}*'.format(prefix)

    pattern = '{}:{}/{}/{}/{}'.format(origin, kind.value_nick, name, arch, branch)

    return fnmatch.filter(refs, pattern)


def mtree_add_file(repo, mtree, stream, file_info):
    file_info.set_file_type(Gio.FileType.REGULAR)
    file_info.set_attribute_uint32('unix::uid', 0)
    file_info.set_attribute_uint32('unix::gid', 0)
    file_info.set_attribute_uint32('unix::mode', 0o100644)

    _, content_stream, content_length = OSTree.raw_file_to_content_stream(
        stream, file_info, None
    )
    _, csum_bytes = repo.write_content(None, content_stream, content_length)
    csum = OSTree.checksum_from_bytes(csum_bytes)
    mtree.replace_file(file_info.get_name(), csum)


def mtree_load_file(repo, mtree, name):
    files = mtree.get_files()
    csum = files[name]
    _, stream, info, _ = repo.load_file(csum)
    info.set_name(name)
    return stream, info


def mtree_rename_file(repo, mtree, old_name, new_name):
    stream, info = mtree_load_file(repo, mtree, old_name)
    info.set_name(new_name)
    mtree_add_file(repo, mtree, stream, info)
    mtree.remove(old_name, False)


def mtree_load_keyfile(repo, mtree, name):
    stream, info = mtree_load_file(repo, mtree, name)
    data = stream.read_bytes(info.get_size())
    keyfile = GLib.KeyFile()
    keyfile.load_from_bytes(data,
                            GLib.KeyFileFlags.KEEP_COMMENTS |
                            GLib.KeyFileFlags.KEEP_TRANSLATIONS)
    return keyfile, info


def mtree_add_keyfile(repo, mtree, keyfile, info):
    keyfile_string = keyfile.to_data()[0]
    keyfile_bytes = keyfile_string.encode('utf-8')
    stream = Gio.MemoryInputStream.new_from_data(keyfile_bytes)
    info.set_size(len(keyfile_bytes))
    mtree_add_file(repo, mtree, stream, info)
    return keyfile_string


def _get_with_default(func, group, key, default):
    try:
        return func(group, key)
    except GLib.GError as e:
        if not e.matches(GLib.KeyFile.error_quark(),
                         GLib.KeyFileError.KEY_NOT_FOUND):
            raise
        return default


def mangle_desktop_file(desktop_file, old_name, old_id, new_id):
    # if the app relied on D-Bus activation, the desktop file ID
    # will no longer match the name that is implemented internally
    # so we have to disable it
    da = _get_with_default(
        desktop_file.get_boolean,
        GLib.KEY_FILE_DESKTOP_GROUP,
        GLib.KEY_FILE_DESKTOP_KEY_DBUS_ACTIVATABLE,
        False
    )
    if da:
        logging.debug('Removing DBusActivatable=true')
        desktop_file.remove_key(
            GLib.KEY_FILE_DESKTOP_GROUP,
            GLib.KEY_FILE_DESKTOP_KEY_DBUS_ACTIVATABLE
        )

    # update the icon name which matches the ID
    value = _get_with_default(
        desktop_file.get_string,
        GLib.KEY_FILE_DESKTOP_GROUP,
        GLib.KEY_FILE_DESKTOP_KEY_ICON,
        None
    )
    if value is not None:
        value = value.replace(old_id, new_id)
        desktop_file.set_string(
            GLib.KEY_FILE_DESKTOP_GROUP,
            GLib.KEY_FILE_DESKTOP_KEY_ICON,
            value
        )

    # Add old name to X-Flatpak-RenamedFrom
    renamed_from = _get_with_default(
        desktop_file.get_string_list,
        GLib.KEY_FILE_DESKTOP_GROUP,
        'X-Flatpak-RenamedFrom',
        []
    )
    if old_name not in renamed_from:
        renamed_from.append(old_name)
    desktop_file.set_string_list(GLib.KEY_FILE_DESKTOP_GROUP,
                                 'X-Flatpak-RenamedFrom',
                                 renamed_from)


def rename_exports(repo, mtree, old_id, new_id, parent_mtree=None, path='export'):
    logging.debug('Renaming files in {}'.format(path))
    vendor_prefixes = set()

    files = mtree.get_files()
    for name, csum in files.items():
        if name.startswith(old_id):
            new_name = new_id + name[len(old_id):]

            if path == 'export/share/applications' and \
               name.endswith('.desktop'):
                logging.info('Rewriting {} as {}'.format(name, new_name))
                desktop_file, info = mtree_load_keyfile(repo, mtree, name)

                mangle_desktop_file(desktop_file, name, old_id, new_id)

                info.set_name(new_name)
                mtree_add_keyfile(repo, mtree, desktop_file, info)
                mtree.remove(name, False)
            elif (
                os.path.dirname(path) == 'export/share/applications' and
                name.endswith('.desktop')
            ):
                # Vendor subdirectory, as used by kde4-... apps
                vendor = os.path.basename(path)
                vendor_prefixes.add(vendor)

                logging.info('Rewriting {}/{} as {}'.format(vendor, name, new_name))
                desktop_file, info = mtree_load_keyfile(repo, mtree, name)

                # kde4/foo.desktop is treated as kde4-foo.desktop, so put that as the
                # old name into the rewritten file.
                old_name = vendor + '-' + name
                mangle_desktop_file(desktop_file, old_name, old_id, new_id)

                info.set_name(new_name)
                mtree_add_keyfile(repo, parent_mtree, desktop_file, info)
                mtree.remove(name, False)
            elif path == 'export/share/dbus-1/services' and name.endswith('.service'):
                logging.info('Not changing exported D-Bus service {}'.format(name))
            else:
                logging.info('Renaming {} to {}'.format(name, new_name))
                mtree_rename_file(repo, mtree, name, new_name)
        else:
            logging.debug('Not changing {}'.format(name))

    dirs = mtree.get_subdirs()
    for k, v in dirs.items():
        vendor_prefixes.update(
            rename_exports(repo, v, old_id, new_id, mtree, os.path.join(path, k))
        )

    return vendor_prefixes


FLATPAK_METADATA_FILE = 'metadata'
FLATPAK_METADATA_APPLICATION = 'Application'
FLATPAK_METADATA_KEY_NAME = 'name'
FLATPAK_METADATA_GROUP_SESSION_BUS_POLICY = 'Session Bus Policy'
FLATPAK_METADATA_BUS_POLICY_OWN = 'own'
FLATPAK_METADATA_GROUP_EXTRA_DATA = 'Extra Data'
FLATPAK_METADATA_GROUP_CONTEXT = "Context"
FLATPAK_METADATA_FILESYSTEMS = "filesystems"
FLATPAK_METADATA_PERSISTENT = "persistent"


def rewrite_metadata(repo, mtree, old_id, new_id, mangle_metadata=None):
    metadata, info = mtree_load_keyfile(repo, mtree, FLATPAK_METADATA_FILE)

    if old_id != new_id:
        metadata.set_string(FLATPAK_METADATA_APPLICATION,
                            FLATPAK_METADATA_KEY_NAME, new_id)

        # allow the app to own its old bus name (if the app was relying on this,
        # it would lose the right to this name with the new ID)
        metadata.set_string(FLATPAK_METADATA_GROUP_SESSION_BUS_POLICY,
                            old_id, FLATPAK_METADATA_BUS_POLICY_OWN)

    # extra data apps are handled by removing the [Extra Data] from the metadata
    # of the migrated app to allow the deploy to be done offline. instead,
    # after the new app is installed, this script will copy the /app/extra
    # folder from the pre-migration deploy folder. the migration function
    # examines the commit metadata of the app being migrated to determine
    # whether the re-deploy has to carry out this extra step.
    try:
        metadata.remove_group(FLATPAK_METADATA_GROUP_EXTRA_DATA)
        logging.info('Removing [Extra Data] from {} metadata'.format(old_id))
    except GLib.GError as e:
        if not e.matches(GLib.key_file_error_quark(),
                         GLib.KeyFileError.GROUP_NOT_FOUND):
            raise

    if mangle_metadata is not None:
        mangle_metadata(metadata)

    metadata_str = mtree_add_keyfile(repo, mtree, metadata, info)

    if old_id == new_id:
        vendor_prefixes = frozenset()
        return metadata_str, vendor_prefixes

    dirs = mtree.get_subdirs()
    if 'export' in dirs:
        export_mtree = dirs['export']
        vendor_prefixes = rename_exports(repo, export_mtree, old_id, new_id, mtree)
    else:
        logging.warning('Cannot find export dir in {} mtree'
                        .format(old_id))
        vendor_prefixes = frozenset()

    return metadata_str, vendor_prefixes


COMMIT_SUBJECT_INDEX = 3
COMMIT_BODY_INDEX = 4


def copy_commit(repo, src_rev, dest_ref,
                old_app_id=None, new_app_id=None,
                collection_id=None,
                mangle_metadata=None):
    """Copy commit src_rev to dest_ref
    This makes the new commit at dest_ref have the proper collection
    binding for this repo. The caller is expected to manage the
    ostree transaction.
    This is like "flatpak build-commit-from", but we need more
    control over the transaction.
    """
    logging.debug('Copying commit %s to %s', src_rev, dest_ref)

    _, src_root, _ = repo.read_commit(src_rev)
    _, src_variant, src_state = repo.load_commit(src_rev)

    # Only copy normal commits
    if src_state != 0:
        raise Exception('Cannot copy irregular commit {}'
                        .format(src_rev))

    # If the dest ref exists, use the current commit as the new
    # commit's parent
    _, dest_parent = repo.resolve_rev_ext(
        dest_ref, allow_noent=True,
        flags=OSTree.RepoResolveRevExtFlags.NONE)
    if dest_parent is not None:
        logging.info('Using %s as new commit parent', dest_parent)

    # Make a copy of the commit metadata to update. Like flatpak
    # build-commit-from, the detached metadata is not copied since
    # the only known usage is for GPG signatures, which would become
    # invalid.
    commit_metadata = GLib.VariantDict.new(src_variant.get_child_value(0))

    # Set the collection binding if the repo has a collection ID,
    # otherwise remove it
    if collection_id is not None:
        commit_metadata.insert_value(
            OSTree.COMMIT_META_KEY_COLLECTION_BINDING,
            GLib.Variant('s', collection_id))
    else:
        commit_metadata.remove(
            OSTree.COMMIT_META_KEY_COLLECTION_BINDING)

    # Include the destination ref in the ref bindings
    ref_bindings = commit_metadata.lookup_value(
        OSTree.COMMIT_META_KEY_REF_BINDING,
        GLib.VariantType('as'))
    if ref_bindings is None:
        ref_bindings = []
    ref_bindings = set(ref_bindings)
    ref_bindings.add(dest_ref)
    commit_metadata.insert_value(
        OSTree.COMMIT_META_KEY_REF_BINDING,
        GLib.Variant('as', sorted(ref_bindings)))

    # Add flatpak specific metadata. xa.ref is deprecated, but some
    # flatpak clients might expect it. xa.from_commit will be used
    # by the app verifier to make sure the commit it sent actually
    # got there
    commit_metadata.insert_value('xa.ref',
                                 GLib.Variant('s', dest_ref))
    commit_metadata.insert_value('xa.from_commit',
                                 GLib.Variant('s', src_rev))

    # Copy other commit data from source commit
    commit_subject = src_variant[COMMIT_SUBJECT_INDEX]
    commit_body = src_variant[COMMIT_BODY_INDEX]
    commit_time = OSTree.commit_get_timestamp(src_variant)

    # Make the new commit assuming the caller started a transaction
    mtree = OSTree.MutableTree.new()
    repo.write_directory_to_mtree(src_root, mtree, None)

    if old_app_id and new_app_id:
        metadata, vendor_prefixes = rewrite_metadata(repo, mtree, old_app_id,
                                                     new_app_id, mangle_metadata)
        commit_metadata.insert_value('xa.metadata',
                                     GLib.Variant('s', metadata))

        # unconditionally remove extra data, the caller determines whether the
        # the extra data dir needs migrating
        commit_metadata.remove('xa.extra-data-sources')
    else:
        vendor_prefixes = frozenset()

    # Convert from GVariantDict to GVariant vardict
    commit_metadata = commit_metadata.end()

    _, dest_root = repo.write_mtree(mtree)
    _, dest_checksum = repo.write_commit_with_time(dest_parent,
                                                   commit_subject,
                                                   commit_body,
                                                   commit_metadata,
                                                   dest_root,
                                                   commit_time)
    logging.info('Created new commit %s', dest_checksum)

    return dest_checksum, vendor_prefixes


def migrate_extra_data(old_ref, new_ref):
    old_deploy = old_ref.get_deploy_dir()
    old_extra = os.path.join(old_deploy, 'files', 'extra')

    new_deploy = new_ref.get_deploy_dir()
    new_extra = os.path.join(new_deploy, 'files', 'extra')

    if not os.path.exists(old_extra):
        return

    if os.path.exists(new_extra):
        # either the user manually installed Skype from Flathub, or we got
        # this far but died before uninstalling the old ref last time
        logging.info('%s already has extra data dir', new_ref.format_ref())
    else:
        logging.debug('Copying extra data from %s to %s',
                      old_ref.format_ref(), new_ref.format_ref())
        subprocess.check_call(['cp', '-alT', old_extra, new_extra + '.tmp'])
        os.rename(new_extra + '.tmp', new_extra)


def _migrate_installed_flatpaks():
    insts = Flatpak.get_system_installations()
    inst = insts[0]
    refs = inst.list_installed_refs()

    triggers_needed = False

    # we need to also operate directly on the ostree refs that shadow each
    # flatpak so we don't leave behind local refs to the old origin and branch.
    repo = OSTree.Repo.new(inst.get_path().get_child('repo'))
    repo.open()
    _, ostree_refs = repo.list_refs()  # dictionary of ref -> commit

    for migrate in FLATPAKS_TO_MIGRATE:
        kind = migrate['kind']
        old_origin = migrate['old-origin']

        name = migrate.get('name')
        prefix = migrate.get('prefix')
        assert name or prefix

        old_branch = migrate.get('old-branch')

        # we build what should be the same list twice - from the flatpak
        # perspective by examining the installation and deployed apps, and from
        # the ostree perspective by examining the refs in the repo. ordinarily
        # we would only see them differ due to a bug in previous versions of
        # this script where the refs were not renamed during a migration.
        # everything in the matching_ostree_refs list will be copied to its
        # new name if we change (or previously changed) the branch or origin,
        # and then deleted after the deployed apps are updated. (T22763)
        matching_refs = _filter_refs(refs, name=name, prefix=prefix, kind=kind,
                                     branch=old_branch, origin=old_origin)
        matching_ostree_refs = _filter_ostree_refs(ostree_refs.keys(),
                                                   old_origin, kind,
                                                   name=name, prefix=prefix,
                                                   branch=old_branch)

        # also search for name.* runtimes to migrate to catch extensions
        # unless we are already searching runtimes by prefix - in this case
        # we will already have found the matching extensions
        if name or kind == Flatpak.RefKind.APP:
            if not prefix:
                prefix = name + '.'
            matching_refs += _filter_refs(refs, prefix=prefix,
                                          kind=Flatpak.RefKind.RUNTIME,
                                          branch=old_branch, origin=old_origin)

            matching_ostree_refs += _filter_ostree_refs(ostree_refs.keys(),
                                                        old_origin,
                                                        Flatpak.RefKind.RUNTIME,
                                                        prefix=prefix,
                                                        branch=old_branch)

        if len(matching_refs) == 0:
            if name:
                logging.debug('Found no matches to migrate for {} {}'
                              .format(kind.value_nick, name))
            else:
                logging.debug('Found no matches to migrate for {} {}*'
                              .format(kind.value_nick, prefix))

        if 'new-name' in migrate:
            assert 'name' in migrate
            if len(matching_refs) > 1:
                logging.warning('Avoiding undefined behaviour, cannot rename '
                                'an app with extensions: {}'.format(name))
                continue

        for ref in matching_refs:
            origin = ref.get_origin()
            kind = ref.get_kind()
            name = ref.get_name()
            arch = ref.get_arch()
            branch = ref.get_branch()
            old_ostree_ref = '{}:{}'.format(origin, ref.format_ref())

            new_origin = migrate.get('new-origin', origin)
            new_name = migrate.get('new-name', name)
            new_branch = migrate.get('new-branch', branch)
            new_ostree_ref = '{}:{}/{}/{}/{}'.format(new_origin, kind.value_nick,
                                                     new_name, arch, new_branch)
            mangle_metadata = migrate.get('mangle-metadata')

            logging.info('Found {} to migrate to {}'.format(old_ostree_ref,
                         new_ostree_ref))

            # if we're not changing anything, why are we here?
            assert (old_ostree_ref != new_ostree_ref)

            new_refs = _filter_refs(refs, name=new_name, kind=kind,
                                    arch=arch, branch=new_branch)
            if new_refs:
                new_ref = new_refs[0]
            else:
                new_ref = None

            try:
                vendor_prefixes = frozenset()

                # copy the old ref to the new one, pointing at the same commit
                if old_ostree_ref in ostree_refs and not new_ref:
                    logging.info('Copying OSTree ref {} [{}] to {}'
                                 .format(old_ostree_ref,
                                         ostree_refs[old_ostree_ref],
                                         new_ostree_ref))
                    repo.prepare_transaction()
                    try:
                        kwargs = {}
                        kwargs["mangle_metadata"] = mangle_metadata
                        if kind == Flatpak.RefKind.APP:
                            kwargs['old_app_id'] = name
                            kwargs['new_app_id'] = new_name
                        new_commit, vendor_prefixes = copy_commit(
                            repo,
                            ostree_refs[old_ostree_ref],
                            new_ostree_ref.split(':')[1],
                            **kwargs,
                        )
                        repo.commit_transaction()
                    finally:
                        repo.abort_transaction()
                    repo.set_ref_immediate(new_origin,
                                           new_ostree_ref.split(':')[1],
                                           new_commit)
                    ostree_refs[new_ostree_ref] = new_commit

                if new_branch != branch or new_name != name:
                    if new_ref:
                        logging.info('Found already installed: {}'
                                     .format(new_ref.format_ref()))

                        # adopt installed origin if app was already installed
                        new_origin = new_ref.get_origin()
                    else:
                        logging.info(
                            'Deploying with new ref: {}'.format(new_ostree_ref)
                        )
                        new_ref = inst.install_full(Flatpak.InstallFlags.NO_PULL |
                                                    Flatpak.InstallFlags.NO_TRIGGERS,
                                                    new_origin, kind, new_name,
                                                    arch, new_branch)
                        triggers_needed = True
                        refs.append(new_ref)

                    migrate_extra_data(ref, new_ref)

                    try:
                        logging.info('Uninstalling old ref: {}'.format(old_ostree_ref))
                        inst.uninstall_full(Flatpak.UninstallFlags.NO_PRUNE |
                                            Flatpak.UninstallFlags.NO_TRIGGERS,
                                            kind, name, arch, branch)

                        triggers_needed = True
                    except GLib.Error as err:
                        if err.matches(Flatpak.error_quark(),
                                       Flatpak.Error.NOT_INSTALLED):
                            logging.warning("Uninstall failed", exc_info=True)
                        else:
                            raise

                    # TODO: ideally we would defer the removal of the old app
                    # until after the deploy file edits had been completed, so
                    # we could pick up any prior previous-ids and carry them
                    # over. however at present we are performing no such repeat
                    # migrations (and may never need to).

                    ref = new_ref
                    origin = new_origin

                deploy_dir = ref.get_deploy_dir()
                deploy = os.path.join(deploy_dir, 'deploy')

                if new_origin != origin:
                    update_deploy_file_with_origin(deploy, new_origin)

                # If, for app "org.example.Foo", we found a vendor-prefixed desktop file
                # like exports/share/applications/kde4/org.example.Foo.Bar.desktop,
                # pretend that "kde4-org.example.Foo" was an old ID of this application.
                # This will allow the X-Flatpak-RenamedFrom key to continue to include
                # the vendor-prefixed name across app updates.
                if new_name != name or vendor_prefixes:
                    previous_ids = {name}
                    for vendor_prefix in vendor_prefixes:
                        previous_ids.add(vendor_prefix + '-' + name)
                    update_deploy_file_with_previous_ids(deploy, previous_ids)

            except Exception:
                logging.exception('Failure applying migration to {}'
                                  .format(name))

                # preserve the old ref so that the migration can be re-attempted
                if old_ostree_ref in matching_ostree_refs:
                    matching_ostree_refs.remove(old_ostree_ref)

        # any refs we copied, along with any matches for apps we migrated
        # in the past, should be deleted.
        for old_ostree_ref in matching_ostree_refs:
            try:
                logging.info('Deleting old/stale OSTree ref {} [{}]'
                             .format(old_ostree_ref,
                                     ostree_refs[old_ostree_ref]))
                repo.set_ref_immediate(old_ostree_ref.split(':')[0],
                                       old_ostree_ref.split(':')[1],
                                       None)
            except Exception:
                logging.exception('Failure deleting ref {}'
                                  .format(old_ostree_ref))
                continue

    if triggers_needed:
        inst.run_triggers()


if __name__ == '__main__':
    # Send logging messages both to the console and the journal
    handlers = [journal.JournalHandler()]
    if not GLib.log_writer_is_journald(sys.stderr.fileno()):
        handlers.append(logging.StreamHandler(sys.stderr))
    logging.basicConfig(level=logging.INFO, handlers=handlers)

    parser = argparse.ArgumentParser()
    parser.add_argument('--debug', '-d', dest='debug', action='store_true')
    parsed_args, otherargs = parser.parse_known_args()

    if parsed_args.debug:
        logging.root.setLevel(logging.DEBUG)

    # ensure default remotes exist, such as eos-sdk and flathub
    _add_flatpak_repos()

    # remove any unused refs, remotes and runtimes
    _remove_ostree_refs()
    _remove_remotes()
    _remove_runtimes()

    # fix incorrect remote URLs
    _fix_remote_urls()

    # migrate remotes
    _migrate_remotes()

    # move apps and runtimes between branches and origins as specified above
    _migrate_installed_flatpaks()