issuebot

#!/usr/bin/python3

from __future__ import annotations

import datetime
import sys
import os
import pathlib
import subprocess
import shutil
from typing import List, Optional, Set, Iterator
from collections import defaultdict
import re

from ruamel.yaml import YAML
from ruamel.yaml.util import load_yaml_guess_indent

from github import GitHub, Issue, IssueStateReason
from myutils import file_lock

from lilac2.lilacyaml import (
  iter_pkgdir, load_lilac_yaml as _load_lilac_yaml,
)

from webhooks.issue import parse_issue_text
from webhooks.util import OrphanResult, annotate_maints, Dependent
from webhooks import lilac, config

REQUEST_WAITING_TIME = datetime.timedelta(days=30)
ORPHANING_WAITING_TIME = datetime.timedelta(days=14)
OFFICIAL_WAITING_TIME = datetime.timedelta(days=4)
LICENSE_WAITING_TIME = datetime.timedelta(days=14)

LILAC_LOCK = os.path.expanduser('~/.lilac/.lock')
REPO = pathlib.Path('~/archgitrepo/archlinuxcn').expanduser()

PACKAGE_LIST_BREAKDOWN = 5

def git_push() -> None:
  while True:
    try:
      subprocess.check_output(['git', 'push'], cwd=REPO)
      break
    except subprocess.CalledProcessError as e:
      if b'non-fast-forward' in e.output or b'fetch first' in e.output:
        subprocess.check_call(["git", "pull", "--rebase"])
      else:
        sys.stdout.write(e.output)
        raise

def filter_issue(it: Iterator[Issue]) -> Iterator[Issue]:
  for issue in it:
    if 'no-lilac' in issue.labels:
      continue
    yield issue

def process_package_requests(
  gh: GitHub, repo: str, now: datetime.datetime,
) -> None:
  for issue in filter_issue(gh.get_repo_issues(repo, labels='package-request')):
    if now - issue.updated_at > REQUEST_WAITING_TIME:
      print(f'Marking {issue} as failed')
      issue.comment('''\
请求太久无维护者接手，关闭。

This request has been waiting for its maintainer for too long, closing.''')
      issue.add_labels(['request-failed'])
      issue.close(IssueStateReason.not_planned)

def process_one_orphaned(
  name: str,
  all_orphans: List[str],
) -> OrphanResult:
  p = REPO / name
  dependents = lilac.find_dependent_packages_ext(REPO, name)
  # FIXME: don't remove if d.pkgbase is to be orphaned but not removed
  dependents = [d for d in dependents if d.pkgbase not in all_orphans]
  if dependents:
    remove_maintainers(p)
    return OrphanResult.Depended(dependents)

  try:
    # make directories writable so we can delete their contents
    # Go may create such files like pkg/mod/github.com/go-sql-driver/mysql@v1.5.0
    subprocess.run(
      ['find', '.', '-type', 'd', '!', '-perm', '/u=w', '-exec', 'chmod', 'u+w', '{}', '+'],
      cwd = p,
    )
    shutil.rmtree(p)
  except FileNotFoundError:
    return OrphanResult.NotFound
  else:
    subprocess.check_call(['git', 'add', name], cwd=REPO)
    return OrphanResult.Removed

def process_orphaned_packages(
  gh: GitHub, repo: str, now: datetime.datetime,
) -> None:
  issues = [issue for issue in filter_issue(gh.get_repo_issues(repo, labels='orphaning'))
            if now - issue.created_at > ORPHANING_WAITING_TIME
            and 'seen-by-lilac' not in issue.labels]

  for issue in issues:
    _issuetype, packages = parse_issue_text(issue.body)
    print(f'Removing orphaned {issue}, packages: {packages}')
    if not packages:
      issue.comment('''\
lilac 无法解析出涉及的软件包名，请手动处理。

lilac can't parse out the relevant package names, please handle manually.''')
      issue.add_labels(['seen-by-lilac'])
      continue

    results = {}

    with file_lock(LILAC_LOCK):
      subprocess.check_output(['git', 'pull'], cwd=REPO)

      for name in packages:
        results[name] = process_one_orphaned(name, packages)

      removed = [x for x, r in results.items()
                 if r == OrphanResult.Removed]
      depended = {x: r.dependents
                  for x, r in results.items()
                  if r.ty == 'Depended'}
      notfound = [x for x, r in results.items()
                  if r == OrphanResult.NotFound]

      if removed or depended:
        if not depended and len(removed) <= PACKAGE_LIST_BREAKDOWN:
          # simple case
          msg = f'{", ".join(removed)}: orphaned, removing. closes #{issue.number}'
        else:
          removed_pkg_msg = ''.join(f'- {x}\n' for x in removed)
          if depended:
            def ds_str(ds: List[Dependent]) -> str:
              return ', '.join(
                annotate_maints(x.pkgbase, x.maintainers)
                for x in ds
              )
            depended_pkg_msg = ''.join(f'- {x}: depended by {ds_str(ds)}\n' for x, ds in depended.items())
          else:
            depended_pkg_msg = ''
          notfound_pkg_msg = ''.join(f'- {x}\n' for x in notfound)

          removed_msg = f'Removing {len(removed)} orphaned packages:\n\n{removed_pkg_msg}\n'
          depended_msg = f'Removing maintainers from {len(depended)} orphaned packages:\n\n{depended_pkg_msg}\n'

          if removed and not depended:
            head_msg = removed_msg
          elif depended and not removed:
            head_msg = depended_msg
          else:
            head_msg = (f'Processing {len(removed) + len(depended)} orphaned packages:\n\n'
                        + removed_msg + depended_msg)
          if notfound:
            head_msg += f'The following packages are not found:\n\n{notfound_pkg_msg}\n'

          close_msg = f'closes #{issue.number}'
          msg = head_msg + close_msg

        subprocess.check_output(['git', 'commit', '-m', msg], cwd=REPO)
        git_push()

    if not removed and not depended:
      issue.comment('相关包已被删除。\n\nPackages already removed.')
      issue.close()
      # else we've closed the issue by a commit

def remove_repo_depends(pkgdir: pathlib.Path, packages: List[str]) -> bool:
  # use ruamel.yaml for yaml manipulation with preserving indents and comments
  lilac_yaml_path = pkgdir / 'lilac.yaml'

  with open(lilac_yaml_path) as f:
    lilac_yaml, indent, block_seq_indent = load_yaml_guess_indent(f.read())

  repo_depends = lilac_yaml.get('repo_depends', [])
  if not repo_depends:
    return False

  # Find out all repo_depends entries to remove. Not using list comprehension
  # here so that comments are preserved.
  target_indexes = []
  for idx, repo_depend in enumerate(repo_depends):
    if isinstance(repo_depend, dict):
      repo_depend = list(repo_depend.keys())[0]
    if repo_depend in packages:
      target_indexes.append(idx)

  if target_indexes:
    for target_idx in sorted(target_indexes, reverse=True):
      del lilac_yaml['repo_depends'][target_idx]
    if len(lilac_yaml['repo_depends']) == 0:
      del lilac_yaml['repo_depends']
    with open(lilac_yaml_path, 'w') as f:
      yaml = YAML()
      yaml.preserve_quotes = True
      yaml.default_flow_style = None
      yaml.indent(sequence=indent, offset=block_seq_indent)
      yaml.dump(lilac_yaml, stream=f)
    subprocess.check_call(['git', 'add', pkgdir.name], cwd=REPO)
    return True
  else:
    return False

def remove_maintainers(pkgdir: pathlib.Path) -> None:
  # use ruamel.yaml for yaml manipulation with preserving indents and comments
  lilac_yaml_path = pkgdir / 'lilac.yaml'

  with open(lilac_yaml_path) as f:
    lilac_yaml, indent, block_seq_indent = load_yaml_guess_indent(f.read())

  maintainers = lilac_yaml.get('maintainers', [])
  if not maintainers:
    return

  lilac_yaml['maintainers'] = []

  with open(lilac_yaml_path, 'w') as f:
    yaml = YAML()
    yaml.preserve_quotes = True
    yaml.default_flow_style = None
    yaml.indent(sequence=indent, offset=block_seq_indent)
    yaml.dump(lilac_yaml, stream=f)
  subprocess.check_call(['git', 'add', pkgdir.name], cwd=REPO)

def process_in_official(
  gh: GitHub, repo: str, now: datetime.datetime,
) -> None:
  issues = [
    issue for issue in filter_issue(gh.get_repo_issues(
      repo, labels='in-official-repos'))
    if now - issue.updated_at > OFFICIAL_WAITING_TIME
    and 'seen-by-lilac' not in issue.labels
  ]

  for issue in issues:
    print(f'Removing in-official-repos {issue}')
    _issuetype, packages = parse_issue_text(issue.body)
    if not packages:
      issue.comment('''\
lilac 无法解析出涉及的软件包名，请手动处理。

lilac can't parse out the relevant package names, please handle manually.''')
      issue.add_labels(['seen-by-lilac'])
      continue

    changed = False

    with file_lock(LILAC_LOCK):
      subprocess.check_call(['git', 'pull'], cwd=REPO)

      for pkgdir in iter_pkgdir(REPO):
        try:
          if remove_repo_depends(pkgdir, packages):
            changed = True
        except Exception:
          print(f'Error processing {pkgdir}:', file=sys.stderr)
          raise

      for name in packages:
        try:
          shutil.rmtree(REPO / name)
        except FileNotFoundError:
          pass
        else:
          subprocess.check_call(['git', 'add', name], cwd=REPO)
          changed = True

      if changed:
        msg = None
        if len(packages) > PACKAGE_LIST_BREAKDOWN:
          affected = "\n".join(f"- {x}" for x in packages)
          msg = f'''Removing {len(packages)} packages \
and corresponding repo_depends entries because they are already in official repos.

Affected packages:
{affected}

closes #{issue.number}'''
        else:
          msg = f'{", ".join(packages)}: in official repos, removing packages and repo_depends entries if any. closes #{issue.number}'
        subprocess.check_call(['git', 'commit', '-m', msg], cwd=REPO)
        git_push()

    if changed:
      issue.comment('''\
已自动删除。

Automatically removed.''')
    else:
      issue.comment('''\
已被删除。

Already removed.''')
      issue.close()

def process_license_issue(
  gh: GitHub, repo: str, now: datetime.datetime,
) -> None:
  issues = [
    issue for issue in filter_issue(gh.get_repo_issues(
      repo, labels='license'))
    if now - issue.updated_at > LICENSE_WAITING_TIME
    and 'seen-by-lilac' not in issue.labels
  ]

  for issue in issues:
    print(f'Removing license issue: {issue}')
    _issuetype, packages = parse_issue_text(issue.body)
    if not packages and issue.is_pull:
      issue.comment('''Unresolved license issues, closing.''')
      issue.close()
      continue
    elif not packages:
      issue.comment('''\
lilac 无法解析出涉及的软件包名，请手动处理。

lilac can't parse out the relevant package names, please handle manually.''')
      issue.add_labels(['seen-by-lilac'])
      continue

    changed = False

    with file_lock(LILAC_LOCK):
      subprocess.check_call(['git', 'pull'], cwd=REPO)

      for pkgdir in iter_pkgdir(REPO):
        try:
          if remove_repo_depends(pkgdir, packages):
            changed = True
        except Exception:
          print(f'Error processing {pkgdir}:', file=sys.stderr)
          raise

      for name in packages:
        try:
          shutil.rmtree(REPO / name)
        except FileNotFoundError:
          pass
        else:
          subprocess.check_call(['git', 'add', name], cwd=REPO)
          changed = True

      if changed:
        msg = None
        if len(packages) > PACKAGE_LIST_BREAKDOWN:
          affected = "\n".join(f"- {x}" for x in packages)
          msg = f'''Removing {len(packages)} packages \
and corresponding repo_depends entries because license issues.

Affected packages:
{affected}

closes #{issue.number}'''
        else:
          msg = f'{", ".join(packages)}: license issues, removing packages and repo_depends entries if any. closes #{issue.number}'
        subprocess.check_call(['git', 'commit', '-m', msg], cwd=REPO)
        git_push()

    if changed:
      issue.comment('''\
已自动删除。

Automatically removed.''')
    else:
      issue.comment('''\
已被删除。

Already removed.''')
      issue.close()

def _find_orphaning_issue(repo: pathlib.Path, pkg: str) -> Optional[int]:
  out = subprocess.check_output([
    'git', 'log', f'--author={config.MYMAIL}',
    '-1', '-P', '--all-match',
    '--grep=orphaned packages', r'--grep=closes #\d+',
    '--', f'{pkg}/lilac.yaml',
  ], text=True, cwd=REPO)

  out = out.strip()
  m = re.search(r'\bcloses #(\d+)', out)
  if m:
    return int(m.group(1))
  else:
    return None

def load_lilac_yaml(x):
  try:
    return _load_lilac_yaml(x)
  except Exception:
    return None

def find_orphaned_unmaintained_packages(
  repopath: pathlib.Path,
) -> Set[str]:
  lilac_yamls = {
    x.name: ly
    for x in iter_pkgdir(repopath)
    if (ly := load_lilac_yaml(x)) is not None
  }
  orphaned_pkgs = {
    pkg
    for pkg, ly in lilac_yamls.items()
    if not ly.get('maintainers')
  }

  depended_pkgs: Set[str] = set()
  for name, ly in lilac_yamls.items():
    if name in orphaned_pkgs:
      continue
    depended_pkgs.update(
      d for d, _ in
      ly.get('repo_depends', ())
      if d in orphaned_pkgs
    )

  removing_pkgs = orphaned_pkgs - depended_pkgs
  return removing_pkgs

def try_remove_orphaned(gh: GitHub, repo: str) -> None:
  removing_pkgs = find_orphaned_unmaintained_packages(REPO)
  if not removing_pkgs:
    return

  for pkg in removing_pkgs:
    shutil.rmtree(REPO / pkg)
    subprocess.check_call(['git', 'add', pkg], cwd=REPO)

  issue_to_pkgs = defaultdict(list)

  for pkg in removing_pkgs:
    issue_nr = _find_orphaning_issue(REPO, pkg)
    if issue_nr:
      issue_to_pkgs[issue_nr].append(pkg)

  for issue_nr, pkgs in issue_to_pkgs.items():
    gh.add_issue_comment(
      repo, issue_nr,
      f'{", ".join(pkgs)} removed because no packages depend on them any more',
    )

  if len(issue_to_pkgs) == 1:
    issue_nr, pkgs = next(iter(issue_to_pkgs.items()))
    msg = f'Removing packages {", ".join(pkgs)} which were orphaned in #{issue_nr}'
  elif not issue_to_pkgs:
    msg = f'Removing orphaned packages {", ".join(pkgs)} that no packages depend on'
  else:
    msg_part = [f'Removing {len(removing_pkgs)} orphaned packages that no packages depend on\n']
    leftover = removing_pkgs.copy()
    for issue_nr, pkgs in issue_to_pkgs.items():
      leftover.difference_update(pkgs)
      msg_part.append(f'* {", ".join(pkgs)} (orphaned by #{issue_nr})')
    if leftover:
      msg_part.append(f'* {", ".join(leftover)} (no issue found)')
    msg = '\n'.join(msg_part)

  subprocess.check_call(
    ['git', 'commit', '-m', msg], cwd=REPO)
  git_push()

def main() -> None:
  token = os.environ['GITHUB_TOKEN']
  gh = GitHub(token)
  repo = 'archlinuxcn/repo'
  now = datetime.datetime.now(datetime.timezone.utc)

  process_package_requests(gh, repo, now)
  process_orphaned_packages(gh, repo, now)
  process_in_official(gh, repo, now)
  process_license_issue(gh, repo, now)
  try_remove_orphaned(gh, repo)

if __name__ == '__main__':
  main()