Feature: Migrate docs

.codespellignore

@@ -0,0 +1 @@
+pullrequest

.github/workflows/integration_test.yaml

@@ -9,7 +9,7 @@ jobs:
     secrets: inherit
     with:
       pre-run-script: tests/integration/pre_run.sh
-      modules: '["discourse", "init"]'
+      modules: '["discourse", "reconcile", "migrate"]'
   self-tests:
     runs-on: ubuntu-22.04
     steps:
@@ -55,7 +55,7 @@ jobs:
         run: echo '${{ steps.selfTestDraft.outputs.urls_with_actions }}'
       - name: Check draft
         run: |
-          sudo apt update && sudo apt install python3-pip
+          sudo apt update && sudo apt install python3-pip git
           pip3 install -r requirements.txt
           ./discourse_check_cleanup.py --action check-draft --action-kwargs '{"expected_url_results": []}' '${{ steps.selfTestDraft.outputs.urls_with_actions }}' '${{ steps.selfTestDraft.outputs.discourse_config }}'
       - name: Create self test

Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.10-slim
 
+RUN apt-get update && apt-get install -y --no-install-recommends git=1:2.30.*
+
 RUN mkdir /usr/src/app
 WORKDIR /usr/src/app
 COPY requirements.txt /usr/src/app

README.md

@@ -8,6 +8,8 @@ charmhub.
 
 ## Getting Started
 
+### Sync docs
+
 1. Create the `docs` folder in the repository.
 2. Optionally, create a file `docs/index.md` for any content you would like to
     display above the navigation table on discourse. This content does not get
@@ -75,6 +77,37 @@ charmhub.
     is also available under the `index_url` output of the action. This needs to
     be added to the `metadata.yaml` under the `docs` key.
 
+### Migrate docs
+
+1. Create a `docs` key in `metadata.yaml` with the link to the documentation on
+    charmhub.
+2. Add the action to your desired workflow as mentioned in step 5 of
+    [Sync docs section](#sync-docs) with github_token. For example:
+
+    ```yaml
+    jobs:
+      publish-docs:
+        name: Publish docs
+        runs-on: ubuntu-22.04
+        steps:
+          - uses: actions/checkout@v3
+          - name: Publish documentation
+            uses: canonical/upload-charm-docs@main
+            id: publishDocumentation
+            with:
+              discourse_host: discourse.charmhub.io
+              discourse_api_username: ${{ secrets.DISCOURSE_API_USERNAME }}
+              discourse_api_key: ${{ secrets.DISCOURSE_API_KEY }}
+              github_token: ${{ secrets.GITHUB_TOKEN }}
+    ```
+
+    a branch name with `upload-charm-docs/migrate` will be created and a pull 
+    request named `[upload-charm-docs] Migrate charm docs` will be created 
+    towards the working branch the workflow was triggered with. 
+    In order to ensure that the branches can be created successfully, please 
+    make sure that there are no existing branches clashing with the name above.
+    Please note that `dry_run` parameter has no effect on migrate mode.
+
 The action will now compare the discourse topics with the files and directories
 under the `docs` directory and make any changes based on differences.
 Additional recommended steps:

action.yaml

@@ -16,7 +16,7 @@ inputs:
     required: false
     type: boolean
   discourse_host:
-    description: The discourse host name.
+    description: The base path(hostname) to the discourse server.
     required: true
     type: string
   discourse_api_username:
@@ -34,6 +34,12 @@ inputs:
     default: 41
     required: false
     type: integer
+  github_token:
+    description: |
+      The github access token (secrets.GITHUB_TOKEN) to create pull request on Github.
+      Required if running in migration mode.
+    required: false
+    type: string
 outputs:
   urls_with_actions:
     description: |

discourse_check_cleanup.py

@@ -34,7 +34,7 @@ class Action(str, Enum):
     CLEANUP = "cleanup"
 
 
-def main():
+def main() -> None:
     """Clean up created Discourse pages.
 
     Raises:

main.py

@@ -11,53 +11,100 @@
 import pathlib
 from functools import partial
 
-from src import run
+from src import GETTING_STARTED, exceptions, run, types_
 from src.discourse import create_discourse
 
 
-def main():
-    """Execute the action."""
-    logging.basicConfig(level=logging.INFO)
+def _parse_env_vars() -> types_.UserInputs:
+    """Instantiate user inputs from environment variables.
 
-    # Read input
+    Returns:
+        Wrapped user input variables.
+    """
+    discourse_host = os.getenv("INPUT_DISCOURSE_HOST", "")
+    discourse_category_id = os.getenv("INPUT_DISCOURSE_CATEGORY_ID", "")
+    discourse_api_username = os.getenv("INPUT_DISCOURSE_API_USERNAME", "")
+    discourse_api_key = os.getenv("INPUT_DISCOURSE_API_KEY", "")
     delete_topics = os.getenv("INPUT_DELETE_TOPICS") == "true"
     dry_run = os.getenv("INPUT_DRY_RUN") == "true"
-    discourse_host = os.getenv("INPUT_DISCOURSE_HOST")
-    discourse_category_id = os.getenv("INPUT_DISCOURSE_CATEGORY_ID")
-    discourse_api_username = os.getenv("INPUT_DISCOURSE_API_USERNAME")
-    discourse_api_key = os.getenv("INPUT_DISCOURSE_API_KEY")
+    github_access_token = os.getenv("INPUT_GITHUB_TOKEN")
 
-    # Execute action
-    create_discourse_kwargs = {
-        "hostname": discourse_host,
-        "category_id": discourse_category_id,
-        "api_username": discourse_api_username,
-        "api_key": discourse_api_key,
-    }
-    discourse = create_discourse(**create_discourse_kwargs)
-    urls_with_actions_dict = run(
-        base_path=pathlib.Path(),
-        discourse=discourse,
-        dry_run=dry_run,
+    return types_.UserInputs(
+        discourse_hostname=discourse_host,
+        discourse_category_id=discourse_category_id,
+        discourse_api_username=discourse_api_username,
+        discourse_api_key=discourse_api_key,
         delete_pages=delete_topics,
+        dry_run=dry_run,
+        github_access_token=github_access_token,
     )
 
-    # Write output
-    github_output = pathlib.Path(os.getenv("GITHUB_OUTPUT"))
+
+def _write_github_output(
+    urls_with_actions_dict: dict[str, str], user_inputs: types_.UserInputs
+) -> None:
+    """Writes results produced by the action to github_output.
+
+    Args:
+        urls_with_actions_dict: key value pairs of link to result of action.
+        user_inputs: parsed input variables used to run the action.
+
+    Raises:
+        InputError: if not running inside a github actions environment.
+    """
+    github_output = os.getenv("GITHUB_OUTPUT")
+    if not github_output:
+        raise exceptions.InputError(
+            f"Invalid 'GITHUB_OUTPUT' input, it must be non-empty, got {github_output=!r}"
+            f"This action is intended to run inside github-actions. {GETTING_STARTED}"
+        )
+
+    github_output_path = pathlib.Path(github_output)
     compact_json = partial(json.dumps, separators=(",", ":"))
     urls_with_actions = compact_json(urls_with_actions_dict)
     if urls_with_actions_dict:
         *_, index_url = urls_with_actions_dict.keys()
     else:
         index_url = ""
-    discourse_config = compact_json(create_discourse_kwargs)
-    github_output.write_text(
+    discourse_config = compact_json(
+        {
+            "hostname": user_inputs.discourse_hostname,
+            "category_id": user_inputs.discourse_category_id,
+            "api_username": user_inputs.discourse_api_username,
+            "api_key": user_inputs.discourse_api_key,
+        }
+    )
+    github_output_path.write_text(
         f"urls_with_actions={urls_with_actions}\n"
         f"index_url={index_url}\n"
         f"discourse_config={discourse_config}\n",
         encoding="utf-8",
     )
 
 
+def main() -> None:
+    """Execute the action."""
+    logging.basicConfig(level=logging.INFO)
+
+    # Read input
+    user_inputs = _parse_env_vars()
+
+    # Execute action
+    discourse = create_discourse(
+        hostname=user_inputs.discourse_hostname,
+        category_id=user_inputs.discourse_category_id,
+        api_username=user_inputs.discourse_api_username,
+        api_key=user_inputs.discourse_api_key,
+    )
+    urls_with_actions_dict = run(
+        base_path=pathlib.Path(),
+        discourse=discourse,
+        user_inputs=user_inputs,
+    )
+
+    # Write output
+    _write_github_output(urls_with_actions_dict=urls_with_actions_dict, user_inputs=user_inputs)
+
+
 if __name__ == "__main__":
     main()

pyproject.toml

@@ -18,7 +18,7 @@ show_missing = true
 [tool.pytest.ini_options]
 minversion = "6.0"
 log_cli_level = "INFO"
-markers = ["init", "discourse"]
+markers = ["reconcile", "migrate", "discourse"]
 
 # Formatting tools configuration
 [tool.black]
@@ -49,3 +49,9 @@ copyright-regexp = "Copyright\\s\\d{4}([-,]\\d{4})*\\s+%(author)s"
 
 [tool.mypy]
 ignore_missing_imports = true
+check_untyped_defs = true
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+disallow_untyped_defs = false

requirements.txt

@@ -1,3 +1,5 @@
+GitPython>=3.1,<3.2
 pydiscourse>=1.3,<1.4
+PyGithub>=1.57,<1.58
 PyYAML>=6.0,<6.1
 requests>=2.28,<2.29

src/__init__.py

@@ -8,24 +8,36 @@
 from .action import DRY_RUN_NAVLINK_LINK, FAIL_NAVLINK_LINK
 from .action import run_all as run_all_actions
 from .discourse import Discourse
+from .docs_directory import has_docs_directory
 from .docs_directory import read as read_docs_directory
-from .index import DOCUMENTATION_FOLDER_NAME
+from .exceptions import InputError
+from .index import DOCUMENTATION_FOLDER_NAME, contents_from_page
 from .index import get as get_index
 from .metadata import get as get_metadata
+from .migration import run as migrate_contents
 from .navigation_table import from_page as navigation_table_from_page
+from .pull_request import RepositoryClient, create_pull_request, create_repository_client
 from .reconcile import run as run_reconcile
+from .types_ import ActionResult, Metadata, UserInputs
 
+GETTING_STARTED = (
+    "To get started with upload-charm-docs, "
+    "please refer to https://github.com/canonical/upload-charm-docs#getting-started"
+)
 
-def run(
+
+def _run_reconcile(
     base_path: Path,
+    metadata: Metadata,
     discourse: Discourse,
     dry_run: bool,
     delete_pages: bool,
 ) -> dict[str, str]:
     """Upload the documentation to charmhub.
 
     Args:
-        base_path: The base path to look for the metadata file in.
+        base_path: The base path of the repository.
+        metadata: Information about the charm.
         discourse: A client to the documentation server.
         dry_run: If enabled, only log the action that would be taken.
         delete_pages: Whether to delete pages that are no longer needed.
@@ -34,7 +46,6 @@ def run(
         All the URLs that had an action with the result of that action.
 
     """
-    metadata = get_metadata(base_path)
     index = get_index(metadata=metadata, base_path=base_path, server_client=discourse)
     path_infos = read_docs_directory(docs_path=base_path / DOCUMENTATION_FOLDER_NAME)
     server_content = (
@@ -50,9 +61,79 @@ def run(
         delete_pages=delete_pages,
     )
     return {
-        report.url: report.result
+        str(report.location): report.result
         for report in reports
-        if report.url is not None
-        and report.url != DRY_RUN_NAVLINK_LINK
-        and report.url != FAIL_NAVLINK_LINK
+        if report.location is not None
+        and report.location != DRY_RUN_NAVLINK_LINK
+        and report.location != FAIL_NAVLINK_LINK
     }
+
+
+def _run_migrate(
+    base_path: Path, metadata: Metadata, discourse: Discourse, repository: RepositoryClient
+) -> dict[str, str]:
+    """Migrate existing docs from charmhub to local repository.
+
+    Args:
+        base_path: The base path of the repository.
+        metadata: Information about the charm.
+        discourse: A client to the documentation server.
+        repository: Repository client for managing both local and remote git repositories.
+
+    Returns:
+        A single key-value pair dictionary containing a link to the Pull Request containing
+        migrated documentation as key and successful action result as value.
+    """
+    index = get_index(metadata=metadata, base_path=base_path, server_client=discourse)
+    server_content = (
+        index.server.content if index.server is not None and index.server.content else ""
+    )
+    index_content = contents_from_page(server_content)
+    table_rows = navigation_table_from_page(page=server_content, discourse=discourse)
+    migrate_contents(
+        table_rows=table_rows,
+        index_content=index_content,
+        discourse=discourse,
+        docs_path=base_path / DOCUMENTATION_FOLDER_NAME,
+    )
+
+    pr_link = create_pull_request(repository=repository)
+
+    return {pr_link: ActionResult.SUCCESS}
+
+
+def run(base_path: Path, discourse: Discourse, user_inputs: UserInputs) -> dict[str, str]:
+    """Interact with charmhub to upload documentation or migrate to local repository.
+
+    Args:
+        base_path: The base path to look for the metadata file in.
+        discourse: A client to the documentation server.
+        user_inputs: Configurable inputs for running upload-charm-docs.
+
+    Raises:
+        InputError: if no valid running mode is matched.
+
+    Returns:
+        All the URLs that had an action with the result of that action.
+    """
+    metadata = get_metadata(base_path)
+    has_docs_dir = has_docs_directory(base_path=base_path)
+    if metadata.docs and not has_docs_dir:
+        repository = create_repository_client(
+            access_token=user_inputs.github_access_token, base_path=base_path
+        )
+        return _run_migrate(
+            base_path=base_path,
+            metadata=metadata,
+            discourse=discourse,
+            repository=repository,
+        )
+    if has_docs_dir:
+        return _run_reconcile(
+            base_path=base_path,
+            metadata=metadata,
+            discourse=discourse,
+            dry_run=user_inputs.dry_run,
+            delete_pages=user_inputs.delete_pages,
+        )
+    raise InputError(GETTING_STARTED)