Skip to content

Commit

Permalink
fix: rate limiting caused by git repos cache check
Browse files Browse the repository at this point in the history
See the extended conversation here:
#163

Close #161
  • Loading branch information
gabor-boros authored and regisb committed Dec 8, 2023
1 parent e7277ed commit c4ed70c
Show file tree
Hide file tree
Showing 5 changed files with 17 additions and 45 deletions.
6 changes: 1 addition & 5 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -138,14 +138,10 @@ Other MFE developers can take advantage of this plugin to deploy their own MFEs.
"repository": "https://github.com/myorg/mymfe",
"port": 2001,
"version": "me/my-custom-branch", # optional, will default to the Open edX current tag.
"refs": https://api.github.com/repos/myorg/mymfe/git/refs/heads", # optional
}
return mfes

The MFE assets will then be bundled in the "mfe" Docker image whenever it is rebuilt with ``tutor images build mfe``. Providing a ``refs`` URL will ensure the build cache for that MFE is invalidated whenever a change is detected upstream at the git version in question. You can use the `GitHub references API`_ or the `GitLab branches API`_ for this.

.. _GitHub references API: https://docs.github.com/en/rest/git/refs?apiVersion=2022-11-28#get-a-reference
.. _GitLab branches API: https://docs.gitlab.com/ee/api/branches.html#get-single-repository-branch
The MFE assets will then be bundled in the "mfe" Docker image whenever it is rebuilt with ``tutor images build mfe``.

Assets will be served at ``http(s)://{{ MFE_HOST }}/mymfe``. Developers are free to add extra template patches to their plugins, as usual: for instance LMS setting patches to make sure that the LMS correctly connects to the MFEs.

Expand Down
3 changes: 3 additions & 0 deletions changelog.d/20231208_193911_regis_mfe_refs_for_master.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
- [Bugfix] Support MFE remotes that are not on GitHub. (by @gabor-boros and @regisb)
- 💥[Bugfix] Bypass rate-limiting when checking for upstream git changes. To achieve that, we use the `ADD --keep-git-dir` option, which is only compatible with BuildKit 0.11, from January 2023. Also, we get rid of the `get_github_refs_path` function. (by @gabor-boros and @regisb)

2 changes: 1 addition & 1 deletion tutormfe/hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@

from tutor.core.hooks import Filter

MFE_ATTRS_TYPE = t.Dict[t.Literal["repository", "refs", "port"], t.Union["str", int]]
MFE_ATTRS_TYPE = t.Dict[t.Literal["repository", "port", "version"], t.Union["str", int]]

MFE_APPS: Filter[dict[str, MFE_ATTRS_TYPE], []] = Filter()
45 changes: 10 additions & 35 deletions tutormfe/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,70 +29,45 @@
},
}


def get_github_refs_path(name: str) -> str:
"""
Generate a URL to access refs in heads (nightly) or tags (stable) via Github API.
Args:
name (str): Consisted of the repository owner and the repository name, as a string in 'owner/repo' format.
Returns:
str: A string URL to the Github API, pointing to heads if version_suffix is set, tags otherwise.
"""

return f"https://api.github.com/repos/{name}/git/refs/{'heads' if __version_suffix__ else 'tags'}"


CORE_MFE_APPS: dict[str, MFE_ATTRS_TYPE] = {
"authn": {
"repository": "https://github.com/openedx/frontend-app-authn",
"refs": get_github_refs_path("openedx/frontend-app-authn"),
"repository": "https://github.com/openedx/frontend-app-authn.git",
"port": 1999,
},
"account": {
"repository": "https://github.com/openedx/frontend-app-account",
"refs": get_github_refs_path("openedx/frontend-app-account"),
"repository": "https://github.com/openedx/frontend-app-account.git",
"port": 1997,
},
"communications": {
"repository": "https://github.com/openedx/frontend-app-communications",
"refs": get_github_refs_path("openedx/frontend-app-communications"),
"repository": "https://github.com/openedx/frontend-app-communications.git",
"port": 1984,
},
"course-authoring": {
"repository": "https://github.com/openedx/frontend-app-course-authoring",
"refs": get_github_refs_path("openedx/frontend-app-course-authoring"),
"repository": "https://github.com/openedx/frontend-app-course-authoring.git",
"port": 2001,
},
"discussions": {
"repository": "https://github.com/openedx/frontend-app-discussions",
"refs": get_github_refs_path("openedx/frontend-app-discussions"),
"repository": "https://github.com/openedx/frontend-app-discussions.git",
"port": 2002,
},
"gradebook": {
"repository": "https://github.com/openedx/frontend-app-gradebook",
"refs": get_github_refs_path("openedx/frontend-app-gradebook"),
"repository": "https://github.com/openedx/frontend-app-gradebook.git",
"port": 1994,
},
"learner-dashboard": {
"repository": "https://github.com/openedx/frontend-app-learner-dashboard",
"refs": get_github_refs_path("openedx/frontend-app-learner-dashboard"),
"repository": "https://github.com/openedx/frontend-app-learner-dashboard.git",
"port": 1996,
},
"learning": {
"repository": "https://github.com/openedx/frontend-app-learning",
"refs": get_github_refs_path("openedx/frontend-app-learning"),
"repository": "https://github.com/openedx/frontend-app-learning.git",
"port": 2000,
},
"ora-grading": {
"repository": "https://github.com/openedx/frontend-app-ora-grading",
"refs": get_github_refs_path("openedx/frontend-app-ora-grading"),
"repository": "https://github.com/openedx/frontend-app-ora-grading.git",
"port": 1993,
},
"profile": {
"repository": "https://github.com/openedx/frontend-app-profile",
"refs": get_github_refs_path("openedx/frontend-app-profile"),
"repository": "https://github.com/openedx/frontend-app-profile.git",
"port": 1995,
},
}
Expand Down
6 changes: 2 additions & 4 deletions tutormfe/templates/mfe/build/mfe/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# syntax=docker/dockerfile:1
# https://hub.docker.com/_/node/tags
FROM docker.io/node:18.16.0-bullseye-slim AS base

Expand Down Expand Up @@ -34,10 +35,7 @@ RUN echo "copying i18n data" \
######## {{ app_name }} (git)
FROM base AS {{ app_name }}-git
{#- Invalidate the build cache if a change is detected upstream #}
{%- if app.get("refs") %}
ADD {{ app["refs"] }}/{{ app.get("version", MFE_COMMON_VERSION) }} /tmp/gitref-{{ app_name }}
{%- endif %}
RUN git clone {{ app["repository"] }} --branch {{ app.get("version", MFE_COMMON_VERSION) }} --depth 1 .
ADD --keep-git-dir=true {{ app["repository"] }}#{{ app.get("version", MFE_COMMON_VERSION) }} .

######## {{ app_name }} (src)
# Empty layer with just the repo at the root, for build-time bind-mounts
Expand Down

0 comments on commit c4ed70c

Please sign in to comment.