Build: support cloning private repos with token (#12115)

First I wanted to pass the env var just in the clone step, but we don't
allow passing additional env vars once the environment is created, so
it's available in the whole "clone" environment. The access token we
create is read-only, and should be scoped to just one project as well
(waiting on PyGithub/PyGithub#3287).

Once the clone is done, the token is stored in the .git/config file, so
that token isn't always kept secret from the rest of the build like ssh
keys, but since the token is read-only and scoped to the current
project, and temporary (1 hour). It should be fine. Additionally, the
token is only created for private repos, meaning that only people with
explicit access to the repo may be able to extract the token, but again,
since they already have access to the repo, there is no additional
permissions the token is granting to the user (will document this in
#12114).

Author: stsewd

readthedocs/api/v2/serializers.py

@@ -93,6 +93,7 @@ class Meta(ProjectSerializer.Meta):
             "environment_variables",
             "max_concurrent_builds",
             "readthedocs_yaml_path",
+            "clone_token",
         )
 
 

readthedocs/doc_builder/director.py

@@ -679,6 +679,7 @@ def get_vcs_env_vars(self):
         env = self.get_rtd_env_vars()
         # Don't prompt for username, this requires Git 2.3+
         env["GIT_TERMINAL_PROMPT"] = "0"
+        env["READTHEDOCS_GIT_CLONE_TOKEN"] = self.data.project.clone_token
         return env
 
     def get_rtd_env_vars(self):

readthedocs/doc_builder/environments.py

@@ -391,6 +391,7 @@ def _escape_command(self, cmd):
         not_escape_variables = (
             "READTHEDOCS_OUTPUT",
             "READTHEDOCS_VIRTUALENV_PATH",
+            "READTHEDOCS_GIT_CLONE_TOKEN",
             "CONDA_ENVS_PATH",
             "CONDA_DEFAULT_ENV",
         )

readthedocs/oauth/services/base.py

@@ -38,6 +38,7 @@ class Service:
     default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL
     default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL
     supports_build_status = False
+    supports_clone_token = False
 
     @classmethod
     def for_project(cls, project):
@@ -328,7 +329,3 @@ def sync_repositories(self):
 
     def sync_organizations(self):
         raise NotImplementedError
-
-    def get_clone_token(self, project):
-        """User services make use of SSH keys only for cloning."""
-        return None

readthedocs/oauth/services/githubapp.py

@@ -34,6 +34,7 @@ class GitHubAppService(Service):
     vcs_provider_slug = GITHUB_APP
     allauth_provider = GitHubAppProvider
     supports_build_status = True
+    supports_clone_token = True
 
     def __init__(self, installation: GitHubAppInstallation):
         self.installation = installation
@@ -462,17 +463,29 @@ def get_clone_token(self, project):
         """
         Return a token for HTTP-based Git access to the repository.
 
+        The token is scoped to have read-only access to the content of the repository attached to the project.
+        The token expires after one hour (this is given by GitHub and can't be changed).
+
         See:
         - https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation
         - https://docs.github.com/en/rest/apps/apps?apiVersion=2022-11-28#create-an-installation-access-token-for-an-app
         """
-        # NOTE: we can pass the repository_ids to get a token with access to specific repositories.
-        # We should upstream this feature to PyGithub.
-        # We can also pass a specific permissions object to get a token with specific permissions
-        # if we want to scope this token even more.
         try:
-            access_token = self.gh_app_client.get_access_token(self.installation.installation_id)
-            return f"x-access-token:{access_token.token}"
+            # TODO: Use self.gh_app_client.get_access_token instead,
+            # once https://github.com/PyGithub/PyGithub/pull/3287 is merged.
+            _, response = self.gh_app_client.requester.requestJsonAndCheck(
+                "POST",
+                f"/app/installations/{self.installation.installation_id}/access_tokens",
+                headers=self.gh_app_client._get_headers(),
+                input={
+                    "repository_ids": [int(project.remote_repository.remote_id)],
+                    "permissions": {
+                        "contents": "read",
+                    },
+                },
+            )
+            token = response["token"]
+            return f"x-access-token:{token}"
         except GithubException:
             log.info(
                 "Failed to get clone token for project",

readthedocs/projects/models.py

@@ -968,6 +968,7 @@ def vcs_repo(self, environment, version):
                 self,
                 version=version,
                 environment=environment,
+                use_token=bool(self.clone_token),
             )
         return repo
 
@@ -1398,6 +1399,29 @@ def get_subproject_candidates(self, user):
     def organization(self):
         return self.organizations.first()
 
+    @property
+    def clone_token(self) -> str | None:
+        """
+        Return a HTTP-based Git access token to the repository.
+
+        .. note::
+
+           - A token is only returned for projects linked to a private repository.
+           - Only repositories granted access by a GitHub app installation will return a token.
+        """
+        service_class = self.get_git_service_class()
+        if not service_class or not self.remote_repository.private:
+            return None
+
+        if not service_class.supports_clone_token:
+            return None
+
+        for service in service_class.for_project(self):
+            token = service.get_clone_token(self)
+            if token:
+                return token
+        return None
+
 
 class APIProject(Project):
     """
@@ -1414,12 +1438,17 @@ class APIProject(Project):
     """
 
     features = []
+    # This is a property in the original model, in order to
+    # be able to assign it a value in the constructor, we need to re-declare it
+    # as an attribute here.
+    clone_token = None
 
     class Meta:
         proxy = True
 
     def __init__(self, *args, **kwargs):
         self.features = kwargs.pop("features", [])
+        self.clone_token = kwargs.pop("clone_token", None)
         environment_variables = kwargs.pop("environment_variables", {})
         ad_free = not kwargs.pop("show_advertising", True)
         # These fields only exist on the API return, not on the model, so we'll

readthedocs/projects/tasks/builds.py

@@ -205,6 +205,7 @@ def execute(self):
             version=self.data.version,
             environment={
                 "GIT_TERMINAL_PROMPT": "0",
+                "READTHEDOCS_GIT_CLONE_TOKEN": self.data.project.clone_token,
             },
             # Pass the api_client so that all environments have it.
             # This is needed for ``readthedocs-corporate``.