From 9afd8ab08d433fe992fdc898c74be60701c6a91f Mon Sep 17 00:00:00 2001
From: George Katsikas <giorgakis.katsikas@gmail.com>
Date: Mon, 15 May 2023 16:02:54 +0200
Subject: [PATCH] add basic gitlab integration with main server
add python-gitlab package to requirements
add gitlab variables to settings
add advance repos django command
---
requirements.txt | 3 +
scipost_django/SciPost_v1/settings/base.py | 5 +
.../production/management/__init__.py | 0
.../management/commands/__init__.py | 0
.../management/commands/advance_git_repos.py | 154 ++++++++++++++++++
scipost_django/production/models.py | 10 +-
6 files changed, 169 insertions(+), 3 deletions(-)
create mode 100644 scipost_django/production/management/__init__.py
create mode 100644 scipost_django/production/management/commands/__init__.py
create mode 100644 scipost_django/production/management/commands/advance_git_repos.py
diff --git a/requirements.txt b/requirements.txt
index 63519f6e9..a3a6d753e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -86,3 +86,6 @@ mailchimp3==3.0.18 # 2023-05-09
django-referrer-policy==1.0 # 2020-09-19 no new updates for 3 years
django-csp==3.7 # 2020-09-19
django-feature-policy==3.4.0 # 2020-09-19
+
+# Version Control
+python-gitlab==3.14.0 # 2023-05-15
\ No newline at end of file
diff --git a/scipost_django/SciPost_v1/settings/base.py b/scipost_django/SciPost_v1/settings/base.py
index 9d3501b30..f00e676ff 100644
--- a/scipost_django/SciPost_v1/settings/base.py
+++ b/scipost_django/SciPost_v1/settings/base.py
@@ -570,3 +570,8 @@ DISCOURSE_SSO_SECRET = get_secret("DISCOURSE_SSO_SECRET")
CORS_ALLOWED_ORIGINS = [
"https://git.scipost.org",
]
+
+# GitLab API
+GITLAB_ROOT = "SciPost"
+GITLAB_URL = "https://git.scipost.org/"
+GITLAB_KEY = get_secret("GITLAB_KEY")
diff --git a/scipost_django/production/management/__init__.py b/scipost_django/production/management/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scipost_django/production/management/commands/__init__.py b/scipost_django/production/management/commands/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scipost_django/production/management/commands/advance_git_repos.py b/scipost_django/production/management/commands/advance_git_repos.py
new file mode 100644
index 000000000..af977dd98
--- /dev/null
+++ b/scipost_django/production/management/commands/advance_git_repos.py
@@ -0,0 +1,154 @@
+__copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
+__license__ = "AGPL v3"
+
+from django.core.management.base import BaseCommand, CommandParser
+from django.conf import settings
+
+from common.utils import get_current_domain
+
+from gitlab import Gitlab
+from gitlab.v4.objects import Group
+from gitlab.exceptions import GitlabGetError
+
+
+from production.models import ProofsRepository
+from production.constants import (
+ PROOFS_REPO_UNINITIALIZED,
+ PROOFS_REPO_CREATED,
+)
+
+
+class Command(BaseCommand):
+ """
+ This command handles the creation and updating of git repositories.
+ """
+
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+
+ # Check that the global GITLAB_ROOT constant is set
+ if not hasattr(settings, "GITLAB_ROOT") or settings.GITLAB_ROOT == "":
+ raise LookupError(
+ "Constant `GITLAB_ROOT` is either not present in settings file or empty, please add it."
+ )
+
+ self.GL: Gitlab = self._instanciate_gitlab()
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument(
+ "--id",
+ type=int,
+ required=False,
+ help="The submission preprint identifier to handle a specific submission, leave blank to handle all",
+ )
+
+ def _instanciate_gitlab(self) -> Gitlab:
+ """
+ Test the connection to the git server, returns a Gitlab object.
+ """
+
+ if not hasattr(settings, "GITLAB_KEY") or settings.GITLAB_KEY == "":
+ raise LookupError(
+ "Constant `GITLAB_KEY` is either not present in secret file or empty, please add it."
+ )
+
+ if not hasattr(settings, "GITLAB_URL") or settings.GITLAB_URL == "":
+ raise LookupError(
+ "Constant `GITLAB_URL` is either not present in secret file or empty, please add it."
+ )
+
+ GL = Gitlab(url=settings.GITLAB_URL, private_token=settings.GITLAB_KEY)
+
+ try:
+ GL.auth()
+ except Exception as e:
+ raise AssertionError(
+ "Could not authenticate with GitLab, please check your credentials."
+ ) from e
+
+ return GL
+
+ def _get_or_create_nested_group(self, group_path: str) -> Group:
+ """
+ Create a new group on the git server based on a path of nested folders.
+ """
+ parent_group = None
+ group_path_segments = group_path.split("/")
+
+ # Traverse the path segments (up to the second-to-last one)
+ # and create the groups if they do not exist
+ for i, group_path_segment in enumerate(group_path_segments[:-1]):
+ path_up_to_segment_i = "/".join(group_path_segments[: i + 1])
+
+ # Check if group exists in the server
+ try:
+ group = self.GL.groups.get(path_up_to_segment_i)
+
+ # If it does not exist, create it
+ except GitlabGetError:
+ # Guard against the root group not existing
+ if parent_group is None:
+ raise AssertionError(
+ f"The parent group of {path_up_to_segment_i} does not exist. \
+ This should not happen normally (and would not be fixable \
+ because GitLab does not allow root groups to be created)."
+ )
+
+ # Create the group
+ group = self.GL.groups.create(
+ {
+ "name": group_path_segment,
+ "path": group_path_segment,
+ "parent_id": parent_group.id,
+ "visibility": "private",
+ }
+ )
+
+ # Set the parent group to the current group
+ parent_group = group
+
+ return group
+
+ def _create_git_repo(self, repo: ProofsRepository):
+ """
+ Create a new git repository for the submission.
+ """
+ # Check if repo exists in the server
+ try:
+ project = self.GL.projects.get(repo.git_path)
+
+ # Create the repo on the server
+ except GitlabGetError:
+ # Get the namespace id
+ parent_group_id = self._get_or_create_nested_group(repo.git_path).id
+ project = self.GL.projects.create(
+ {
+ "name": repo.name,
+ "namespace_id": parent_group_id,
+ "visibility": "private",
+ "description": "Proofs for https://{domain}/submissions/{preprint_id}".format(
+ domain=get_current_domain(),
+ preprint_id=repo.stream.submission.preprint.identifier_w_vn_nr,
+ ),
+ }
+ )
+
+ self.stdout.write(
+ self.style.SUCCESS(f"Created git repository at {repo.git_path}")
+ )
+
+ def handle(self, *args, **options):
+ # Limit the actions to a specific submission if requested
+ if preprint_id := options.get("id"):
+ repos = ProofsRepository.objects.filter(
+ stream__submission__preprint__identifier_w_vn_nr=preprint_id
+ )
+ else:
+ repos = ProofsRepository.objects.all()
+
+ # Create the repos
+ repos_to_be_created = repos.filter(status=PROOFS_REPO_UNINITIALIZED)
+ for repo in repos_to_be_created:
+ self._create_git_repo(repo)
+ repo.status = PROOFS_REPO_CREATED
+ repo.save()
diff --git a/scipost_django/production/models.py b/scipost_django/production/models.py
index c970ff471..f29affeb9 100644
--- a/scipost_django/production/models.py
+++ b/scipost_django/production/models.py
@@ -13,7 +13,7 @@ from django.db.models.signals import post_save
from django.dispatch import receiver
from django.db.models import Value
from django.db.models.functions import Concat
-
+from django.conf import settings
from .constants import (
PRODUCTION_STREAM_STATUS,
@@ -300,7 +300,8 @@ class ProofsRepository(models.Model):
"%Y-%m"
).split("-")
- return "/Proofs/{journal}/{year}/{month}/{repo_name}".format(
+ return "{ROOT}/Proofs/{journal}/{year}/{month}/{repo_name}".format(
+ ROOT=settings.GITLAB_ROOT,
journal=self.journal_path_abbrev,
year=creation_year,
month=creation_month,
@@ -309,7 +310,10 @@ class ProofsRepository(models.Model):
@property
def template_path(self) -> str:
- return "Templates/{journal}".format(journal=self.journal_path_abbrev)
+ return "{ROOT}/Templates/{journal}".format(
+ ROOT=settings.GITLAB_ROOT,
+ journal=self.journal_path_abbrev,
+ )
def __str__(self) -> str:
return f"Proofs repo for {self.stream}"
--
GitLab