Merge branch 'master' into 'master'

Add git integration to server

See merge request !42

.gitignore

@@ -16,11 +16,11 @@ __pycache__
 
 # Package managers
 /venv*
+/.venv*
 /node_modules/
 *webpack-stats.json
 
 .python-version
-*secrets.json
 !package.json
 !package-lock.json
 !package.vue.json
@@ -51,3 +51,10 @@ start_flower.sh
 whoosh_index
 
 *.pid
+
+# Sensitive files
+*/secrets.json
+/scipost_django/SciPost_v1/settings/local_*.py
+
+# Editor configs
+*.code-workspace
\ No newline at end of file

cronjobs/cronjob_production_eachhour.sh

 #!/bin/bash
 
-# Per minute cronjobs for production area
+# Per hour cronjobs for production area
 
 cd /home/scipost/SciPost/scipost_django
 source ../venv-3.8.5/bin/activate
@@ -8,6 +8,7 @@ source ../venv-3.8.5/bin/activate
 # Do tasks
 python manage.py check_celery --settings=SciPost_v1.settings.production_do1
 python manage.py update_coi_via_arxiv --settings=SciPost_v1.settings.production_do1
+python manage.py advance_git_repos --settings=SciPost_v1.settings.production_do1
 
 # Do a update_index of the last hour
 python manage.py update_index -r -v 0 -a 1 --settings=SciPost_v1.settings.production_do1

requirements.txt

@@ -3,7 +3,7 @@ argon2-cffi==20.1.0		# 2021-07-18 Password hashing algorithm
 Babel==2.9.1			# 2022-01-23
 Django==3.2.18			# 2023-05-06
 feedparser~=6.0.8     # Check: not updated since 2016. [JdW, 2021-09-25] Upgrade to v6; v5 fails.
-psycopg2==2.8.6	  		# 2020-09-19 PostgreSQL engine
+psycopg2==2.9.5	  		# 2020-09-19 PostgreSQL engine -- 2023-05-10 update for python 3.11
 pytz==2021.3			# 2022-11-18 Timezone package
 # djangorestframework==3.9.3	# DEPREC, see next entry -- 2019-12-05  IMPORTANT: update templates/rest_framework/base.html if corresponding file rest_framework/templates/rest_framework/base.html has changed
 git+https://github.com/SciPost/django-rest-framework.git@bootstrap-v5
@@ -80,9 +80,15 @@ celery==5.2.7 		            # 2022-11-18
 django-celery-results==2.4.0 	    # 2022-11-18
 django-celery-beat==2.4.0	    # 2022-11-18
 flower==1.2.0 			    # 2022-11-18
-
+mailchimp3==3.0.18         # 2023-05-09
 
 # Security-related packages
 django-referrer-policy==1.0	# 2020-09-19 no new updates for 3 years
 django-csp==3.7			# 2020-09-19
 django-feature-policy==3.4.0	# 2020-09-19
+
+# Version Control
+python-gitlab==3.14.0		# 2023-05-15
+
+# Preprint server packages
+arxiv==1.4.7            # 2023-05-19
\ No newline at end of file

scipost_django/SciPost_v1/settings/base.py

@@ -570,3 +570,8 @@ DISCOURSE_SSO_SECRET = get_secret("DISCOURSE_SSO_SECRET")
 CORS_ALLOWED_ORIGINS = [
     "https://git.scipost.org",
 ]
+
+# GitLab API
+GITLAB_ROOT = "SciPost"
+GITLAB_URL = "git.scipost.org"
+GITLAB_KEY = get_secret("GITLAB_KEY")

scipost_django/SciPost_v1/settings/production_do1.py

@@ -88,3 +88,8 @@ CSP_REPORT_ONLY = False
 CORS_ALLOWED_ORIGINS = [
     "https://git.scipost.org",
 ]
+
+# GitLab API
+GITLAB_ROOT = "SciPost"
+GITLAB_URL = "git.scipost.org"
+GITLAB_KEY = get_secret("GITLAB_KEY")

scipost_django/production/admin.py

@@ -12,6 +12,7 @@ from .models import (
     ProductionUser,
     Proofs,
     ProductionEventAttachment,
+    ProofsRepository,
 )
 
 
@@ -95,3 +96,17 @@ admin.site.register(Proofs, ProductionProofsAdmin)
 
 
 admin.site.register(ProductionEventAttachment)
+
+
+class ProofsRepositoryAdmin(GuardedModelAdmin):
+    search_fields = [
+        "stream__submission__author_list",
+        "stream__submission__title",
+        "stream__submission__preprint__identifier_w_vn_nr",
+    ]
+    list_filter = ["status"]
+    list_display = ["stream", "status", "git_path"]
+    readonly_fields = ["template_path", "git_path"]
+
+
+admin.site.register(ProofsRepository, ProofsRepositoryAdmin)

scipost_django/production/management/commands/advance_git_repos.py

+__copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
+__license__ = "AGPL v3"
+
+from datetime import datetime
+from functools import reduce
+from itertools import cycle
+from typing import Any, Callable, Dict, List, Tuple
+from django.core.management.base import BaseCommand, CommandParser
+from django.conf import settings
+
+from common.utils import get_current_domain
+
+from gitlab import Gitlab
+from gitlab.v4.objects import Group, Project
+from gitlab.exceptions import GitlabGetError
+
+import arxiv
+import requests
+import tarfile
+from base64 import b64encode
+
+
+from production.models import ProofsRepository
+from production.constants import (
+    PROOFS_REPO_UNINITIALIZED,
+    PROOFS_REPO_CREATED,
+    PROOFS_REPO_TEMPLATE_ONLY,
+    PROOFS_REPO_TEMPLATE_FORMATTED,
+    PROOFS_REPO_PRODUCTION_READY,
+)
+
+
+class Command(BaseCommand):
+    """
+    This command handles the creation and updating of git repositories.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        # Check that the global GITLAB_ROOT constant is set
+        if not hasattr(settings, "GITLAB_ROOT") or settings.GITLAB_ROOT == "":
+            raise LookupError(
+                "Constant `GITLAB_ROOT` is either not present in settings file or empty, please add it."
+            )
+
+        self.GL: Gitlab = self._instanciate_gitlab()
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "--id",
+            type=int,
+            required=False,
+            help="The submission preprint identifier to handle a specific submission, leave blank to handle all",
+        )
+
+    def _instanciate_gitlab(self) -> Gitlab:
+        """
+        Test the connection to the git server, returns a Gitlab object.
+        """
+
+        if not hasattr(settings, "GITLAB_KEY") or settings.GITLAB_KEY == "":
+            raise LookupError(
+                "Constant `GITLAB_KEY` is either not present in secret file or empty, please add it."
+            )
+
+        if not hasattr(settings, "GITLAB_URL") or settings.GITLAB_URL == "":
+            raise LookupError(
+                "Constant `GITLAB_URL` is either not present in secret file or empty, please add it."
+            )
+
+        GL = Gitlab(
+            url="https://" + settings.GITLAB_URL,
+            private_token=settings.GITLAB_KEY,
+        )
+
+        try:
+            GL.auth()
+        except Exception as e:
+            raise AssertionError(
+                "Could not authenticate with GitLab, please check your credentials."
+            ) from e
+
+        return GL
+
+    def _get_or_create_nested_group(self, group_path: str) -> Group:
+        """
+        Create a new group on the git server based on a path of nested folders.
+        """
+        parent_group = None
+        group_path_segments = group_path.split("/")
+
+        # Traverse the path segments (up to the second-to-last one)
+        # and create the groups if they do not exist
+        for i, group_path_segment in enumerate(group_path_segments[:-1]):
+            path_up_to_segment_i = "/".join(group_path_segments[: i + 1])
+
+            # Check if group exists in the server
+            try:
+                group = self.GL.groups.get(path_up_to_segment_i)
+
+            # If it does not exist, create it
+            except GitlabGetError:
+                # Guard against the root group not existing
+                if parent_group is None:
+                    raise AssertionError(
+                        f"The parent group of {path_up_to_segment_i} does not exist. "
+                        "This should not happen normally (and would not be fixable "
+                        "because GitLab does not allow root groups to be created)."
+                    )
+
+                # Create the group
+                group = self.GL.groups.create(
+                    {
+                        "name": group_path_segment,
+                        "path": group_path_segment,
+                        "parent_id": parent_group.id,
+                        "visibility": "private",
+                    }
+                )
+
+            # Set the parent group to the current group
+            parent_group = group
+
+        return group
+
+    def _create_git_repo(self, repo: ProofsRepository):
+        """
+        Create a new git repository for the submission.
+        """
+        # Check if repo exists in the server
+        try:
+            project = self.GL.projects.get(repo.git_path)
+
+        # Create the repo on the server
+        except GitlabGetError:
+            # Get the namespace id
+            parent_group_id = self._get_or_create_nested_group(repo.git_path).id
+            project = self.GL.projects.create(
+                {
+                    "name": repo.name,
+                    "namespace_id": parent_group_id,
+                    "visibility": "private",
+                    "description": "Proofs for https://{domain}/submissions/{preprint_id}".format(
+                        domain=get_current_domain(),
+                        preprint_id=repo.stream.submission.preprint.identifier_w_vn_nr,
+                    ),
+                }
+            )
+
+        self.stdout.write(
+            self.style.SUCCESS(f"Created git repository at {repo.git_path}")
+        )
+
+    def _get_project_cloning_actions(self, project: Project) -> List[Dict[str, Any]]:
+        """
+        Return a list of gitlab actions required to fully clone a project.
+        """
+        filenames = list(
+            map(lambda x: x["path"], project.repository_tree(get_all=True))
+        )
+
+        actions = []
+        for filename in filenames:
+            try:
+                file = project.files.get(file_path=filename, ref="main")
+            except:
+                self.stdout.write(
+                    self.style.WARNING(f"File {filename} not found in {project.name}")
+                )
+                continue
+
+            actions.append(
+                {
+                    "action": "create",
+                    "file_path": filename,
+                    "content": file.content,
+                    "encoding": "base64",
+                }
+            )
+
+        return actions
+
+    def _copy_pure_templates(self, repo: ProofsRepository):
+        """
+        Copy the pure templates to the repo.
+        """
+        project = self.GL.projects.get(repo.git_path)
+
+        journal_template_project = self.GL.projects.get(repo.template_path)
+        base_template_project = self.GL.projects.get(
+            "{ROOT}/Templates/Base".format(ROOT=settings.GITLAB_ROOT)
+        )
+
+        base_actions = self._get_project_cloning_actions(base_template_project)
+        journal_actions = self._get_project_cloning_actions(journal_template_project)
+
+        # Commit the actions
+        project.commits.create(
+            {
+                "branch": "main",
+                "commit_message": "copy pure templates",
+                "actions": base_actions + journal_actions,
+            }
+        )
+
+        self.stdout.write(
+            self.style.SUCCESS(f"Copied pure templates to {repo.git_path}")
+        )
+
+    def _format_skeleton(self, repo: ProofsRepository):
+        """
+        Format the Skeleton.tex file of the repo to include basic information about the submission.
+        """
+
+        SHAPES = ["star", "dagger", "ddagger", "circ", "S", "P", "parallel"]
+        SLASH = "\\"
+        NEWLINE = f"{SLASH}{SLASH}"
+
+        def abbreviate_author(author: str) -> str:
+            """
+            Abbreviate an author's name by taking the first letter\
+            of their first and middle names, and their full last name.
+            """
+
+            # TODO: This is somewhat naive, but it should work for now.
+            first_name, *middle_names, last_name = author.split(" ")
+            # Ideally, I would like to search for matching authors in the database
+            # and abbreviate their names accordingly to the journal's style.
+            # Right now, I abbreviate only the very first name and leave the rest as is.
+
+            # Map each part of the (optionally) hyphenated first name to its abbreviation
+            # (e.g. "John-Edward" -> "J.-E.")
+            first_name_hyphen_parts = first_name.split("-")
+            first_name_hyphen_parts_abbrev = list(
+                map(lambda x: x[0].upper() + ".", first_name_hyphen_parts)
+            )
+
+            # Add different name parts to the abbreviation, glue them together with space
+            # (e.g. "John-Edward Brown Smith" -> "J.-E. Brown Smith")
+            abbreviation_parts = [
+                "-".join(first_name_hyphen_parts_abbrev),  # Abbreviated first name
+                *middle_names,
+                last_name,
+            ]
+
+            return " ".join(abbreviation_parts)
+
+        # Define the formatting functions
+        def format_authors(authors: List[str]) -> str:
+            *other_authors, last_author = authors
+
+            if len(other_authors) == 0:
+                return last_author
+            else:
+                return ", ".join(other_authors) + " and " + last_author
+
+        def format_title(title: str) -> str:
+            return title + NEWLINE
+
+        def format_copyright(authors: List[str]) -> str:
+            """
+            Format the copyright statement depending on the number\
+            of authors in the submission:
+            - 1 author: "© Author"
+            - 2 authors: "© Author1 and Author2"
+            - 3+ authors: "© Author1 et al"
+            """
+            if len(authors) == 1:
+                return f"Copyright {authors[0]}"
+            elif len(authors) == 2:
+                return f"Copyright {authors[0]} and {authors[1]}"
+            else:
+                return f"Copyright {authors[0]} {{{SLASH}it et al}}"
+
+        def format_emails(authors: List[str]) -> str:
+            """
+            Format the emails of the authors in the submission, grouped by 3 per line.\
+            The emails are padded with \\quad spacing and are prepended with a shape.
+            """
+            # Create a list array of emails, grouped by 3
+            mail_lines = [[]]
+            mail_line_i = 0
+            for i, (_, shape) in enumerate(zip(authors, cycle(SHAPES))):
+                mail_lines[mail_line_i].append(
+                    f"${SLASH}{shape}$ {SLASH}href{{mailto:email{i+1}}}{{{SLASH}small email{i+1}}}"
+                )
+
+                # Create a new mail group every 3 emails
+                if (i + 1) % 3 == 0:
+                    mail_line_i += 1
+                    mail_lines.append([])
+
+            # Flatten the inner lists and join them with "\,,\quad"
+            flattened_mail_lines = [
+                f"{SLASH},,{SLASH}quad\n".join(line) for line in mail_lines
+            ]
+
+            # Join the lines with "\,,\\"
+            flattened_mails = f"{SLASH},,{NEWLINE}\n".join(flattened_mail_lines)
+
+            return flattened_mails
+
+        def format_affiliations(authors: List[str]) -> str:
+            """
+            Format the affiliations of the authors in the submission,
+            by including the author's name and the affiliation number.
+            There is one affiliation per author by default.
+            """
+            affiliations = []
+            for i, author in enumerate(authors):
+                affiliations += [f"{{{SLASH}bf {i+1}}} Affiliation {author}"]
+
+            return f"\n{NEWLINE}\n".join(affiliations)
+
+        def format_date_human_readable(date: datetime) -> str:
+            """
+            Format a date in a human-readable format (DD-MM-YYY).
+            """
+            return date.strftime("%d-%m-%Y")
+
+        project = self.GL.projects.get(repo.git_path)
+        project_filenames = list(
+            map(lambda x: x["path"], project.repository_tree(get_all=True))
+        )
+
+        skeleton_filename = next(
+            filter(lambda x: x.endswith("Skeleton.tex"), project_filenames)
+        )
+        skeleton_file = project.files.get(file_path=skeleton_filename, ref="main")
+        skeleton_content = skeleton_file.decode().decode("utf-8")
+
+        # Collect the information about the paper
+        paper_title = repo.stream.submission.title
+        paper_abbreviated_authors = list(
+            map(abbreviate_author, repo.stream.submission.authors_as_list)
+        )
+        paper_received_date = repo.stream.submission.original_submission_date
+        paper_acceptance_date = repo.stream.submission.acceptance_date
+
+        # Create the replacement dictionary from placeholders and information
+        # key = placeholder, value = (formatting_function, *args)
+        replacements_dict = {
+            "<|TITLE|>": (format_title, paper_title),
+            "<|AUTHORS|>": (format_authors, repo.stream.submission.authors_as_list),
+            "<|EMAILS|>": (format_emails, paper_abbreviated_authors),
+            "<|COPYRIGHT|>": (format_copyright, paper_abbreviated_authors),
+            "<|AFFILIATIONS|>": (format_affiliations, paper_abbreviated_authors),
+            "<|RECEIVED|>": (format_date_human_readable, paper_received_date),
+            "<|ACCEPTED|>": (format_date_human_readable, paper_acceptance_date),
+        }
+
+        # Define a helper function to try to format and replace a placeholder
+        # which catches any errors and prints them to the console non-intrusively
+        def try_format_replace(
+            text: str,
+            key: str,
+            value: Tuple[Callable[[Any], str], Any],
+        ):
+            try:
+                formatting_function, *args = value
+                formatted_value = formatting_function(*args)
+                return text.replace(key, formatted_value)
+            except:
+                self.stdout.write(
+                    self.style.ERROR(
+                        f"Could not format and replace {key} with {value} in {repo.git_path}"
+                    )
+                )
+                return text
+
+        # Replace the placeholders with the submission information
+        # by iteratively applying the formatting functions to the skeleton
+        skeleton_content = reduce(
+            lambda text, replace_pair: try_format_replace(text, *replace_pair),
+            replacements_dict.items(),
+            skeleton_content,
+        )
+
+        # Commit the changes to the skeleton file and change its name
+        project.commits.create(
+            {
+                "branch": "main",
+                "commit_message": f"format skeleton file",
+                "actions": [
+                    {
+                        "action": "move",
+                        "content": skeleton_content,
+                        "previous_path": skeleton_filename,
+                        # Change the "Skeleton" part from the filename to the repo name
+                        # and remove the extraneous "scipost_" label from the identifier slug
+                        "file_path": skeleton_filename.replace(
+                            "Skeleton",
+                            repo.name.replace("scipost_", ""),
+                        ),
+                    },
+                ],
+            }
+        )
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Successfully formatted the skeleton of {repo.git_path}"
+            )
+        )
+
+    def _copy_arxiv_source_files(self, repo: ProofsRepository):
+        paper = next(
+            arxiv.Search(
+                id_list=[repo.stream.submission.preprint.identifier_w_vn_nr]
+            ).results()
+        )
+        source_stream = requests.get(paper.pdf_url.replace("pdf", "src"), stream=True)
+
+        # Create file creation actions for each file in the source tar
+        actions = []
+        with tarfile.open(fileobj=source_stream.raw) as tar:
+            for member in tar:
+                if not member.isfile():
+                    continue
+
+                f = tar.extractfile(member)
+                try:
+                    bin_content = f.read()
+                    actions.append(
+                        {
+                            "action": "create",
+                            "file_path": member.name,
+                            "encoding": "base64",
+                            # Encode the binary content in base64, required by the API
+                            "content": b64encode(bin_content).decode("utf-8"),
+                        }
+                    )
+
+                except:
+                    self.stdout.write(
+                        self.style.ERROR(
+                            f"Could not read {member.name} from the arXiv source files, skipping..."
+                        )
+                    )
+
+        # Filter out the files that already exist in the repo to avoid conflicts
+        project = self.GL.projects.get(repo.git_path)
+        project_existing_filenames = list(
+            map(lambda x: x["path"], project.repository_tree(get_all=True))
+        )
+
+        non_existing_file_actions = [
+            action
+            for action in actions
+            if action["file_path"] not in project_existing_filenames
+        ]
+
+        # Commit the creation of the files
+        project.commits.create(
+            {
+                "branch": "main",
+                "commit_message": f"copy arXiv source files",
+                "actions": non_existing_file_actions,
+            }
+        )
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Successfully copied the author source files to {repo.git_path}"
+            )
+        )
+
+    def handle(self, *args, **options):
+        # Limit the actions to a specific submission if requested
+        if preprint_id := options.get("id"):
+            repos = ProofsRepository.objects.filter(
+                stream__submission__preprint__identifier_w_vn_nr=preprint_id
+            )
+        else:
+            repos = ProofsRepository.objects.all()
+
+        # Create the repos
+        repos_to_be_created = repos.filter(status=PROOFS_REPO_UNINITIALIZED)
+        for repo in repos_to_be_created:
+            try:
+                self._create_git_repo(repo)
+                repo.status = PROOFS_REPO_CREATED
+                repo.save()
+            except Exception as e:
+                self.stdout.write(
+                    self.style.ERROR(
+                        f"Could not create the git repo for {repo.git_path}, error: {e}"
+                    )
+                )
+
+        # Copy the pure templates
+        repos_to_be_templated = repos.filter(status=PROOFS_REPO_CREATED)
+        for repo in repos_to_be_templated:
+            try:
+                self._copy_pure_templates(repo)
+                repo.status = PROOFS_REPO_TEMPLATE_ONLY
+                repo.save()
+            except Exception as e:
+                self.stdout.write(
+                    self.style.ERROR(
+                        f"Could not copy the pure templates to {repo.git_path}, error: {e}"
+                    )
+                )
+
+        # Format the skeleton files
+        repos_to_be_formatted = repos.filter(status=PROOFS_REPO_TEMPLATE_ONLY)
+        for repo in repos_to_be_formatted:
+            try:
+                self._format_skeleton(repo)
+                repo.status = PROOFS_REPO_TEMPLATE_FORMATTED
+                repo.save()
+            except Exception as e:
+                self.stdout.write(
+                    self.style.ERROR(
+                        f"Could not format the skeleton of {repo.git_path}, error: {e}"
+                    )
+                )
+
+        # Copy the arXiv source files
+        repos_to_be_copied = repos.filter(status=PROOFS_REPO_TEMPLATE_FORMATTED)
+        for repo in repos_to_be_copied:
+            try:
+                if "arxiv.org" in repo.stream.submission.preprint.url:
+                    self._copy_arxiv_source_files(repo)
+                    repo.status = PROOFS_REPO_PRODUCTION_READY
+                    repo.save()
+            except Exception as e:
+                self.stdout.write(
+                    self.style.ERROR(
+                        f"Could not copy the arXiv source files to {repo.git_path}, error: {e}"
+                    )
+                )

scipost_django/production/migrations/0006_proofsrepository.py

+# Generated by Django 3.2.18 on 2023-05-15 14:25
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('production', '0005_auto_20190511_1141'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ProofsRepository',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('uninitialized', 'The repository does not exist'), ('created', 'The repository exists but is empty'), ('template_only', 'The repository contains the bare template'), ('template_formatted', 'The repository contains the automatically formatted template'), ('production_ready', 'The repository is ready for production')], default='uninitialized', max_length=32)),
+                ('stream', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='proofs_repository', to='production.productionstream')),
+            ],
+            options={
+                'verbose_name_plural': 'proofs repositories',
+            },
+        ),
+    ]

scipost_django/production/models.py

@@ -6,8 +6,14 @@ from django.db import models
 from django.contrib.contenttypes.fields import GenericRelation
 from django.urls import reverse
 from django.contrib.auth.models import User
+from profiles.models import Profile
 from django.utils import timezone
 from django.utils.functional import cached_property
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from django.db.models import Value
+from django.db.models.functions import Concat
+from django.conf import settings
 
 from .constants import (
     PRODUCTION_STREAM_STATUS,
@@ -126,17 +132,11 @@ class ProductionStream(models.Model):
 
 
 class ProductionEvent(models.Model):
-    stream = models.ForeignKey(
-        ProductionStream, on_delete=models.CASCADE, related_name="events"
-    )
-    event = models.CharField(
-        max_length=64, choices=PRODUCTION_EVENTS, default=EVENT_MESSAGE
-    )
+    stream = models.ForeignKey(ProductionStream, on_delete=models.CASCADE, related_name="events")
+    event = models.CharField(max_length=64, choices=PRODUCTION_EVENTS, default=EVENT_MESSAGE)
     comments = models.TextField(blank=True, null=True)
     noted_on = models.DateTimeField(default=timezone.now)
-    noted_by = models.ForeignKey(
-        "production.ProductionUser", on_delete=models.CASCADE, related_name="events"
-    )
+    noted_by = models.ForeignKey("production.ProductionUser", on_delete=models.CASCADE, related_name="events")
     noted_to = models.ForeignKey(
         "production.ProductionUser",
         on_delete=models.CASCADE,
@@ -159,10 +159,7 @@ class ProductionEvent(models.Model):
 
     @cached_property
     def editable(self):
-        return (
-            self.event in [EVENT_MESSAGE, EVENT_HOUR_REGISTRATION]
-            and not self.stream.completed
-        )
+        return self.event in [EVENT_MESSAGE, EVENT_HOUR_REGISTRATION] and not self.stream.completed
 
 
 def production_event_upload_location(instance, filename):
@@ -185,9 +182,7 @@ class ProductionEventAttachment(models.Model):
         on_delete=models.CASCADE,
         related_name="attachments",
     )
-    attachment = models.FileField(
-        upload_to=production_event_upload_location, storage=SecureFileStorage()
-    )
+    attachment = models.FileField(upload_to=production_event_upload_location, storage=SecureFileStorage())
 
     def get_absolute_url(self):
         return reverse(
@@ -213,20 +208,12 @@ class Proofs(models.Model):
     Proofs are directly related to a ProductionStream and Submission in SciPost.
     """
 
-    attachment = models.FileField(
-        upload_to=proofs_upload_location, storage=SecureFileStorage()
-    )
+    attachment = models.FileField(upload_to=proofs_upload_location, storage=SecureFileStorage())
     version = models.PositiveSmallIntegerField(default=0)
-    stream = models.ForeignKey(
-        "production.ProductionStream", on_delete=models.CASCADE, related_name="proofs"
-    )
-    uploaded_by = models.ForeignKey(
-        "production.ProductionUser", on_delete=models.CASCADE, related_name="+"
-    )
+    stream = models.ForeignKey("production.ProductionStream", on_delete=models.CASCADE, related_name="proofs")
+    uploaded_by = models.ForeignKey("production.ProductionUser", on_delete=models.CASCADE, related_name="+")
     created = models.DateTimeField(auto_now_add=True)
-    status = models.CharField(
-        max_length=16, choices=PROOFS_STATUSES, default=PROOFS_UPLOADED
-    )
+    status = models.CharField(max_length=16, choices=PROOFS_STATUSES, default=PROOFS_UPLOADED)
     accessible_for_authors = models.BooleanField(default=False)
 
     objects = ProofsQuerySet.as_manager()
@@ -239,9 +226,7 @@ class Proofs(models.Model):
         return reverse("production:proofs_pdf", kwargs={"slug": self.slug})
 
     def __str__(self):
-        return "Proofs {version} for Stream {stream}".format(
-            version=self.version, stream=self.stream.submission.title
-        )
+        return "Proofs {version} for Stream {stream}".format(version=self.version, stream=self.stream.submission.title)
 
     def save(self, *args, **kwargs):
         # Control Report count per Submission.
@@ -252,3 +237,156 @@ class Proofs(models.Model):
     @property
     def slug(self):
         return proofs_id_to_slug(self.id)
+
+
+class ProofsRepository(models.Model):
+    """
+    ProofsRepository is a GitLab repository of Proofs for a Submission.
+    """
+
+    PROOFS_REPO_UNINITIALIZED = "uninitialized"
+    PROOFS_REPO_CREATED = "created"
+    PROOFS_REPO_TEMPLATE_ONLY = "template_only"
+    PROOFS_REPO_TEMPLATE_FORMATTED = "template_formatted"
+    PROOFS_REPO_PRODUCTION_READY = "production_ready"
+    PROOFS_REPO_STATUSES = (
+        (PROOFS_REPO_UNINITIALIZED, "The repository does not exist"),
+        (PROOFS_REPO_CREATED, "The repository exists but is empty"),
+        (PROOFS_REPO_TEMPLATE_ONLY, "The repository contains the bare template"),
+        (
+            PROOFS_REPO_TEMPLATE_FORMATTED,
+            "The repository contains the automatically formatted template",
+        ),
+        (PROOFS_REPO_PRODUCTION_READY, "The repository is ready for production"),
+    )
+
+    stream = models.OneToOneField(
+        ProductionStream,
+        on_delete=models.CASCADE,
+        related_name="proofs_repository",
+    )
+    status = models.CharField(
+        max_length=32,
+        choices=PROOFS_REPO_STATUSES,
+        default=PROOFS_REPO_UNINITIALIZED,
+    )
+
+    @property
+    def name(self) -> str:
+        """
+        Return the name of the repository in the form of "id_lastname".
+        """
+        # Get the last name of the first author by getting the first author string from the submission
+        first_author_str = self.stream.submission.authors_as_list[0]
+        first_author_profile = (
+            Profile.objects.annotate(
+                full_name=Concat("first_name", Value(" "), "last_name")
+            )
+            .filter(full_name=first_author_str)
+            .first()
+        )
+        if first_author_profile is None:
+            first_author_last_name = first_author_str.split(" ")[-1]
+        else:
+            first_author_last_name = first_author_profile.last_name
+            # Keep only the last of the last names
+            first_author_last_name = first_author_last_name.split(" ")[-1]
+
+        return "{preprint_id}_{last_name}".format(
+            preprint_id=self.stream.submission.preprint.identifier_w_vn_nr,
+            last_name=first_author_last_name,
+        )
+
+    @property
+    def journal_abbrev(self) -> str:
+        # The DOI label is used to determine the path of the repository and template
+        return self.stream.submission.editorial_decision.for_journal.doi_label
+
+    @property
+    def journal_subdivision(self) -> str:
+        """
+        Return the subdivision of the repository depending on the journal type.
+        Regular journals are subdivided per year and month,
+        while proceedings are subdivided per year and conference.
+        """
+
+        # TODO: Removing the whitespace should be more standardised
+        # Refactor: journal and year are common to both cases
+        # perhaps it is best to only return the subdivision month/conference
+        if proceedings_issue := self.stream.submission.proceedings:
+            return "{journal}/{year}/{conference}".format(
+                journal=self.journal_abbrev,
+                year=self.stream.submission.proceedings.event_end_date.year,
+                conference=proceedings_issue.event_suffix.replace(" ", ""),
+            )
+        else:
+            # Get creation date of the stream
+            # Warning: The month grouping of streams was done using the tasked date,
+            # but should now instead be the creation (opened) date.
+            opened_year, opened_month = self.stream.opened.strftime("%Y-%m").split("-")
+
+            return "{journal}/{year}/{month}".format(
+                journal=self.journal_abbrev,
+                year=opened_year,
+                month=opened_month,
+            )
+
+    @property
+    def git_path(self) -> str:
+        return "{ROOT}/Proofs/{journal_subdivision}/{repo_name}".format(
+            ROOT=settings.GITLAB_ROOT,
+            journal_subdivision=self.journal_subdivision,
+            repo_name=self.name,
+        )
+
+    @property
+    def git_url(self) -> str:
+        return "https://{GITLAB_URL}/{git_path}".format(
+            GITLAB_URL=settings.GITLAB_URL,
+            git_path=self.git_path,
+        )
+
+    @property
+    def git_ssh_clone_url(self) -> str:
+        return "git:{GITLAB_URL}/{git_path}.git".format(
+            GITLAB_URL=settings.GITLAB_URL,
+            git_path=self.git_path,
+        )
+
+    @property
+    def template_path(self) -> str:
+        """
+        Return the path to the template repository.
+        """
+        if self.stream.submission.proceedings is not None:
+            return "{ROOT}/Templates/{journal_subdivision}".format(
+                ROOT=settings.GITLAB_ROOT,
+                journal_subdivision=self.journal_subdivision,
+            )
+        else:
+            return "{ROOT}/Templates/{journal}".format(
+                ROOT=settings.GITLAB_ROOT,
+                journal=self.journal_abbrev,
+            )
+
+    def __str__(self) -> str:
+        return f"Proofs repo for {self.stream}"
+
+    class Meta:
+        verbose_name_plural = "proofs repositories"
+
+
+@receiver(post_save, sender=ProductionStream)
+def production_stream_create_proofs_repo(sender, instance, created, **kwargs):
+    """
+    When a ProductionStream instance is created, a ProofsRepository instance is created
+    and linked to it.
+    """
+    if created:
+        ProofsRepository.objects.create(
+            stream=instance,
+            status=ProofsRepository.PROOFS_REPO_UNINITIALIZED,
+        )
+
+
+post_save.connect(production_stream_create_proofs_repo, sender=ProductionStream)

scipost_django/production/tests/test_models.py

 __copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
 __license__ = "AGPL v3"
 
+import datetime
 
 from django.test import TestCase
 
 # Create your tests here.
+
+from submissions.constants import EIC_REC_PUBLISH
+from journals.models import Journal, Issue
+from submissions.models import Submission, EditorialDecision
+from production.models import ProductionStream, ProofsRepository
+from preprints.models import Preprint
+from ontology.models import AcademicField, Branch, Specialty
+from colleges.models import College
+from scipost.models import Contributor
+from profiles.models import Profile
+from proceedings.models import Proceedings
+
+from django.contrib.auth.models import User
+from django.conf import settings
+
+
+class TestProofRepository(TestCase):
+    def _create_submitter_contributor(self):
+        random_user = User.objects.create_user(
+            username="testuser",
+            password="testpassword",
+        )
+        user_profile = Profile.objects.create(
+            title="DR",
+            first_name="Test",
+            last_name="User",
+        )
+        Contributor.objects.create(user=random_user, profile=user_profile)
+
+    def _create_college(self):
+        College.objects.create(
+            name="College of Quantum Physics",
+            acad_field=AcademicField.objects.get(name="Quantum Physics"),
+            slug="college-of-quantum-physics",
+            order=10,
+        )
+
+    def _create_journal(self):
+        Journal.objects.create(
+            college=College.objects.get(name="College of Quantum Physics"),
+            name="SciPost Physics",
+            name_abbrev="SciPost Phys.",
+            doi_label="SciPostPhys",
+            cf_metrics='{"":""}',
+        )
+
+    def _create_editorial_decision(self):
+        EditorialDecision.objects.create(
+            submission=Submission.objects.get(
+                preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+            ),
+            for_journal=Journal.objects.get(name="SciPost Physics"),
+            decision=EIC_REC_PUBLISH,
+            status=EditorialDecision.FIXED_AND_ACCEPTED,
+        )
+
+    def _create_specialty(self):
+        Specialty.objects.create(
+            acad_field=AcademicField.objects.get(name="Quantum Physics"),
+            name="Quantum Information",
+            slug="quantum-information",
+            order=10,
+        )
+
+    def _create_academic_field(self):
+        AcademicField.objects.create(
+            branch=Branch.objects.get(name="Physics"),
+            name="Quantum Physics",
+            slug="quantum-physics",
+            order=10,
+        )
+
+    def _create_branch(self):
+        Branch.objects.create(
+            name="Physics",
+            slug="physics",
+            order=10,
+        )
+
+    def _create_preprint(self):
+        Preprint.objects.create(identifier_w_vn_nr="scipost_202101_00001v1")
+
+    def _create_submission(self):
+        submission = Submission.objects.create(
+            preprint=Preprint.objects.get(identifier_w_vn_nr="scipost_202101_00001v1"),
+            submitted_to=Journal.objects.get(name="SciPost Physics"),
+            title="Test submission",
+            abstract="Test abstract",
+            author_list="Test User",
+            acad_field=AcademicField.objects.get(name="Quantum Physics"),
+            # specialties=Specialty.objects.filter(name="Quantum Information"),
+            submitted_by=Contributor.objects.get(user__username="testuser"),
+        )
+        submission.authors.add(Contributor.objects.get(user__username="testuser"))
+        submission.save()
+
+    def _create_production_stream(self):
+        stream = ProductionStream.objects.create(
+            submission=Submission.objects.get(
+                preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+            ),
+        )
+        stream.opened = datetime.datetime(
+            2021, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc
+        )
+        stream.save()
+
+    def setUp(self):
+        self._create_submitter_contributor()
+        self._create_branch()
+        self._create_academic_field()
+        self._create_specialty()
+        self._create_college()
+        self._create_journal()
+        self._create_preprint()
+        self._create_submission()
+        self._create_editorial_decision()
+        self._create_production_stream()
+
+    def test_repo_name_existing_profile(self):
+        proofs_repo = ProofsRepository.objects.get(
+            stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+        )
+
+        self.assertEqual(proofs_repo.name, "scipost_202101_00001v1_User")
+
+    def test_repo_name_nonexisting_profile(self):
+        proofs_repo = ProofsRepository.objects.get(
+            stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+        )
+
+        # delete profile
+        Contributor.objects.get(user__username="testuser").profile.delete()
+
+        self.assertEqual(proofs_repo.name, "scipost_202101_00001v1_User")
+
+    def test_repo_name_double_last_name_profile(self):
+        proofs_repo = ProofsRepository.objects.get(
+            stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+        )
+
+        proofs_repo.stream.submission.author_list = "Test Usable User"
+
+        user_profile = Contributor.objects.get(user__username="testuser").profile
+        user_profile.last_name = "Usable User"
+        user_profile.save()
+
+        self.assertEqual(proofs_repo.name, "scipost_202101_00001v1_User")
+
+    def test_repo_name_two_authors(self):
+        proofs_repo = ProofsRepository.objects.get(
+            stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+        )
+
+        proofs_repo.stream.submission.author_list = (
+            "Another Personable Person, Test Usable User"
+        )
+
+        self.assertEqual(proofs_repo.name, "scipost_202101_00001v1_Person")
+
+    def test_repo_paths_scipostphys(self):
+        proofs_repo = ProofsRepository.objects.get(
+            stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+        )
+
+        settings.GITLAB_ROOT = "ProjectRoot"
+
+        self.assertEqual(
+            proofs_repo.git_path,
+            "ProjectRoot/Proofs/SciPostPhys/2021/01/scipost_202101_00001v1_User",
+        )
+
+        self.assertEqual(
+            proofs_repo.template_path,
+            "ProjectRoot/Templates/SciPostPhys",
+        )
+
+    def test_repo_paths_scipostphysproc(self):
+        proofs_repo = ProofsRepository.objects.get(
+            stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+        )
+
+        journal = Journal.objects.get(name="SciPost Physics")
+        journal.name = "SciPost Physics Proceedings"
+        journal.doi_label = "SciPostPhysProc"
+        journal.structure = "IO"  # proceedings, as Issues Only
+        journal.save()
+
+        issue = Issue.objects.create(
+            in_journal=journal,
+            number=1,
+            slug="proc-1",
+            doi_label="SciPostPhysProc.1",
+        )
+
+        proceedings = Proceedings.objects.create(
+            issue=issue,
+            submissions_open=datetime.datetime.now(),
+            submissions_close=datetime.datetime.now(),
+            submissions_deadline=datetime.datetime.now(),
+            event_end_date=datetime.datetime(2021, 5, 5),
+            event_start_date=datetime.datetime(2021, 5, 1),
+            event_suffix="ProcName21",
+        )
+
+        submission = Submission.objects.get(
+            preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+        )
+
+        submission.proceedings = proceedings
+        submission.save()
+
+        settings.GITLAB_ROOT = "ProjectRoot"
+
+        self.assertEqual(
+            proofs_repo.git_path,
+            "ProjectRoot/Proofs/SciPostPhysProc/2021/ProcName21/scipost_202101_00001v1_User",
+        )
+
+        self.assertEqual(
+            proofs_repo.template_path,
+            "ProjectRoot/Templates/SciPostPhysProc/2021/ProcName21",
+        )

scipost_django/scipost/management/commands/vet_superusers.py

+__copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
+__license__ = "AGPL v3"
+
+
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User, Group
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        superusers = User.objects.filter(is_superuser=True)
+        admin_group = Group.objects.get(name='SciPost Administrators')
+
+        for superuser in superusers:
+            superuser.groups.add(admin_group)
+            superuser.save()
+
+        self.stdout.write(self.style.SUCCESS(f"Successfully vetted {len(superusers)} superusers."))
\ No newline at end of file

scipost_django/submissions/factories/assignment.py

@@ -5,7 +5,7 @@ __license__ = "AGPL v3"
 import factory
 
 from scipost.models import Contributor
-from submissions.constants import ASSIGNMENT_STATUSES
+from submissions.models.submission import Submission
 from submissions.models import EditorialAssignment
 
 
@@ -18,7 +18,7 @@ class EditorialAssignmentFactory(factory.django.DjangoModelFactory):
 
     submission = None
     to = factory.Iterator(Contributor.objects.all())
-    status = factory.Iterator(ASSIGNMENT_STATUSES, getter=lambda c: c[0])
+    status = factory.Iterator(Submission.SUBMISSION_STATUSES, getter=lambda c: c[0])
     date_created = factory.lazy_attribute(lambda o: o.submission.latest_activity)
     date_answered = factory.lazy_attribute(lambda o: o.submission.latest_activity)
 

scipost_django/submissions/factories/referee_invitation.py

@@ -4,6 +4,7 @@ __license__ = "AGPL v3"
 
 import factory
 import pytz
+import random
 
 from faker import Faker
 
@@ -46,6 +47,7 @@ class AcceptedRefereeInvitationFactory(RefereeInvitationFactory):
     @factory.post_generation
     def report(self, create, extracted, **kwargs):
         if create:
+            from submissions.factories import VettedReportFactory
             VettedReportFactory(submission=self.submission, author=self.referee)
 
 

scipost_django/submissions/factories/submission.py

@@ -4,6 +4,7 @@ __license__ = "AGPL v3"
 
 import factory
 import pytz
+import random
 
 from faker import Faker
 
@@ -308,6 +309,11 @@ class PublishedSubmissionFactory(InRefereeingSubmissionFactory):
 
     @factory.post_generation
     def referee_invites(self, create, extracted, **kwargs):
+        from submissions.factories import (
+            FulfilledRefereeInvitationFactory,
+            CancelledRefereeInvitationFactory
+        )
+        
         for i in range(random.randint(2, 4)):
             FulfilledRefereeInvitationFactory(submission=self)
         for i in range(random.randint(0, 2)):