diff --git a/.gitignore b/.gitignore
index ad0553d4399d9b09c9050f1223d7836a6c7cec46..e32685c0e24f3a2218ddc2e073c732e54b188f46 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,11 +16,11 @@ __pycache__
# Package managers
/venv*
+/.venv*
/node_modules/
*webpack-stats.json
.python-version
-*secrets.json
!package.json
!package-lock.json
!package.vue.json
@@ -51,3 +51,10 @@ start_flower.sh
whoosh_index
*.pid
+
+# Sensitive files
+*/secrets.json
+/scipost_django/SciPost_v1/settings/local_*.py
+
+# Editor configs
+*.code-workspace
\ No newline at end of file
diff --git a/cronjobs/cronjob_production_eachhour.sh b/cronjobs/cronjob_production_eachhour.sh
index 4189c42039067ed6b6d37cf505aefbc1e3f30183..4cae191cb52431c1c9210d882b55b84e4a69d044 100755
--- a/cronjobs/cronjob_production_eachhour.sh
+++ b/cronjobs/cronjob_production_eachhour.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-# Per minute cronjobs for production area
+# Per hour cronjobs for production area
cd /home/scipost/SciPost/scipost_django
source ../venv-3.8.5/bin/activate
@@ -8,6 +8,7 @@ source ../venv-3.8.5/bin/activate
# Do tasks
python manage.py check_celery --settings=SciPost_v1.settings.production_do1
python manage.py update_coi_via_arxiv --settings=SciPost_v1.settings.production_do1
+python manage.py advance_git_repos --settings=SciPost_v1.settings.production_do1
# Do a update_index of the last hour
python manage.py update_index -r -v 0 -a 1 --settings=SciPost_v1.settings.production_do1
diff --git a/requirements.txt b/requirements.txt
index f66a1294c612a0d2b04587ac93c586a2d5b6a496..351d26ebd052cd4bfc4323ed13aaaeb156974c0a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ argon2-cffi==20.1.0 # 2021-07-18 Password hashing algorithm
Babel==2.9.1 # 2022-01-23
Django==3.2.18 # 2023-05-06
feedparser~=6.0.8 # Check: not updated since 2016. [JdW, 2021-09-25] Upgrade to v6; v5 fails.
-psycopg2==2.8.6 # 2020-09-19 PostgreSQL engine
+psycopg2==2.9.5 # 2020-09-19 PostgreSQL engine -- 2023-05-10 update for python 3.11
pytz==2021.3 # 2022-11-18 Timezone package
# djangorestframework==3.9.3 # DEPREC, see next entry -- 2019-12-05 IMPORTANT: update templates/rest_framework/base.html if corresponding file rest_framework/templates/rest_framework/base.html has changed
git+https://github.com/SciPost/django-rest-framework.git@bootstrap-v5
@@ -80,9 +80,15 @@ celery==5.2.7 # 2022-11-18
django-celery-results==2.4.0 # 2022-11-18
django-celery-beat==2.4.0 # 2022-11-18
flower==1.2.0 # 2022-11-18
-
+mailchimp3==3.0.18 # 2023-05-09
# Security-related packages
django-referrer-policy==1.0 # 2020-09-19 no new updates for 3 years
django-csp==3.7 # 2020-09-19
django-feature-policy==3.4.0 # 2020-09-19
+
+# Version Control
+python-gitlab==3.14.0 # 2023-05-15
+
+# Preprint server packages
+arxiv==1.4.7 # 2023-05-19
\ No newline at end of file
diff --git a/scipost_django/SciPost_v1/settings/base.py b/scipost_django/SciPost_v1/settings/base.py
index 9d3501b308216bc335cf58bffae9af5cd022f918..64bce3e662eb780d2bbb62384e5f44f344deb24f 100644
--- a/scipost_django/SciPost_v1/settings/base.py
+++ b/scipost_django/SciPost_v1/settings/base.py
@@ -570,3 +570,8 @@ DISCOURSE_SSO_SECRET = get_secret("DISCOURSE_SSO_SECRET")
CORS_ALLOWED_ORIGINS = [
"https://git.scipost.org",
]
+
+# GitLab API
+GITLAB_ROOT = "SciPost"
+GITLAB_URL = "git.scipost.org"
+GITLAB_KEY = get_secret("GITLAB_KEY")
diff --git a/scipost_django/SciPost_v1/settings/production_do1.py b/scipost_django/SciPost_v1/settings/production_do1.py
index fc9f610f80ae5791813ba11719cd65b809fa5013..14a7feea3d912faf4d534efc6c2f9715388c4001 100644
--- a/scipost_django/SciPost_v1/settings/production_do1.py
+++ b/scipost_django/SciPost_v1/settings/production_do1.py
@@ -88,3 +88,8 @@ CSP_REPORT_ONLY = False
CORS_ALLOWED_ORIGINS = [
"https://git.scipost.org",
]
+
+# GitLab API
+GITLAB_ROOT = "SciPost"
+GITLAB_URL = "git.scipost.org"
+GITLAB_KEY = get_secret("GITLAB_KEY")
diff --git a/scipost_django/production/admin.py b/scipost_django/production/admin.py
index ac10d9871ead9b7ff01ec0ab3e7a5de867dcbfa6..645fc77ac952b5335078de76f3639077e25a2bcb 100644
--- a/scipost_django/production/admin.py
+++ b/scipost_django/production/admin.py
@@ -12,6 +12,7 @@ from .models import (
ProductionUser,
Proofs,
ProductionEventAttachment,
+ ProofsRepository,
)
@@ -95,3 +96,17 @@ admin.site.register(Proofs, ProductionProofsAdmin)
admin.site.register(ProductionEventAttachment)
+
+
+class ProofsRepositoryAdmin(GuardedModelAdmin):
+ search_fields = [
+ "stream__submission__author_list",
+ "stream__submission__title",
+ "stream__submission__preprint__identifier_w_vn_nr",
+ ]
+ list_filter = ["status"]
+ list_display = ["stream", "status", "git_path"]
+ readonly_fields = ["template_path", "git_path"]
+
+
+admin.site.register(ProofsRepository, ProofsRepositoryAdmin)
diff --git a/scipost_django/production/management/__init__.py b/scipost_django/production/management/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/scipost_django/production/management/commands/__init__.py b/scipost_django/production/management/commands/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/scipost_django/production/management/commands/advance_git_repos.py b/scipost_django/production/management/commands/advance_git_repos.py
new file mode 100644
index 0000000000000000000000000000000000000000..957d417be9d8f5513ec20013761183f4169232fc
--- /dev/null
+++ b/scipost_django/production/management/commands/advance_git_repos.py
@@ -0,0 +1,533 @@
+__copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
+__license__ = "AGPL v3"
+
+from datetime import datetime
+from functools import reduce
+from itertools import cycle
+from typing import Any, Callable, Dict, List, Tuple
+from django.core.management.base import BaseCommand, CommandParser
+from django.conf import settings
+
+from common.utils import get_current_domain
+
+from gitlab import Gitlab
+from gitlab.v4.objects import Group, Project
+from gitlab.exceptions import GitlabGetError
+
+import arxiv
+import requests
+import tarfile
+from base64 import b64encode
+
+
+from production.models import ProofsRepository
+from production.constants import (
+ PROOFS_REPO_UNINITIALIZED,
+ PROOFS_REPO_CREATED,
+ PROOFS_REPO_TEMPLATE_ONLY,
+ PROOFS_REPO_TEMPLATE_FORMATTED,
+ PROOFS_REPO_PRODUCTION_READY,
+)
+
+
+class Command(BaseCommand):
+ """
+ This command handles the creation and updating of git repositories.
+ """
+
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+
+ # Check that the global GITLAB_ROOT constant is set
+ if not hasattr(settings, "GITLAB_ROOT") or settings.GITLAB_ROOT == "":
+ raise LookupError(
+ "Constant `GITLAB_ROOT` is either not present in settings file or empty, please add it."
+ )
+
+ self.GL: Gitlab = self._instanciate_gitlab()
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument(
+ "--id",
+ type=int,
+ required=False,
+ help="The submission preprint identifier to handle a specific submission, leave blank to handle all",
+ )
+
+ def _instanciate_gitlab(self) -> Gitlab:
+ """
+ Test the connection to the git server, returns a Gitlab object.
+ """
+
+ if not hasattr(settings, "GITLAB_KEY") or settings.GITLAB_KEY == "":
+ raise LookupError(
+ "Constant `GITLAB_KEY` is either not present in secret file or empty, please add it."
+ )
+
+ if not hasattr(settings, "GITLAB_URL") or settings.GITLAB_URL == "":
+ raise LookupError(
+ "Constant `GITLAB_URL` is either not present in secret file or empty, please add it."
+ )
+
+ GL = Gitlab(
+ url="https://" + settings.GITLAB_URL,
+ private_token=settings.GITLAB_KEY,
+ )
+
+ try:
+ GL.auth()
+ except Exception as e:
+ raise AssertionError(
+ "Could not authenticate with GitLab, please check your credentials."
+ ) from e
+
+ return GL
+
+ def _get_or_create_nested_group(self, group_path: str) -> Group:
+ """
+ Create a new group on the git server based on a path of nested folders.
+ """
+ parent_group = None
+ group_path_segments = group_path.split("/")
+
+ # Traverse the path segments (up to the second-to-last one)
+ # and create the groups if they do not exist
+ for i, group_path_segment in enumerate(group_path_segments[:-1]):
+ path_up_to_segment_i = "/".join(group_path_segments[: i + 1])
+
+ # Check if group exists in the server
+ try:
+ group = self.GL.groups.get(path_up_to_segment_i)
+
+ # If it does not exist, create it
+ except GitlabGetError:
+ # Guard against the root group not existing
+ if parent_group is None:
+ raise AssertionError(
+ f"The parent group of {path_up_to_segment_i} does not exist. "
+ "This should not happen normally (and would not be fixable "
+ "because GitLab does not allow root groups to be created)."
+ )
+
+ # Create the group
+ group = self.GL.groups.create(
+ {
+ "name": group_path_segment,
+ "path": group_path_segment,
+ "parent_id": parent_group.id,
+ "visibility": "private",
+ }
+ )
+
+ # Set the parent group to the current group
+ parent_group = group
+
+ return group
+
+ def _create_git_repo(self, repo: ProofsRepository):
+ """
+ Create a new git repository for the submission.
+ """
+ # Check if repo exists in the server
+ try:
+ project = self.GL.projects.get(repo.git_path)
+
+ # Create the repo on the server
+ except GitlabGetError:
+ # Get the namespace id
+ parent_group_id = self._get_or_create_nested_group(repo.git_path).id
+ project = self.GL.projects.create(
+ {
+ "name": repo.name,
+ "namespace_id": parent_group_id,
+ "visibility": "private",
+ "description": "Proofs for https://{domain}/submissions/{preprint_id}".format(
+ domain=get_current_domain(),
+ preprint_id=repo.stream.submission.preprint.identifier_w_vn_nr,
+ ),
+ }
+ )
+
+ self.stdout.write(
+ self.style.SUCCESS(f"Created git repository at {repo.git_path}")
+ )
+
+ def _get_project_cloning_actions(self, project: Project) -> List[Dict[str, Any]]:
+ """
+ Return a list of gitlab actions required to fully clone a project.
+ """
+ filenames = list(
+ map(lambda x: x["path"], project.repository_tree(get_all=True))
+ )
+
+ actions = []
+ for filename in filenames:
+ try:
+ file = project.files.get(file_path=filename, ref="main")
+ except:
+ self.stdout.write(
+ self.style.WARNING(f"File {filename} not found in {project.name}")
+ )
+ continue
+
+ actions.append(
+ {
+ "action": "create",
+ "file_path": filename,
+ "content": file.content,
+ "encoding": "base64",
+ }
+ )
+
+ return actions
+
+ def _copy_pure_templates(self, repo: ProofsRepository):
+ """
+ Copy the pure templates to the repo.
+ """
+ project = self.GL.projects.get(repo.git_path)
+
+ journal_template_project = self.GL.projects.get(repo.template_path)
+ base_template_project = self.GL.projects.get(
+ "{ROOT}/Templates/Base".format(ROOT=settings.GITLAB_ROOT)
+ )
+
+ base_actions = self._get_project_cloning_actions(base_template_project)
+ journal_actions = self._get_project_cloning_actions(journal_template_project)
+
+ # Commit the actions
+ project.commits.create(
+ {
+ "branch": "main",
+ "commit_message": "copy pure templates",
+ "actions": base_actions + journal_actions,
+ }
+ )
+
+ self.stdout.write(
+ self.style.SUCCESS(f"Copied pure templates to {repo.git_path}")
+ )
+
+ def _format_skeleton(self, repo: ProofsRepository):
+ """
+ Format the Skeleton.tex file of the repo to include basic information about the submission.
+ """
+
+ SHAPES = ["star", "dagger", "ddagger", "circ", "S", "P", "parallel"]
+ SLASH = "\\"
+ NEWLINE = f"{SLASH}{SLASH}"
+
+ def abbreviate_author(author: str) -> str:
+ """
+ Abbreviate an author's name by taking the first letter\
+ of their first and middle names, and their full last name.
+ """
+
+ # TODO: This is somewhat naive, but it should work for now.
+ first_name, *middle_names, last_name = author.split(" ")
+ # Ideally, I would like to search for matching authors in the database
+ # and abbreviate their names accordingly to the journal's style.
+ # Right now, I abbreviate only the very first name and leave the rest as is.
+
+ # Map each part of the (optionally) hyphenated first name to its abbreviation
+ # (e.g. "John-Edward" -> "J.-E.")
+ first_name_hyphen_parts = first_name.split("-")
+ first_name_hyphen_parts_abbrev = list(
+ map(lambda x: x[0].upper() + ".", first_name_hyphen_parts)
+ )
+
+ # Add different name parts to the abbreviation, glue them together with space
+ # (e.g. "John-Edward Brown Smith" -> "J.-E. Brown Smith")
+ abbreviation_parts = [
+ "-".join(first_name_hyphen_parts_abbrev), # Abbreviated first name
+ *middle_names,
+ last_name,
+ ]
+
+ return " ".join(abbreviation_parts)
+
+ # Define the formatting functions
+ def format_authors(authors: List[str]) -> str:
+ *other_authors, last_author = authors
+
+ if len(other_authors) == 0:
+ return last_author
+ else:
+ return ", ".join(other_authors) + " and " + last_author
+
+ def format_title(title: str) -> str:
+ return title + NEWLINE
+
+ def format_copyright(authors: List[str]) -> str:
+ """
+ Format the copyright statement depending on the number\
+ of authors in the submission:
+ - 1 author: "© Author"
+ - 2 authors: "© Author1 and Author2"
+ - 3+ authors: "© Author1 et al"
+ """
+ if len(authors) == 1:
+ return f"Copyright {authors[0]}"
+ elif len(authors) == 2:
+ return f"Copyright {authors[0]} and {authors[1]}"
+ else:
+ return f"Copyright {authors[0]} {{{SLASH}it et al}}"
+
+ def format_emails(authors: List[str]) -> str:
+ """
+ Format the emails of the authors in the submission, grouped by 3 per line.\
+ The emails are padded with \\quad spacing and are prepended with a shape.
+ """
+ # Create a list array of emails, grouped by 3
+ mail_lines = [[]]
+ mail_line_i = 0
+ for i, (_, shape) in enumerate(zip(authors, cycle(SHAPES))):
+ mail_lines[mail_line_i].append(
+ f"${SLASH}{shape}$ {SLASH}href{{mailto:email{i+1}}}{{{SLASH}small email{i+1}}}"
+ )
+
+ # Create a new mail group every 3 emails
+ if (i + 1) % 3 == 0:
+ mail_line_i += 1
+ mail_lines.append([])
+
+ # Flatten the inner lists and join them with "\,,\quad"
+ flattened_mail_lines = [
+ f"{SLASH},,{SLASH}quad\n".join(line) for line in mail_lines
+ ]
+
+ # Join the lines with "\,,\\"
+ flattened_mails = f"{SLASH},,{NEWLINE}\n".join(flattened_mail_lines)
+
+ return flattened_mails
+
+ def format_affiliations(authors: List[str]) -> str:
+ """
+ Format the affiliations of the authors in the submission,
+ by including the author's name and the affiliation number.
+ There is one affiliation per author by default.
+ """
+ affiliations = []
+ for i, author in enumerate(authors):
+ affiliations += [f"{{{SLASH}bf {i+1}}} Affiliation {author}"]
+
+ return f"\n{NEWLINE}\n".join(affiliations)
+
+ def format_date_human_readable(date: datetime) -> str:
+ """
+ Format a date in a human-readable format (DD-MM-YYY).
+ """
+ return date.strftime("%d-%m-%Y")
+
+ project = self.GL.projects.get(repo.git_path)
+ project_filenames = list(
+ map(lambda x: x["path"], project.repository_tree(get_all=True))
+ )
+
+ skeleton_filename = next(
+ filter(lambda x: x.endswith("Skeleton.tex"), project_filenames)
+ )
+ skeleton_file = project.files.get(file_path=skeleton_filename, ref="main")
+ skeleton_content = skeleton_file.decode().decode("utf-8")
+
+ # Collect the information about the paper
+ paper_title = repo.stream.submission.title
+ paper_abbreviated_authors = list(
+ map(abbreviate_author, repo.stream.submission.authors_as_list)
+ )
+ paper_received_date = repo.stream.submission.original_submission_date
+ paper_acceptance_date = repo.stream.submission.acceptance_date
+
+ # Create the replacement dictionary from placeholders and information
+ # key = placeholder, value = (formatting_function, *args)
+ replacements_dict = {
+ "<|TITLE|>": (format_title, paper_title),
+ "<|AUTHORS|>": (format_authors, repo.stream.submission.authors_as_list),
+ "<|EMAILS|>": (format_emails, paper_abbreviated_authors),
+ "<|COPYRIGHT|>": (format_copyright, paper_abbreviated_authors),
+ "<|AFFILIATIONS|>": (format_affiliations, paper_abbreviated_authors),
+ "<|RECEIVED|>": (format_date_human_readable, paper_received_date),
+ "<|ACCEPTED|>": (format_date_human_readable, paper_acceptance_date),
+ }
+
+ # Define a helper function to try to format and replace a placeholder
+ # which catches any errors and prints them to the console non-intrusively
+ def try_format_replace(
+ text: str,
+ key: str,
+ value: Tuple[Callable[[Any], str], Any],
+ ):
+ try:
+ formatting_function, *args = value
+ formatted_value = formatting_function(*args)
+ return text.replace(key, formatted_value)
+ except:
+ self.stdout.write(
+ self.style.ERROR(
+ f"Could not format and replace {key} with {value} in {repo.git_path}"
+ )
+ )
+ return text
+
+ # Replace the placeholders with the submission information
+ # by iteratively applying the formatting functions to the skeleton
+ skeleton_content = reduce(
+ lambda text, replace_pair: try_format_replace(text, *replace_pair),
+ replacements_dict.items(),
+ skeleton_content,
+ )
+
+ # Commit the changes to the skeleton file and change its name
+ project.commits.create(
+ {
+ "branch": "main",
+ "commit_message": f"format skeleton file",
+ "actions": [
+ {
+ "action": "move",
+ "content": skeleton_content,
+ "previous_path": skeleton_filename,
+ # Change the "Skeleton" part from the filename to the repo name
+ # and remove the extraneous "scipost_" label from the identifier slug
+ "file_path": skeleton_filename.replace(
+ "Skeleton",
+ repo.name.replace("scipost_", ""),
+ ),
+ },
+ ],
+ }
+ )
+
+ self.stdout.write(
+ self.style.SUCCESS(
+ f"Successfully formatted the skeleton of {repo.git_path}"
+ )
+ )
+
+ def _copy_arxiv_source_files(self, repo: ProofsRepository):
+ paper = next(
+ arxiv.Search(
+ id_list=[repo.stream.submission.preprint.identifier_w_vn_nr]
+ ).results()
+ )
+ source_stream = requests.get(paper.pdf_url.replace("pdf", "src"), stream=True)
+
+ # Create file creation actions for each file in the source tar
+ actions = []
+ with tarfile.open(fileobj=source_stream.raw) as tar:
+ for member in tar:
+ if not member.isfile():
+ continue
+
+ f = tar.extractfile(member)
+ try:
+ bin_content = f.read()
+ actions.append(
+ {
+ "action": "create",
+ "file_path": member.name,
+ "encoding": "base64",
+ # Encode the binary content in base64, required by the API
+ "content": b64encode(bin_content).decode("utf-8"),
+ }
+ )
+
+ except:
+ self.stdout.write(
+ self.style.ERROR(
+ f"Could not read {member.name} from the arXiv source files, skipping..."
+ )
+ )
+
+ # Filter out the files that already exist in the repo to avoid conflicts
+ project = self.GL.projects.get(repo.git_path)
+ project_existing_filenames = list(
+ map(lambda x: x["path"], project.repository_tree(get_all=True))
+ )
+
+ non_existing_file_actions = [
+ action
+ for action in actions
+ if action["file_path"] not in project_existing_filenames
+ ]
+
+ # Commit the creation of the files
+ project.commits.create(
+ {
+ "branch": "main",
+ "commit_message": f"copy arXiv source files",
+ "actions": non_existing_file_actions,
+ }
+ )
+
+ self.stdout.write(
+ self.style.SUCCESS(
+ f"Successfully copied the author source files to {repo.git_path}"
+ )
+ )
+
+ def handle(self, *args, **options):
+ # Limit the actions to a specific submission if requested
+ if preprint_id := options.get("id"):
+ repos = ProofsRepository.objects.filter(
+ stream__submission__preprint__identifier_w_vn_nr=preprint_id
+ )
+ else:
+ repos = ProofsRepository.objects.all()
+
+ # Create the repos
+ repos_to_be_created = repos.filter(status=PROOFS_REPO_UNINITIALIZED)
+ for repo in repos_to_be_created:
+ try:
+ self._create_git_repo(repo)
+ repo.status = PROOFS_REPO_CREATED
+ repo.save()
+ except Exception as e:
+ self.stdout.write(
+ self.style.ERROR(
+ f"Could not create the git repo for {repo.git_path}, error: {e}"
+ )
+ )
+
+ # Copy the pure templates
+ repos_to_be_templated = repos.filter(status=PROOFS_REPO_CREATED)
+ for repo in repos_to_be_templated:
+ try:
+ self._copy_pure_templates(repo)
+ repo.status = PROOFS_REPO_TEMPLATE_ONLY
+ repo.save()
+ except Exception as e:
+ self.stdout.write(
+ self.style.ERROR(
+ f"Could not copy the pure templates to {repo.git_path}, error: {e}"
+ )
+ )
+
+ # Format the skeleton files
+ repos_to_be_formatted = repos.filter(status=PROOFS_REPO_TEMPLATE_ONLY)
+ for repo in repos_to_be_formatted:
+ try:
+ self._format_skeleton(repo)
+ repo.status = PROOFS_REPO_TEMPLATE_FORMATTED
+ repo.save()
+ except Exception as e:
+ self.stdout.write(
+ self.style.ERROR(
+ f"Could not format the skeleton of {repo.git_path}, error: {e}"
+ )
+ )
+
+ # Copy the arXiv source files
+ repos_to_be_copied = repos.filter(status=PROOFS_REPO_TEMPLATE_FORMATTED)
+ for repo in repos_to_be_copied:
+ try:
+ if "arxiv.org" in repo.stream.submission.preprint.url:
+ self._copy_arxiv_source_files(repo)
+ repo.status = PROOFS_REPO_PRODUCTION_READY
+ repo.save()
+ except Exception as e:
+ self.stdout.write(
+ self.style.ERROR(
+ f"Could not copy the arXiv source files to {repo.git_path}, error: {e}"
+ )
+ )
diff --git a/scipost_django/production/migrations/0006_proofsrepository.py b/scipost_django/production/migrations/0006_proofsrepository.py
new file mode 100644
index 0000000000000000000000000000000000000000..8865b3441e8fe8335de26d6a681dd95d8a99a440
--- /dev/null
+++ b/scipost_django/production/migrations/0006_proofsrepository.py
@@ -0,0 +1,25 @@
+# Generated by Django 3.2.18 on 2023-05-15 14:25
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('production', '0005_auto_20190511_1141'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='ProofsRepository',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('status', models.CharField(choices=[('uninitialized', 'The repository does not exist'), ('created', 'The repository exists but is empty'), ('template_only', 'The repository contains the bare template'), ('template_formatted', 'The repository contains the automatically formatted template'), ('production_ready', 'The repository is ready for production')], default='uninitialized', max_length=32)),
+ ('stream', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='proofs_repository', to='production.productionstream')),
+ ],
+ options={
+ 'verbose_name_plural': 'proofs repositories',
+ },
+ ),
+ ]
diff --git a/scipost_django/production/models.py b/scipost_django/production/models.py
index 65ef7165884446c0f51b4fe9e3d1ab2cafd7c388..e168ea5a5b91bd1c412399caf30e4227097d2f4a 100644
--- a/scipost_django/production/models.py
+++ b/scipost_django/production/models.py
@@ -6,8 +6,14 @@ from django.db import models
from django.contrib.contenttypes.fields import GenericRelation
from django.urls import reverse
from django.contrib.auth.models import User
+from profiles.models import Profile
from django.utils import timezone
from django.utils.functional import cached_property
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from django.db.models import Value
+from django.db.models.functions import Concat
+from django.conf import settings
from .constants import (
PRODUCTION_STREAM_STATUS,
@@ -126,17 +132,11 @@ class ProductionStream(models.Model):
class ProductionEvent(models.Model):
- stream = models.ForeignKey(
- ProductionStream, on_delete=models.CASCADE, related_name="events"
- )
- event = models.CharField(
- max_length=64, choices=PRODUCTION_EVENTS, default=EVENT_MESSAGE
- )
+ stream = models.ForeignKey(ProductionStream, on_delete=models.CASCADE, related_name="events")
+ event = models.CharField(max_length=64, choices=PRODUCTION_EVENTS, default=EVENT_MESSAGE)
comments = models.TextField(blank=True, null=True)
noted_on = models.DateTimeField(default=timezone.now)
- noted_by = models.ForeignKey(
- "production.ProductionUser", on_delete=models.CASCADE, related_name="events"
- )
+ noted_by = models.ForeignKey("production.ProductionUser", on_delete=models.CASCADE, related_name="events")
noted_to = models.ForeignKey(
"production.ProductionUser",
on_delete=models.CASCADE,
@@ -159,10 +159,7 @@ class ProductionEvent(models.Model):
@cached_property
def editable(self):
- return (
- self.event in [EVENT_MESSAGE, EVENT_HOUR_REGISTRATION]
- and not self.stream.completed
- )
+ return self.event in [EVENT_MESSAGE, EVENT_HOUR_REGISTRATION] and not self.stream.completed
def production_event_upload_location(instance, filename):
@@ -185,9 +182,7 @@ class ProductionEventAttachment(models.Model):
on_delete=models.CASCADE,
related_name="attachments",
)
- attachment = models.FileField(
- upload_to=production_event_upload_location, storage=SecureFileStorage()
- )
+ attachment = models.FileField(upload_to=production_event_upload_location, storage=SecureFileStorage())
def get_absolute_url(self):
return reverse(
@@ -213,20 +208,12 @@ class Proofs(models.Model):
Proofs are directly related to a ProductionStream and Submission in SciPost.
"""
- attachment = models.FileField(
- upload_to=proofs_upload_location, storage=SecureFileStorage()
- )
+ attachment = models.FileField(upload_to=proofs_upload_location, storage=SecureFileStorage())
version = models.PositiveSmallIntegerField(default=0)
- stream = models.ForeignKey(
- "production.ProductionStream", on_delete=models.CASCADE, related_name="proofs"
- )
- uploaded_by = models.ForeignKey(
- "production.ProductionUser", on_delete=models.CASCADE, related_name="+"
- )
+ stream = models.ForeignKey("production.ProductionStream", on_delete=models.CASCADE, related_name="proofs")
+ uploaded_by = models.ForeignKey("production.ProductionUser", on_delete=models.CASCADE, related_name="+")
created = models.DateTimeField(auto_now_add=True)
- status = models.CharField(
- max_length=16, choices=PROOFS_STATUSES, default=PROOFS_UPLOADED
- )
+ status = models.CharField(max_length=16, choices=PROOFS_STATUSES, default=PROOFS_UPLOADED)
accessible_for_authors = models.BooleanField(default=False)
objects = ProofsQuerySet.as_manager()
@@ -239,9 +226,7 @@ class Proofs(models.Model):
return reverse("production:proofs_pdf", kwargs={"slug": self.slug})
def __str__(self):
- return "Proofs {version} for Stream {stream}".format(
- version=self.version, stream=self.stream.submission.title
- )
+ return "Proofs {version} for Stream {stream}".format(version=self.version, stream=self.stream.submission.title)
def save(self, *args, **kwargs):
# Control Report count per Submission.
@@ -252,3 +237,156 @@ class Proofs(models.Model):
@property
def slug(self):
return proofs_id_to_slug(self.id)
+
+
+class ProofsRepository(models.Model):
+ """
+ ProofsRepository is a GitLab repository of Proofs for a Submission.
+ """
+
+ PROOFS_REPO_UNINITIALIZED = "uninitialized"
+ PROOFS_REPO_CREATED = "created"
+ PROOFS_REPO_TEMPLATE_ONLY = "template_only"
+ PROOFS_REPO_TEMPLATE_FORMATTED = "template_formatted"
+ PROOFS_REPO_PRODUCTION_READY = "production_ready"
+ PROOFS_REPO_STATUSES = (
+ (PROOFS_REPO_UNINITIALIZED, "The repository does not exist"),
+ (PROOFS_REPO_CREATED, "The repository exists but is empty"),
+ (PROOFS_REPO_TEMPLATE_ONLY, "The repository contains the bare template"),
+ (
+ PROOFS_REPO_TEMPLATE_FORMATTED,
+ "The repository contains the automatically formatted template",
+ ),
+ (PROOFS_REPO_PRODUCTION_READY, "The repository is ready for production"),
+ )
+
+ stream = models.OneToOneField(
+ ProductionStream,
+ on_delete=models.CASCADE,
+ related_name="proofs_repository",
+ )
+ status = models.CharField(
+ max_length=32,
+ choices=PROOFS_REPO_STATUSES,
+ default=PROOFS_REPO_UNINITIALIZED,
+ )
+
+ @property
+ def name(self) -> str:
+ """
+ Return the name of the repository in the form of "id_lastname".
+ """
+ # Get the last name of the first author by getting the first author string from the submission
+ first_author_str = self.stream.submission.authors_as_list[0]
+ first_author_profile = (
+ Profile.objects.annotate(
+ full_name=Concat("first_name", Value(" "), "last_name")
+ )
+ .filter(full_name=first_author_str)
+ .first()
+ )
+ if first_author_profile is None:
+ first_author_last_name = first_author_str.split(" ")[-1]
+ else:
+ first_author_last_name = first_author_profile.last_name
+ # Keep only the last of the last names
+ first_author_last_name = first_author_last_name.split(" ")[-1]
+
+ return "{preprint_id}_{last_name}".format(
+ preprint_id=self.stream.submission.preprint.identifier_w_vn_nr,
+ last_name=first_author_last_name,
+ )
+
+ @property
+ def journal_abbrev(self) -> str:
+ # The DOI label is used to determine the path of the repository and template
+ return self.stream.submission.editorial_decision.for_journal.doi_label
+
+ @property
+ def journal_subdivision(self) -> str:
+ """
+ Return the subdivision of the repository depending on the journal type.
+ Regular journals are subdivided per year and month,
+ while proceedings are subdivided per year and conference.
+ """
+
+ # TODO: Removing the whitespace should be more standardised
+ # Refactor: journal and year are common to both cases
+ # perhaps it is best to only return the subdivision month/conference
+ if proceedings_issue := self.stream.submission.proceedings:
+ return "{journal}/{year}/{conference}".format(
+ journal=self.journal_abbrev,
+ year=self.stream.submission.proceedings.event_end_date.year,
+ conference=proceedings_issue.event_suffix.replace(" ", ""),
+ )
+ else:
+ # Get creation date of the stream
+ # Warning: The month grouping of streams was done using the tasked date,
+ # but should now instead be the creation (opened) date.
+ opened_year, opened_month = self.stream.opened.strftime("%Y-%m").split("-")
+
+ return "{journal}/{year}/{month}".format(
+ journal=self.journal_abbrev,
+ year=opened_year,
+ month=opened_month,
+ )
+
+ @property
+ def git_path(self) -> str:
+ return "{ROOT}/Proofs/{journal_subdivision}/{repo_name}".format(
+ ROOT=settings.GITLAB_ROOT,
+ journal_subdivision=self.journal_subdivision,
+ repo_name=self.name,
+ )
+
+ @property
+ def git_url(self) -> str:
+ return "https://{GITLAB_URL}/{git_path}".format(
+ GITLAB_URL=settings.GITLAB_URL,
+ git_path=self.git_path,
+ )
+
+ @property
+ def git_ssh_clone_url(self) -> str:
+ return "git:{GITLAB_URL}/{git_path}.git".format(
+ GITLAB_URL=settings.GITLAB_URL,
+ git_path=self.git_path,
+ )
+
+ @property
+ def template_path(self) -> str:
+ """
+ Return the path to the template repository.
+ """
+ if self.stream.submission.proceedings is not None:
+ return "{ROOT}/Templates/{journal_subdivision}".format(
+ ROOT=settings.GITLAB_ROOT,
+ journal_subdivision=self.journal_subdivision,
+ )
+ else:
+ return "{ROOT}/Templates/{journal}".format(
+ ROOT=settings.GITLAB_ROOT,
+ journal=self.journal_abbrev,
+ )
+
+ def __str__(self) -> str:
+ return f"Proofs repo for {self.stream}"
+
+ class Meta:
+ verbose_name_plural = "proofs repositories"
+
+
+@receiver(post_save, sender=ProductionStream)
+def production_stream_create_proofs_repo(sender, instance, created, **kwargs):
+ """
+ When a ProductionStream instance is created, a ProofsRepository instance is created
+ and linked to it.
+ """
+ if created:
+ ProofsRepository.objects.create(
+ stream=instance,
+ status=ProofsRepository.PROOFS_REPO_UNINITIALIZED,
+ )
+
+
+post_save.connect(production_stream_create_proofs_repo, sender=ProductionStream)
diff --git a/scipost_django/production/tests/test_models.py b/scipost_django/production/tests/test_models.py
index ddef03c4df91383dfce15a034976e3a469d77d70..e35c6d72e48182c4ed96b86204e5deeeaf51c62d 100644
--- a/scipost_django/production/tests/test_models.py
+++ b/scipost_django/production/tests/test_models.py
@@ -1,7 +1,231 @@
__copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
__license__ = "AGPL v3"
+import datetime
from django.test import TestCase
# Create your tests here.
+
+from submissions.constants import EIC_REC_PUBLISH
+from journals.models import Journal, Issue
+from submissions.models import Submission, EditorialDecision
+from production.models import ProductionStream, ProofsRepository
+from preprints.models import Preprint
+from ontology.models import AcademicField, Branch, Specialty
+from colleges.models import College
+from scipost.models import Contributor
+from profiles.models import Profile
+from proceedings.models import Proceedings
+
+from django.contrib.auth.models import User
+from django.conf import settings
+
+
+class TestProofRepository(TestCase):
+ def _create_submitter_contributor(self):
+ random_user = User.objects.create_user(
+ username="testuser",
+ password="testpassword",
+ )
+ user_profile = Profile.objects.create(
+ title="DR",
+ first_name="Test",
+ last_name="User",
+ )
+ Contributor.objects.create(user=random_user, profile=user_profile)
+
+ def _create_college(self):
+ College.objects.create(
+ name="College of Quantum Physics",
+ acad_field=AcademicField.objects.get(name="Quantum Physics"),
+ slug="college-of-quantum-physics",
+ order=10,
+ )
+
+ def _create_journal(self):
+ Journal.objects.create(
+ college=College.objects.get(name="College of Quantum Physics"),
+ name="SciPost Physics",
+ name_abbrev="SciPost Phys.",
+ doi_label="SciPostPhys",
+ cf_metrics='{"":""}',
+ )
+
+ def _create_editorial_decision(self):
+ EditorialDecision.objects.create(
+ submission=Submission.objects.get(
+ preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+ ),
+ for_journal=Journal.objects.get(name="SciPost Physics"),
+ decision=EIC_REC_PUBLISH,
+ status=EditorialDecision.FIXED_AND_ACCEPTED,
+ )
+
+ def _create_specialty(self):
+ Specialty.objects.create(
+ acad_field=AcademicField.objects.get(name="Quantum Physics"),
+ name="Quantum Information",
+ slug="quantum-information",
+ order=10,
+ )
+
+ def _create_academic_field(self):
+ AcademicField.objects.create(
+ branch=Branch.objects.get(name="Physics"),
+ name="Quantum Physics",
+ slug="quantum-physics",
+ order=10,
+ )
+
+ def _create_branch(self):
+ Branch.objects.create(
+ name="Physics",
+ slug="physics",
+ order=10,
+ )
+
+ def _create_preprint(self):
+ Preprint.objects.create(identifier_w_vn_nr="scipost_202101_00001v1")
+
+ def _create_submission(self):
+ submission = Submission.objects.create(
+ preprint=Preprint.objects.get(identifier_w_vn_nr="scipost_202101_00001v1"),
+ submitted_to=Journal.objects.get(name="SciPost Physics"),
+ title="Test submission",
+ abstract="Test abstract",
+ author_list="Test User",
+ acad_field=AcademicField.objects.get(name="Quantum Physics"),
+ # specialties=Specialty.objects.filter(name="Quantum Information"),
+ submitted_by=Contributor.objects.get(user__username="testuser"),
+ )
+ submission.authors.add(Contributor.objects.get(user__username="testuser"))
+ submission.save()
+
+ def _create_production_stream(self):
+ stream = ProductionStream.objects.create(
+ submission=Submission.objects.get(
+ preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+ ),
+ )
+ stream.opened = datetime.datetime(
+ 2021, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc
+ )
+ stream.save()
+
+ def setUp(self):
+ self._create_submitter_contributor()
+ self._create_branch()
+ self._create_academic_field()
+ self._create_specialty()
+ self._create_college()
+ self._create_journal()
+ self._create_preprint()
+ self._create_submission()
+ self._create_editorial_decision()
+ self._create_production_stream()
+
+ def test_repo_name_existing_profile(self):
+ proofs_repo = ProofsRepository.objects.get(
+ stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+ )
+
+ self.assertEqual(proofs_repo.name, "scipost_202101_00001v1_User")
+
+ def test_repo_name_nonexisting_profile(self):
+ proofs_repo = ProofsRepository.objects.get(
+ stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+ )
+
+ # delete profile
+ Contributor.objects.get(user__username="testuser").profile.delete()
+
+ self.assertEqual(proofs_repo.name, "scipost_202101_00001v1_User")
+
+ def test_repo_name_double_last_name_profile(self):
+ proofs_repo = ProofsRepository.objects.get(
+ stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+ )
+
+ proofs_repo.stream.submission.author_list = "Test Usable User"
+
+ user_profile = Contributor.objects.get(user__username="testuser").profile
+ user_profile.last_name = "Usable User"
+ user_profile.save()
+
+ self.assertEqual(proofs_repo.name, "scipost_202101_00001v1_User")
+
+ def test_repo_name_two_authors(self):
+ proofs_repo = ProofsRepository.objects.get(
+ stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+ )
+
+ proofs_repo.stream.submission.author_list = (
+ "Another Personable Person, Test Usable User"
+ )
+
+ self.assertEqual(proofs_repo.name, "scipost_202101_00001v1_Person")
+
+ def test_repo_paths_scipostphys(self):
+ proofs_repo = ProofsRepository.objects.get(
+ stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+ )
+
+ settings.GITLAB_ROOT = "ProjectRoot"
+
+ self.assertEqual(
+ proofs_repo.git_path,
+ "ProjectRoot/Proofs/SciPostPhys/2021/01/scipost_202101_00001v1_User",
+ )
+
+ self.assertEqual(
+ proofs_repo.template_path,
+ "ProjectRoot/Templates/SciPostPhys",
+ )
+
+ def test_repo_paths_scipostphysproc(self):
+ proofs_repo = ProofsRepository.objects.get(
+ stream__submission__preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+ )
+
+ journal = Journal.objects.get(name="SciPost Physics")
+ journal.name = "SciPost Physics Proceedings"
+ journal.doi_label = "SciPostPhysProc"
+ journal.structure = "IO" # proceedings, as Issues Only
+ journal.save()
+
+ issue = Issue.objects.create(
+ in_journal=journal,
+ number=1,
+ slug="proc-1",
+ doi_label="SciPostPhysProc.1",
+ )
+
+ proceedings = Proceedings.objects.create(
+ issue=issue,
+ submissions_open=datetime.datetime.now(),
+ submissions_close=datetime.datetime.now(),
+ submissions_deadline=datetime.datetime.now(),
+ event_end_date=datetime.datetime(2021, 5, 5),
+ event_start_date=datetime.datetime(2021, 5, 1),
+ event_suffix="ProcName21",
+ )
+
+ submission = Submission.objects.get(
+ preprint__identifier_w_vn_nr="scipost_202101_00001v1"
+ )
+
+ submission.proceedings = proceedings
+ submission.save()
+
+ settings.GITLAB_ROOT = "ProjectRoot"
+
+ self.assertEqual(
+ proofs_repo.git_path,
+ "ProjectRoot/Proofs/SciPostPhysProc/2021/ProcName21/scipost_202101_00001v1_User",
+ )
+
+ self.assertEqual(
+ proofs_repo.template_path,
+ "ProjectRoot/Templates/SciPostPhysProc/2021/ProcName21",
+ )
diff --git a/scipost_django/scipost/management/commands/vet_superusers.py b/scipost_django/scipost/management/commands/vet_superusers.py
new file mode 100644
index 0000000000000000000000000000000000000000..c790fecf8256872aa2da80bf43e834b6545128a2
--- /dev/null
+++ b/scipost_django/scipost/management/commands/vet_superusers.py
@@ -0,0 +1,18 @@
+__copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
+__license__ = "AGPL v3"
+
+
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User, Group
+
+class Command(BaseCommand):
+
+ def handle(self, *args, **options):
+ superusers = User.objects.filter(is_superuser=True)
+ admin_group = Group.objects.get(name='SciPost Administrators')
+
+ for superuser in superusers:
+ superuser.groups.add(admin_group)
+ superuser.save()
+
+ self.stdout.write(self.style.SUCCESS(f"Successfully vetted {len(superusers)} superusers."))
\ No newline at end of file
diff --git a/scipost_django/submissions/factories/assignment.py b/scipost_django/submissions/factories/assignment.py
index d0d0fca6a20d3acdd6162157457b3e33da182198..5bc513f62337cf6be1d1e3bc3981c42defd9890c 100644
--- a/scipost_django/submissions/factories/assignment.py
+++ b/scipost_django/submissions/factories/assignment.py
@@ -5,7 +5,7 @@ __license__ = "AGPL v3"
import factory
from scipost.models import Contributor
-from submissions.constants import ASSIGNMENT_STATUSES
+from submissions.models.submission import Submission
from submissions.models import EditorialAssignment
@@ -18,7 +18,7 @@ class EditorialAssignmentFactory(factory.django.DjangoModelFactory):
submission = None
to = factory.Iterator(Contributor.objects.all())
- status = factory.Iterator(ASSIGNMENT_STATUSES, getter=lambda c: c[0])
+ status = factory.Iterator(Submission.SUBMISSION_STATUSES, getter=lambda c: c[0])
date_created = factory.lazy_attribute(lambda o: o.submission.latest_activity)
date_answered = factory.lazy_attribute(lambda o: o.submission.latest_activity)
diff --git a/scipost_django/submissions/factories/referee_invitation.py b/scipost_django/submissions/factories/referee_invitation.py
index 565387f4069dfb3a1eb7d250c5e609231b47afe4..3f303de013b8eec7ff19d96d4d47f244ee6731f0 100644
--- a/scipost_django/submissions/factories/referee_invitation.py
+++ b/scipost_django/submissions/factories/referee_invitation.py
@@ -4,6 +4,7 @@ __license__ = "AGPL v3"
import factory
import pytz
+import random
from faker import Faker
@@ -46,6 +47,7 @@ class AcceptedRefereeInvitationFactory(RefereeInvitationFactory):
@factory.post_generation
def report(self, create, extracted, **kwargs):
if create:
+ from submissions.factories import VettedReportFactory
VettedReportFactory(submission=self.submission, author=self.referee)
diff --git a/scipost_django/submissions/factories/submission.py b/scipost_django/submissions/factories/submission.py
index 954fd91956f2bb754df526e26b8cb8187a60e470..a8af6df0b59784568ec151e9a2e7d3280bba169d 100644
--- a/scipost_django/submissions/factories/submission.py
+++ b/scipost_django/submissions/factories/submission.py
@@ -4,6 +4,7 @@ __license__ = "AGPL v3"
import factory
import pytz
+import random
from faker import Faker
@@ -308,6 +309,11 @@ class PublishedSubmissionFactory(InRefereeingSubmissionFactory):
@factory.post_generation
def referee_invites(self, create, extracted, **kwargs):
+ from submissions.factories import (
+ FulfilledRefereeInvitationFactory,
+ CancelledRefereeInvitationFactory
+ )
+
for i in range(random.randint(2, 4)):
FulfilledRefereeInvitationFactory(submission=self)
for i in range(random.randint(0, 2)):