From 6669cb6edf8df7275c09ce73d831f0332f72e77a Mon Sep 17 00:00:00 2001 From: PrinxeShamar Date: Tue, 17 Oct 2023 18:52:35 -0400 Subject: [PATCH 1/3] Daily Commit Aggregation, Commit Page --- celerybeat-schedule | Bin 0 -> 16384 bytes portal/services/github.py | 49 +++++++++++++++++ portal/tasks.py | 22 ++++++-- portal/templates/portal/commits/index.html | 53 +++++++++++++++++++ portal/templates/portal/includes/navbar.html | 1 + portal/urls.py | 7 +++ portal/views/commits.py | 25 +++++++++ rcos_io/celery.py | 8 +++ 8 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 celerybeat-schedule create mode 100644 portal/templates/portal/commits/index.html create mode 100644 portal/views/commits.py diff --git a/celerybeat-schedule b/celerybeat-schedule new file mode 100644 index 0000000000000000000000000000000000000000..52335976ea59f1188e98c352a002eb18c6061cad GIT binary patch literal 16384 zcmeI(O>5LZ7zgldiR|_bTH8bEih8nYajMo9kBWGZiU$=$1;Zpeb;D+pl1!9tSy&N7 zA-7I1diN}T0k3-U195_@J?e-D7ou4xGXKjPm^IseO zJnD0OrfI8BqTZ;$0S>*-2gbwRjE8q`-p_d6JuzbQ-7uYYk{0;yjX#{>Yy{u{hh70w zuHJchFqeUMxhFs4eR9(Y`O{dyp+5)yNSe)1X0#QENTjRG7hI}h z^jc6CwLB6RsZ*lYz zBN@hYnliQKQl@msqS8uY5ys026>m}Tk}gvHpqIxhVbF+#dZezZEv|2=q<83Li7wyv zJhNTaX9?0ImfY7JFG|N%y=rDjE!xUI)~2;CXFW2L{mT3RxpSoxbQ2%!uF}9SDIKVH y=k1iLe@JO+B}pZZ*p7$P9AT~PE|&XySo)d71OW&@00Izz00bZa0SG`~JAp44^-OO7 literal 0 HcmV?d00001 diff --git a/portal/services/github.py b/portal/services/github.py index 3737441..91421be 100644 --- a/portal/services/github.py +++ b/portal/services/github.py @@ -138,3 +138,52 @@ def get_repository_details(client: Client, repo_url: str): result = client.execute(query, variable_values={"owner": owner, "name": name}) return result + +def get_repository_commits(client: Client, repo_url: str, since: str): + owner, name = repo_url.split("/")[-2:] + query = gql( + """ + query RepoCommits($owner: String!, $name: String!, $since: GitTimestamp!) { + repository(owner: $owner, name: $name) { + refs(refPrefix: "refs/heads/", orderBy: {direction: DESC, field: TAG_COMMIT_DATE}, first: 100) { + edges { + node { + ... on Ref { + name + target { + ... on Commit { + history(since: $since) { + edges { + node { + ... on Commit { + commitUrl + additions + deletions + author { + user { + login + } + } + } + } + } + } + } + } + } + } + } + } + } + rateLimit { + limit + cost + remaining + resetAt + } + } + """ + ) + + result = client.execute(query, variable_values={"owner": owner, "name": name, "since": since}) + return result \ No newline at end of file diff --git a/portal/tasks.py b/portal/tasks.py index c48a351..917d8e3 100644 --- a/portal/tasks.py +++ b/portal/tasks.py @@ -5,10 +5,10 @@ from django.db.models import Manager from django.utils import timezone from requests import HTTPError +from django.core.cache import cache -from portal.models import Meeting -from portal.services import discord - +from portal.models import Meeting, Semester +from portal.services import discord, github @shared_task def delete_discord_channels(channel_ids: list[str]): @@ -32,3 +32,19 @@ def meetings_alert(): discord.send_message(os.environ["DISCORD_ALERTS_CHANNEL_ID"], { "content": f"Meeting **{meeting}** does not have presentation slides added yet!" }) + +@shared_task +def get_commits(): + semester = Semester.objects.latest("start_date") + for project in semester.projects.filter(is_approved=True).all(): + for repo in project.repositories.all(): + commits = {} + result = github.get_repository_commits(github.client_factory(), repo.url, semester.start_date.strftime('%Y-%m-%dT%H:%M:%S.%f%z')) + for branch in result["repository"]["refs"]["edges"]: + for commit in branch["node"]["target"]["history"]["edges"]: + commits[commit["node"]["commitUrl"]] = { + "author": commit["node"]["author"]["user"]["login"], + "additions": commit["node"]["additions"], + "deletions": commit["node"]["deletions"] + } + cache.set(f"repo_commits_{repo.url}", commits, 60 * 60 * 24) \ No newline at end of file diff --git a/portal/templates/portal/commits/index.html b/portal/templates/portal/commits/index.html new file mode 100644 index 0000000..6c4d09f --- /dev/null +++ b/portal/templates/portal/commits/index.html @@ -0,0 +1,53 @@ +{% extends "portal/base.html" %} +{% load portal_extras %} + +{% block ogp %} + + + + + +{% endblock %} + +{% block title %} +Commits | RCOS IO +{% endblock %} + +{% block content %} +
+
+

Commits

+ {% for url, commit in commits.items %} + {% if commit.author == username %} +
+ {{ url }} + +{{ commit.additions }} + -{{ commit.deletions }} +
+ {% endif %} + {% endfor %} +
+ +
+
+ +{{ commits|json_script:"commits" }} + + +{% endblock %} \ No newline at end of file diff --git a/portal/templates/portal/includes/navbar.html b/portal/templates/portal/includes/navbar.html index 73f0572..c52d3fa 100644 --- a/portal/templates/portal/includes/navbar.html +++ b/portal/templates/portal/includes/navbar.html @@ -21,6 +21,7 @@ {% if request.user.is_authenticated and request.user.is_approved %} {% include "portal/includes/navbaritem.html" with view_name="small_groups_index" display="Small Groups" %} + {% include "portal/includes/navbaritem.html" with view_name="commits_index" display="Commits" %} {% endif %} {% include "portal/includes/navbaritem.html" with view_name="organizations_index" display="Organizations" %} {% include "portal/includes/navbaritem.html" with view_name="handbook" display="Handbook" %} diff --git a/portal/urls.py b/portal/urls.py index 9718151..af1c7f4 100644 --- a/portal/urls.py +++ b/portal/urls.py @@ -9,6 +9,7 @@ from portal.views.mentors import MentorApplicationView, mentor_applications_index from portal.views.organizations import organizations_index from portal.views.small_groups import SmallGroupIndexView, small_group_detail +from portal.views.commits import commits_index from .views.auth import ( discord_flow_callback, @@ -144,4 +145,10 @@ import_google_form_projects, name="import_projects", ), + # Commits + path( + "commits/", + commits_index, + name="commits_index" + ) ] diff --git a/portal/views/commits.py b/portal/views/commits.py new file mode 100644 index 0000000..09fa8b7 --- /dev/null +++ b/portal/views/commits.py @@ -0,0 +1,25 @@ + + +from django.http import HttpRequest, HttpResponse +from django.template.response import TemplateResponse +from django.shortcuts import get_object_or_404 +from django.contrib.auth.decorators import login_required +from portal.services import github +from django.core.cache import cache; +from portal.tasks import get_commits; + +@login_required +def commits_index(request: HttpRequest) -> HttpResponse: + repos = request.user.get_active_enrollment().project.repositories.all() + + commits = {} + for repo in repos: + repo_commits = cache.get(f"repo_commits_{repo.url}") + if repo_commits: + for url in repo_commits: + commits[url] = repo_commits[url] + + return TemplateResponse(request, "portal/commits/index.html", { + "commits": commits, + "username": request.user.github_username + }) \ No newline at end of file diff --git a/rcos_io/celery.py b/rcos_io/celery.py index b1609ac..b22f23c 100644 --- a/rcos_io/celery.py +++ b/rcos_io/celery.py @@ -1,6 +1,7 @@ import os from celery import Celery +from celery.schedules import crontab # Set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rcos_io.settings") @@ -15,3 +16,10 @@ # Load task modules from all registered Django apps. app.autodiscover_tasks() + +app.conf.beat_schedule = { + 'get_commits': { + 'task': 'portal.tasks.get_commits', + 'schedule': crontab(minute=0, hour=0) + }, +} \ No newline at end of file From 2655a20d0293d020f49858f0476c2588d2ea4b52 Mon Sep 17 00:00:00 2001 From: PrinxeShamar Date: Tue, 17 Oct 2023 19:00:14 -0400 Subject: [PATCH 2/3] Cleanup --- celerybeat-schedule | Bin 16384 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 celerybeat-schedule diff --git a/celerybeat-schedule b/celerybeat-schedule deleted file mode 100644 index 52335976ea59f1188e98c352a002eb18c6061cad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI(O>5LZ7zgldiR|_bTH8bEih8nYajMo9kBWGZiU$=$1;Zpeb;D+pl1!9tSy&N7 zA-7I1diN}T0k3-U195_@J?e-D7ou4xGXKjPm^IseO zJnD0OrfI8BqTZ;$0S>*-2gbwRjE8q`-p_d6JuzbQ-7uYYk{0;yjX#{>Yy{u{hh70w zuHJchFqeUMxhFs4eR9(Y`O{dyp+5)yNSe)1X0#QENTjRG7hI}h z^jc6CwLB6RsZ*lYz zBN@hYnliQKQl@msqS8uY5ys026>m}Tk}gvHpqIxhVbF+#dZezZEv|2=q<83Li7wyv zJhNTaX9?0ImfY7JFG|N%y=rDjE!xUI)~2;CXFW2L{mT3RxpSoxbQ2%!uF}9SDIKVH y=k1iLe@JO+B}pZZ*p7$P9AT~PE|&XySo)d71OW&@00Izz00bZa0SG`~JAp44^-OO7 From f673583a9dedd8cfd6222a6fbb9c40ade2f25bdf Mon Sep 17 00:00:00 2001 From: PrinxeShamar Date: Fri, 20 Oct 2023 17:31:01 -0400 Subject: [PATCH 3/3] Get commits on server startup. Use committer over author. --- portal/services/github.py | 3 ++- portal/tasks.py | 11 ++++++----- portal/templates/portal/commits/index.html | 13 +++++++++---- portal/views/commits.py | 16 +++++++++------- rcos_io/celery.py | 8 +++++++- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/portal/services/github.py b/portal/services/github.py index 91421be..6e76f1c 100644 --- a/portal/services/github.py +++ b/portal/services/github.py @@ -157,9 +157,10 @@ def get_repository_commits(client: Client, repo_url: str, since: str): node { ... on Commit { commitUrl + committedDate additions deletions - author { + committer { user { login } diff --git a/portal/tasks.py b/portal/tasks.py index 917d8e3..69ecf28 100644 --- a/portal/tasks.py +++ b/portal/tasks.py @@ -42,9 +42,10 @@ def get_commits(): result = github.get_repository_commits(github.client_factory(), repo.url, semester.start_date.strftime('%Y-%m-%dT%H:%M:%S.%f%z')) for branch in result["repository"]["refs"]["edges"]: for commit in branch["node"]["target"]["history"]["edges"]: - commits[commit["node"]["commitUrl"]] = { - "author": commit["node"]["author"]["user"]["login"], - "additions": commit["node"]["additions"], - "deletions": commit["node"]["deletions"] - } + if (commit["node"]["committer"]["user"] != None): + commits[commit["node"]["commitUrl"]] = { + "committer": commit["node"]["committer"]["user"]["login"], + "additions": commit["node"]["additions"], + "deletions": commit["node"]["deletions"] + } cache.set(f"repo_commits_{repo.url}", commits, 60 * 60 * 24) \ No newline at end of file diff --git a/portal/templates/portal/commits/index.html b/portal/templates/portal/commits/index.html index 6c4d09f..c24635e 100644 --- a/portal/templates/portal/commits/index.html +++ b/portal/templates/portal/commits/index.html @@ -17,8 +17,9 @@

Commits

+ {% for url, commit in commits.items %} - {% if commit.author == username %} + {% if commit.committer == username %}
{{ url }} +{{ commit.additions }} @@ -26,8 +27,12 @@

Commits

{% endif %} {% endfor %} -
- + {% if commits.items|length > 0 %} +
+ + {% else %} +
No Commits Available.
+ {% endif %}
@@ -41,7 +46,7 @@

Commits

function CopyCommits() { let commits_str = "" for (let url in commits) { - if (commits[url].author == username) { + if (commits[url].committer == username) { commits_str += url + "\n" } } diff --git a/portal/views/commits.py b/portal/views/commits.py index 09fa8b7..da4cda2 100644 --- a/portal/views/commits.py +++ b/portal/views/commits.py @@ -10,14 +10,16 @@ @login_required def commits_index(request: HttpRequest) -> HttpResponse: - repos = request.user.get_active_enrollment().project.repositories.all() - + active_enrollment = request.user.get_active_enrollment() commits = {} - for repo in repos: - repo_commits = cache.get(f"repo_commits_{repo.url}") - if repo_commits: - for url in repo_commits: - commits[url] = repo_commits[url] + + if active_enrollment is not None: + repos = active_enrollment.project.repositories.all() + for repo in repos: + repo_commits = cache.get(f"repo_commits_{repo.url}") + if repo_commits: + for url in repo_commits: + commits[url] = repo_commits[url] return TemplateResponse(request, "portal/commits/index.html", { "commits": commits, diff --git a/rcos_io/celery.py b/rcos_io/celery.py index b22f23c..fae947e 100644 --- a/rcos_io/celery.py +++ b/rcos_io/celery.py @@ -2,6 +2,7 @@ from celery import Celery from celery.schedules import crontab +from celery.signals import worker_ready # Set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rcos_io.settings") @@ -22,4 +23,9 @@ 'task': 'portal.tasks.get_commits', 'schedule': crontab(minute=0, hour=0) }, -} \ No newline at end of file +} + +@worker_ready.connect +def startup(sender, **kwargs): + with sender.app.connection() as conn: + sender.app.send_task("portal.tasks.get_commits", connection=conn) \ No newline at end of file