diff --git a/requirements.txt b/requirements.txt index 71131e0..7fcb213 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,22 +3,29 @@ asgiref==3.5.0 async-timeout==4.0.2 billiard==3.6.4.0 celery==5.2.6 +certifi==2021.10.8 +charset-normalizer==2.0.12 click==8.1.2 click-didyoumean==0.3.0 click-plugins==1.1.1 click-repl==0.2.0 Deprecated==1.2.13 Django==4.0.4 +icalendar==4.0.9 +idna==3.3 kombu==5.2.4 librabbitmq==2.0.0 packaging==21.3 prompt-toolkit==3.0.29 psycopg2-binary==2.9.3 pyparsing==3.0.8 +python-dateutil==2.8.2 pytz==2022.1 redis==4.2.2 +requests==2.27.1 six==1.16.0 sqlparse==0.4.2 +urllib3==1.26.9 vine==5.0.0 wcwidth==0.2.5 wrapt==1.14.0 diff --git a/shiftregister/app/admin.py b/shiftregister/app/admin.py index 8e59132..caab3a5 100644 --- a/shiftregister/app/admin.py +++ b/shiftregister/app/admin.py @@ -7,7 +7,7 @@ admin.site.register(Room) @admin.register(Shift) class ShiftAdmin(admin.ModelAdmin): - list_display = ("room_name", "start_at", "free_slots") + list_display = ("room_name", "start_at", "free_slots", "deleted") def room_name(self, object): return object.room.name diff --git a/shiftregister/app/migrations/0006_shift_deleted.py b/shiftregister/app/migrations/0006_shift_deleted.py new file mode 100644 index 0000000..767fa54 --- /dev/null +++ b/shiftregister/app/migrations/0006_shift_deleted.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0.4 on 2022-04-23 00:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("app", "0005_alter_helper_phone"), + ] + + operations = [ + migrations.AddField( + model_name="shift", + name="deleted", + field=models.BooleanField(default=False), + ), + ] diff --git a/shiftregister/app/models.py b/shiftregister/app/models.py index b268d71..e581d98 100644 --- a/shiftregister/app/models.py +++ b/shiftregister/app/models.py @@ -18,7 +18,9 @@ class Shift(models.Model): room = models.ForeignKey(Room, on_delete=models.RESTRICT) start_at = models.DateTimeField() duration = models.DurationField() + deleted = models.BooleanField(default=False) # todo: add helper amount override field + def __str__(self): return f"{self.room.name}: {self.start_at}" diff --git a/shiftregister/app/views.py b/shiftregister/app/views.py index 8382e17..c577d48 100644 --- a/shiftregister/app/views.py +++ b/shiftregister/app/views.py @@ -31,7 +31,11 @@ def index(request): free_shifts = ( Shift.objects.annotate(reg_count=Count("shiftregistration")) - .filter(start_at__gt=timezone.now(), room__required_helpers__gt=F("reg_count")) + .filter( + start_at__gt=timezone.now(), + room__required_helpers__gt=F("reg_count"), + deleted=False, + ) .order_by("start_at") ) @@ -39,7 +43,9 @@ def index(request): free_shifts = ( Shift.objects.annotate(reg_count=Count("shiftregistration")) .filter( - start_at__gt=timezone.now(), room__required_helpers__gt=F("reg_count") + start_at__gt=timezone.now(), + room__required_helpers__gt=F("reg_count"), + deleted=False, ) .filter(~Q(shiftregistration__helper=request.helper)) .order_by("start_at") diff --git a/shiftregister/importer/__init__.py b/shiftregister/importer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shiftregister/importer/admin.py b/shiftregister/importer/admin.py new file mode 100644 index 0000000..4442957 --- /dev/null +++ b/shiftregister/importer/admin.py @@ -0,0 +1,7 @@ +from django.contrib import admin +from .models import Calendar + + +@admin.register(Calendar) +class CalendarAdmin(admin.ModelAdmin): + list_display = ("url", "has_errors") diff --git a/shiftregister/importer/apps.py b/shiftregister/importer/apps.py new file mode 100644 index 0000000..259cbc3 --- /dev/null +++ b/shiftregister/importer/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ImporterConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "shiftregister.importer" diff --git a/shiftregister/importer/importer.py b/shiftregister/importer/importer.py new file mode 100644 index 0000000..cc13b49 --- /dev/null +++ b/shiftregister/importer/importer.py @@ -0,0 +1,83 @@ +from datetime import timezone +from django.conf import settings +from django.db import transaction +from icalendar import Calendar +from .models import Event, Room, Shift +import requests + + +def import_calendar(calendar): + try: + r = requests.get(calendar.url) + r.raise_for_status() + except: + if settings.DEBUG: + raise + return False + + if not r.headers["content-type"].startswith("text/calendar"): + return False + + try: + cal = Calendar.from_ical(r.text) + + rooms = {} + events = {} + for event in cal.walk("vevent"): + uid = event.decoded("uid").decode() + room = ( + event.decoded("location", None) or event.decoded("summary") + ).decode() + start = event.decoded("dtstart").astimezone(timezone.utc) + end = event.decoded("dtend").astimezone(timezone.utc) + + if not uid or not room: + return False + + rooms[room] = None + events[uid] = ( + room, + { + "start_at": start, + "duration": end - start, + "uuid": uid, + "calendar": calendar, + }, + ) + + with transaction.atomic(): + for r in Room.objects.filter(name__in=rooms): + rooms[r.name] = r + for room, r in rooms.items(): + if r == None: + rooms[room] = Room( + name=room, required_helpers=0 + ) # required_helpers=0 ensures a shift in a new room is not displayed until the correct number of required helpers is set + rooms[room].save() + + for e in Event.objects.filter(calendar=calendar, uuid__in=events): + uuid = str(e.uuid) + room, event = events[uuid] + + e.room = rooms[room] + e.start_at = event["start_at"] + e.duration = event["duration"] + e.save() + + events[uuid] = (room, e) + for event in events: + room, e = events[event] + if not isinstance(e, Event): + Event(room=rooms[room], **e).save() + + for event in Event.objects.filter(calendar=calendar).exclude( + uuid__in=events + ): + event.deleted = True + event.save() + except: + if settings.DEBUG: + raise + return False + + return True diff --git a/shiftregister/importer/migrations/0001_initial.py b/shiftregister/importer/migrations/0001_initial.py new file mode 100644 index 0000000..d78604b --- /dev/null +++ b/shiftregister/importer/migrations/0001_initial.py @@ -0,0 +1,49 @@ +# Generated by Django 4.0.4 on 2022-04-23 00:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("app", "0006_shift_deleted"), + ] + + operations = [ + migrations.CreateModel( + name="Calendar", + fields=[ + ("url", models.URLField(primary_key=True, serialize=False)), + ("has_errors", models.BooleanField(default=False, editable=False)), + ], + ), + migrations.CreateModel( + name="Event", + fields=[ + ( + "shift_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + to="app.shift", + ), + ), + ( + "uuid", + models.UUIDField(editable=False, primary_key=True, serialize=False), + ), + ( + "calendar", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="importer.calendar", + ), + ), + ], + bases=("app.shift",), + ), + ] diff --git a/shiftregister/importer/migrations/__init__.py b/shiftregister/importer/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shiftregister/importer/models.py b/shiftregister/importer/models.py new file mode 100644 index 0000000..46f0294 --- /dev/null +++ b/shiftregister/importer/models.py @@ -0,0 +1,12 @@ +from django.db import models +from shiftregister.app.models import * + + +class Calendar(models.Model): + url = models.URLField(primary_key=True) + has_errors = models.BooleanField(default=False, editable=False) + + +class Event(Shift): + uuid = models.UUIDField(primary_key=True, editable=False) + calendar = models.ForeignKey(Calendar, on_delete=models.CASCADE) diff --git a/shiftregister/importer/tasks.py b/shiftregister/importer/tasks.py new file mode 100644 index 0000000..d8eaa93 --- /dev/null +++ b/shiftregister/importer/tasks.py @@ -0,0 +1,10 @@ +from celery import shared_task +from .importer import import_calendar +from .models import Calendar + + +@shared_task +def import_shifts(): + for calendar in Calendar.objects.all(): + calendar.has_errors = not import_calendar(calendar) + calendar.save() diff --git a/shiftregister/importer/tests.py b/shiftregister/importer/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/shiftregister/importer/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/shiftregister/importer/views.py b/shiftregister/importer/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/shiftregister/importer/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/shiftregister/settings.py b/shiftregister/settings.py index acc3930..d6f29ec 100644 --- a/shiftregister/settings.py +++ b/shiftregister/settings.py @@ -35,6 +35,7 @@ ALLOWED_HOSTS = list(filter(lambda s: s != "", getenv("ALLOWED_HOSTS", "").split INSTALLED_APPS = [ "shiftregister.app.apps.AppConfig", + "shiftregister.importer.apps.ImporterConfig", "shiftregister.team.apps.TeamConfig", "django.contrib.admin", "django.contrib.auth", @@ -141,3 +142,10 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" CELERY_BROKER_URL = getenv("CELERY_BROKER_URL", "amqp://guest:guest@localhost:5672//") CELERY_RESULT_BACKEND = getenv("CELERY_RESULT_BACKEND", "redis://") + +CELERY_BEAT_SCHEDULE = { + "import-shifts-every-60-seconds": { + "task": "shiftregister.importer.tasks.import_shifts", + "schedule": float(getenv("SHIFT_IMPORT_INTERVAL", 60.0)), # seconds + }, +}