Updated metadata cherry-pick and management commands for learning activity and duration defaults

contentcuration/contentcuration/admin.py

@@ -1,8 +1,9 @@
 from django.contrib import admin
 
-from contentcuration.models import Exercise, AssessmentItem, License, User
+from contentcuration.models import AssessmentItem
+from contentcuration.models import License
+from contentcuration.models import User
 
-admin.site.register(Exercise)
 admin.site.register(AssessmentItem)
 admin.site.register(License)
 

contentcuration/contentcuration/constants/contentnode.py

@@ -0,0 +1,13 @@
+from le_utils.constants import content_kinds
+from le_utils.constants.labels import learning_activities
+
+
+kind_activity_map = {
+    content_kinds.EXERCISE: learning_activities.PRACTICE,
+    content_kinds.VIDEO: learning_activities.WATCH,
+    content_kinds.AUDIO: learning_activities.LISTEN,
+    content_kinds.DOCUMENT: learning_activities.READ,
+    content_kinds.HTML5: learning_activities.EXPLORE,
+    content_kinds.H5P: learning_activities.EXPLORE,
+    content_kinds.SLIDESHOW: learning_activities.READ,
+}

contentcuration/contentcuration/dev_urls.py

@@ -8,7 +8,6 @@
 from drf_yasg.views import get_schema_view
 from rest_framework import permissions
 
-import contentcuration.views.files as file_views
 from .urls import urlpatterns
 
 
@@ -48,21 +47,6 @@ def webpack_redirect_view(request):
         r"^redoc/$", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"
     ),
     re_path(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")),
-    re_path(
-        r"^" + settings.STORAGE_URL[1:] + "(?P<path>.*)$",
-        file_views.debug_serve_file,
-        name="debug_serve_file",
-    ),
-    re_path(
-        r"^" + settings.CONTENT_DATABASE_URL[1:] + "(?P<path>.*)$",
-        file_views.debug_serve_content_database_file,
-        name="content_database_debug_serve_file",
-    ),
-    re_path(
-        r"^" + settings.CSV_URL[1:] + "(?P<path>.*)$",
-        file_views.debug_serve_file,
-        name="csv_debug_serve_file",
-    ),
 ]
 
 if getattr(settings, "DEBUG_PANEL_ACTIVE", False):

contentcuration/contentcuration/management/commands/loadconstants.py

@@ -1,9 +1,6 @@
-from future import standard_library
-
-standard_library.install_aliases()
-from builtins import str
-from builtins import object
 import logging as logmodule
+from builtins import object
+from builtins import str
 
 from django.conf import settings
 from django.contrib.sites.models import Site

contentcuration/contentcuration/management/commands/set_default_learning_activities.py

@@ -0,0 +1,37 @@
+import logging as logmodule
+import time
+
+from django.core.management.base import BaseCommand
+
+from contentcuration.constants.contentnode import kind_activity_map
+from contentcuration.models import ContentNode
+
+logmodule.basicConfig(level=logmodule.INFO)
+logging = logmodule.getLogger('command')
+
+
+CHUNKSIZE = 10000
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        start = time.time()
+
+        for kind, activity in kind_activity_map.items():
+            kind_start = time.time()
+            map_to_set = {
+                activity: True
+            }
+
+            null_learning_activities = ContentNode.objects.filter(kind=kind, learning_activities__isnull=True).values_list("id", flat=True)
+
+            logging.info("Setting default learning activities for kind: {}".format(kind))
+
+            while null_learning_activities.exists():
+                updated_count = ContentNode.objects.filter(id__in=null_learning_activities[0:CHUNKSIZE]).update(learning_activities=map_to_set)
+                logging.info("Updated {} content nodes of kind {} with learning activity {}".format(updated_count, kind, activity))
+
+            logging.info("Finished setting default learning activities for kind: {} in {} seconds".format(kind, time.time() - kind_start))
+
+        logging.info('Finished setting all null learning activities in {} seconds'.format(time.time() - start))

contentcuration/contentcuration/management/commands/set_file_duration.py

@@ -0,0 +1,72 @@
+import logging as logmodule
+import subprocess
+import time
+
+from django.core.management.base import BaseCommand
+
+from contentcuration.models import File
+from contentcuration.models import MEDIA_PRESETS
+
+logmodule.basicConfig(level=logmodule.INFO)
+logging = logmodule.getLogger('command')
+
+
+CHUNKSIZE = 10000
+
+
+def extract_duration_of_media(f_in):
+    result = subprocess.check_output(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            "-loglevel",
+            "panic",
+            "-"
+        ],
+        stdin=f_in,
+    )
+    return int(float(result.decode("utf-8").strip()))
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        start = time.time()
+
+        logging.info("Setting default duration for media presets: {}".format(MEDIA_PRESETS))
+
+        excluded_files = set()
+
+        null_duration = File.objects.filter(preset_id__in=MEDIA_PRESETS, duration__isnull=True)
+        null_duration_count = null_duration.count()
+        updated_count = 0
+
+        i = 0
+
+        while i < null_duration_count:
+            for file in null_duration[i:i + CHUNKSIZE]:
+                if file.file_on_disk.name in excluded_files:
+                    continue
+                file.refresh_from_db()
+                if file.duration is not None:
+                    continue
+                try:
+                    with file.file_on_disk.open() as f:
+                        duration = extract_duration_of_media(f)
+                    if duration:
+                        updated_count += File.objects.filter(checksum=file.checksum, preset_id__in=MEDIA_PRESETS).update(duration=duration)
+                except FileNotFoundError:
+                    logging.warning("File {} not found".format(file))
+                    excluded_files.add(file.file_on_disk.name)
+                except subprocess.CalledProcessError:
+                    logging.warning("File {} could not be read for duration".format(file))
+                    excluded_files.add(file.file_on_disk.name)
+
+            i += CHUNKSIZE
+
+        logging.info('Finished setting all null duration for {} files in {} seconds'.format(updated_count, time.time() - start))

contentcuration/contentcuration/migrations/0131_auto_20210707_2326.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.2.4 on 2021-07-07 23:26
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contentcuration', '0130_auto_20210706_2005'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='ChannelResourceSize',
+        ),
+    ]

contentcuration/contentcuration/migrations/0132_auto_20210708_0011.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.2.4 on 2021-07-08 00:11
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contentcuration', '0131_auto_20210707_2326'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='Exercise',
+        ),
+    ]

contentcuration/contentcuration/migrations/0133_auto_20220124_2149.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.5 on 2022-01-24 21:49
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contentcuration', '0132_auto_20210708_0011'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='file',
+            name='duration',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddConstraint(
+            model_name='file',
+            constraint=models.CheckConstraint(check=models.Q(models.Q(('duration__gt', 0), ('preset__in', ['audio', 'high_res_video', 'low_res_video'])), ('duration__isnull', True), _connector='OR'), name='file_media_duration_int'),
+        ),
+    ]

contentcuration/contentcuration/migrations/0134_alter_contentkind_kind.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.5 on 2022-02-23 18:42
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("contentcuration", "0133_auto_20220124_2149"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="contentkind",
+            name="kind",
+            field=models.CharField(
+                choices=[
+                    ("topic", "Topic"),
+                    ("video", "Video"),
+                    ("audio", "Audio"),
+                    ("exercise", "Exercise"),
+                    ("document", "Document"),
+                    ("html5", "HTML5 App"),
+                    ("slideshow", "Slideshow"),
+                    ("h5p", "H5P"),
+                    ("zim", "Zim"),
+                    ("quiz", "Quiz"),
+                ],
+                max_length=200,
+                primary_key=True,
+                serialize=False,
+            ),
+        ),
+    ]

contentcuration/contentcuration/migrations/0135_add_metadata_labels.py

@@ -0,0 +1,43 @@
+# Generated by Django 3.2.5 on 2022-02-25 00:23
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contentcuration', '0134_alter_contentkind_kind'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='contentnode',
+            name='accessibility_labels',
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='contentnode',
+            name='categories',
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='contentnode',
+            name='grade_levels',
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='contentnode',
+            name='learner_needs',
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='contentnode',
+            name='learning_activities',
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='contentnode',
+            name='resource_types',
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]