Release 1.1.0

chatbot/admin/media_admin.py

@@ -1,4 +1,4 @@
-from django.contrib import admin
+from django.contrib import admin, messages
 from django import forms
 from django.shortcuts import render
 from django.http import HttpResponseRedirect
@@ -8,12 +8,13 @@
 from chatbot.form.media.media_form import MediaAdminForm
 from chatbot.models import Tag, Profile, TagChoices, TagSourceChoices
 from chatbot.models.media_models import Media, KeyValue, MediaImage
+from chatbot.models.theme_models import MediaSecondaryTheme
 from chatbot.models.enums import FileDisplayMode, ProfileType
 from simple_history.admin import SimpleHistoryAdmin
 from chatbot.utils.knowledge_service.cache_manager import GetCachedItemView
 from chatbot.views.Media.extract_views import BatchMediaExtractView, BatchMediaRetryExtractView
 from chatbot.views.Media.save_views import BatchMediaSaveView, BatchMediaRetrySaveView
-from chatbot.views.Media.status_views import BatchMediaTaskStatusView, VectorDBTaskStatusView
+from chatbot.views.Media.status_views import BatchMediaTaskStatusView, VectorDBTaskStatusView, ThemeMappingStatusView
 from chatbot.views.Media.upload_views import BatchMediaUploadView
 from chatbot.views.Media.drive_upload import DriveUploadView
 from chatbot.views.Media.google_drive_integration import (
@@ -23,6 +24,19 @@
     GoogleDriveFileImportView
 )
 from chatbot.utils.company_utils import get_company_queryset_for_user, get_user_company, get_user_profile
+from chatbot.constants.constants import ADMIN_THEME_WAIT_TIMEOUT_SECONDS
+from chatbot.utils.themes.theme_classification_service import (
+    ThemeMessageLevel, enqueue_theme_classification, theme_outcome_message, wait_for_theme_classification
+)
+
+THEME_CLASSIFICATION_INPUT_FIELDS = frozenset({'name', 'file', 'description', 'extracted_text', 'manual_tags'})
+THEME_SUMMARY_FIELDS = frozenset({'description', 'extracted_text'})
+THEME_MESSAGE_LEVELS = {
+    ThemeMessageLevel.SUCCESS: messages.SUCCESS,
+    ThemeMessageLevel.WARNING: messages.WARNING,
+    ThemeMessageLevel.ERROR: messages.ERROR,
+    ThemeMessageLevel.INFO: messages.INFO,
+}
 
 
 class MediaOrganizationFilter(admin.SimpleListFilter):
@@ -55,6 +69,24 @@ class KeyValueInline(admin.TabularInline):
     extra = 1
 
 
+class MediaSecondaryThemeInline(admin.TabularInline):
+    model = MediaSecondaryTheme
+    extra = 0
+    fields = ('theme', 'theme_status', 'match_type', 'confidence', 'reasoning', 'created_at')
+    readonly_fields = fields
+    can_delete = False
+
+    def get_queryset(self, request):
+        return super().get_queryset(request).select_related('theme')
+
+    @admin.display(description='Status')
+    def theme_status(self, obj):
+        return obj.theme.get_status_display()
+
+    def has_add_permission(self, request, obj=None):
+        return False
+
+
 class MediaImagesInline(admin.TabularInline):
     model = MediaImage
     extra = 1
@@ -67,8 +99,9 @@ class MediaImagesInline(admin.TabularInline):
 class MediaAdmin(SimpleHistoryAdmin, admin.ModelAdmin):
     form = MediaAdminForm
     list_display = (
-        'file_name', 'get_title', 'get_organization', 'media_type', 'source_provider',
-        'display_mode', 'parent__name', 'view_count', 'download_count', 'updated_at', 'created_at'
+        'file_name', 'get_title', 'primary_theme', 'get_secondary_themes', 'get_organization',
+        'media_type', 'source_provider', 'display_mode', 'parent__name', 'view_count', 'download_count',
+        'updated_at', 'created_at'
     )
     list_filter = (
         CustomAdvanceDateFilter,
@@ -77,12 +110,14 @@ class MediaAdmin(SimpleHistoryAdmin, admin.ModelAdmin):
         MediaOrganizationFilter,
         'name',
         'media_type',
-        'company_bot'
+        'company_bot',
+        'primary_theme',
+        'needs_review'
     )
     search_fields = ('name', 'key_values__value')
     actions = ['export_selected', 'change_display_mode_action']
     list_export = ('csv', 'xlsx')
-    inlines = [KeyValueInline, MediaImagesInline]
+    inlines = [KeyValueInline, MediaImagesInline, MediaSecondaryThemeInline]
     raw_id_fields = ('company_bot', 'parent', 'organization')
     date_hierarchy = 'created_at'
     readonly_fields = ('view_count', 'download_count', 'thumbnail')
@@ -110,6 +145,10 @@ def get_title(self, obj):
     get_title.short_description = 'Title'
     get_title.admin_order_field = 'key_values__value'
 
+    @admin.display(description='Secondary themes')
+    def get_secondary_themes(self, obj):
+        return ', '.join(link.theme.name for link in obj.secondary_theme_links.all()) or '-'
+
     def get_organization(self, obj):
         return obj.organization.name if obj.organization else "-"
 
@@ -119,8 +158,8 @@ def get_organization(self, obj):
     def get_queryset(self, request):
         """Optimize queries by prefetching related objects"""
         qs = super().get_queryset(request).select_related(
-            'organization', 'parent'
-        ).prefetch_related('key_values', 'tags')
+            'organization', 'parent', 'primary_theme'
+        ).prefetch_related('key_values', 'tags', 'secondary_theme_links__theme')
 
         if request.user.is_superuser:
             return qs
@@ -137,6 +176,10 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs):
         return super().formfield_for_foreignkey(db_field, request, **kwargs)
 
     def save_model(self, request, obj, form, change):
+        if change and 'primary_theme' in form.changed_data:
+            obj.primary_theme_name = obj.primary_theme.name if obj.primary_theme else None
+            obj.primary_theme_reasoning = None
+            obj.needs_review = obj.primary_theme_id is None
         super().save_model(request, obj, form, change)
 
         manual_tags = getattr(obj, '_manual_tags_to_set', [])
@@ -147,6 +190,45 @@ def save_model(self, request, obj, form, change):
 
         obj.tags.set(manual_tags + auto_tags)
 
+    def save_related(self, request, form, formsets, change):
+        super().save_related(request, form, formsets, change)
+        if self.needs_theme_classification(form, formsets, change):
+            changed = set(form.changed_data)
+            use_file_text = change and 'file' in changed and not THEME_SUMMARY_FIELDS & changed
+            request.theme_classification = (
+                enqueue_theme_classification(form.instance.pk, use_file_text=use_file_text), form.instance.name
+            )
+
+    @staticmethod
+    def needs_theme_classification(form, formsets, change):
+        if form.instance.parent_id:
+            return False
+        if 'primary_theme' in form.changed_data:
+            return False
+        if not change:
+            return True
+        if THEME_CLASSIFICATION_INPUT_FIELDS.intersection(form.changed_data):
+            return True
+        return any(formset.model is KeyValue and formset.has_changed() for formset in formsets)
+
+    def add_view(self, request, form_url='', extra_context=None):
+        response = super().add_view(request, form_url, extra_context)
+        return self.wait_for_theme_mapping(request, response)
+
+    def change_view(self, request, object_id, form_url='', extra_context=None):
+        response = super().change_view(request, object_id, form_url, extra_context)
+        return self.wait_for_theme_mapping(request, response)
+
+    def wait_for_theme_mapping(self, request, response):
+        pending = getattr(request, 'theme_classification', None)
+        if not pending or response.status_code != 302:
+            return response
+        task_id, media_name = pending
+        outcome = wait_for_theme_classification(task_id, ADMIN_THEME_WAIT_TIMEOUT_SECONDS)
+        level, message = theme_outcome_message(media_name, outcome)
+        self.message_user(request, message, THEME_MESSAGE_LEVELS[level])
+        return response
+
     def get_fieldsets(self, request, obj=None):
         # Check if user is a MODERATOR
         is_moderator = False
@@ -173,6 +255,12 @@ def get_fieldsets(self, request, obj=None):
             }),
         ]
 
+        if obj and obj.pk and not obj.parent_id:
+            fieldsets.insert(1, ('Theme', {
+                'fields': ('primary_theme',),
+                'description': 'Mapped automatically on upload; pick another primary theme to override it.'
+            }))
+
         if obj and obj.pk:
             has_auto_tags = obj.tags.filter(
                 source_type__in=[
@@ -303,6 +391,9 @@ def get_urls(self):
             path('api/vector-db-task-status/',
                  self.admin_site.admin_view(VectorDBTaskStatusView.as_view()),
                  name='chatbot_media_vector_db_task_status'),
+            path('api/theme-mapping-status/',
+                 self.admin_site.admin_view(ThemeMappingStatusView.as_view()),
+                 name='chatbot_media_theme_mapping_status'),
             path('api/get-cached-item/',
                  self.admin_site.admin_view(GetCachedItemView.as_view()),
                  name='chatbot_media_get_cached_item'),

chatbot/admin/theme_admin.py

@@ -1,9 +1,33 @@
+import csv
+from itertools import groupby
+
 from django.contrib import admin
+from django.core.exceptions import PermissionDenied
+from django.db.models import Count, Prefetch
+from django.http import HttpResponse
+from django.urls import path
+from django.utils import timezone
 from simple_history.admin import SimpleHistoryAdmin
+from chatbot.constants.constants import TITLE_KEY
 from chatbot.filter.custom_date_from_filter import CustomAdvanceDateFilter
-from chatbot.models import Theme, ThemeType
+from chatbot.models import Theme, ThemeType, ResourceTheme, ThemeStatus, Profile, KeyValue, MediaSecondaryTheme
+from chatbot.utils.company_utils import get_user_company
 # from rangefilter.filters import DateTimeRangeFilter
 
+SECONDARY_MAPPING_EXPORT_FIELDS = (
+    'Resource', 'Assigned Primary Theme', 'AI-Recommended Secondary Themes', 'AI Confidence Scores'
+)
+CSV_FORMULA_PREFIXES = ('=', '+', '-', '@', '\t', '\r')
+def csv_safe(value):
+    return f"'{value}" if isinstance(value, str) and value.startswith(CSV_FORMULA_PREFIXES) else value
+
+
+def csv_response(filename):
+    response = HttpResponse(content_type='text/csv; charset=utf-8')
+    response['Content-Disposition'] = f'attachment; filename="{filename}"'
+    response.write('')
+    return response
+
 
 @admin.register(Theme)
 class ThemeAdmin(SimpleHistoryAdmin):
@@ -37,3 +61,105 @@ def get_form(self, request, obj=None, **kwargs):
     def changeform_view(self, request, object_id=None, form_url='', extra_context=None):
         # Optionally, adjust behavior dynamically if needed
         return super().changeform_view(request, object_id, form_url, extra_context)
+
+
+class ResourceThemeTypeFilter(admin.SimpleListFilter):
+    title = 'Theme type'
+    parameter_name = 'theme_type'
+    PRIMARY = 'primary'
+    SECONDARY = 'secondary'
+    ALL = 'all'
+
+    def lookups(self, request, model_admin):
+        return ((self.ALL, 'All'), (self.PRIMARY, 'Primary'), (self.SECONDARY, 'Secondary'))
+
+    def queryset(self, request, queryset):
+        if self.value() in (self.PRIMARY, self.SECONDARY):
+            return queryset.filter(is_primary=self.value() == self.PRIMARY)
+        return queryset
+
+
+@admin.register(ResourceTheme)
+class ResourceThemeAdmin(admin.ModelAdmin):
+    list_display = ('name', 'code', 'is_primary', 'status', 'media_count', 'created_by', 'created_at')
+    list_filter = (ResourceThemeTypeFilter, 'status', CustomAdvanceDateFilter)
+    list_editable = ('status',)
+    search_fields = ('name', 'code', 'description')
+    raw_id_fields = ('created_by',)
+    readonly_fields = ('created_at', 'updated_at')
+    date_hierarchy = 'created_at'
+    ordering = ('-is_primary', 'name')
+    actions = ('mark_published', 'mark_draft')
+
+    def get_urls(self):
+        custom_urls = [
+            path('export-csv/', self.admin_site.admin_view(self.export_csv_view),
+                 name='chatbot_resourcetheme_export_csv'),
+        ]
+        return custom_urls + super().get_urls()
+
+    def export_csv_view(self, request):
+        if not (self.has_view_permission(request) and request.user.has_perm('chatbot.view_media')):
+            raise PermissionDenied
+        themes = self.get_changelist_instance(request).queryset.filter(is_primary=False)
+        links = (
+            MediaSecondaryTheme.objects.filter(theme__in=themes.values('pk'), media__parent__isnull=True)
+            .select_related('theme', 'media__primary_theme')
+            .prefetch_related(Prefetch(
+                'media__key_values', queryset=KeyValue.objects.filter(key=TITLE_KEY), to_attr='title_values'
+            ))
+            .order_by('media_id', '-confidence')
+        )
+        if not request.user.is_superuser:
+            company = get_user_company(request.user)
+            links = links.filter(media__organization=company) if company else links.none()
+
+        response = csv_response(f'secondary_theme_mapping_{timezone.localdate():%Y%m%d}.csv')
+        writer = csv.writer(response)
+        writer.writerow(SECONDARY_MAPPING_EXPORT_FIELDS)
+        for _, media_links in groupby(links, key=lambda link: link.media.id):
+            media_links = list(media_links)
+            media = media_links[0].media
+            title = media.title_values[0].value if media.title_values else ''
+            writer.writerow([
+                csv_safe(title or media.name),
+                csv_safe(media.primary_theme.name if media.primary_theme else ''),
+                csv_safe(', '.join(link.theme.name for link in media_links)),
+                ', '.join(f'{link.confidence:.2f}' if link.confidence is not None else '-' for link in media_links),
+            ])
+        return response
+
+    def get_queryset(self, request):
+        return super().get_queryset(request).select_related('created_by').annotate(
+            primary_count=Count('primary_media', distinct=True),
+            secondary_count=Count('media_links', distinct=True),
+        )
+
+    @admin.display(description='Media')
+    def media_count(self, obj):
+        return obj.primary_count if obj.is_primary else obj.secondary_count
+
+    def has_add_permission(self, request):
+        return False
+
+    def get_readonly_fields(self, request, obj=None):
+        if obj:
+            return self.readonly_fields + ('code', 'description')
+        return self.readonly_fields
+
+    def save_model(self, request, obj, form, change):
+        if not change and not obj.created_by_id:
+            obj.created_by = Profile.objects.filter(email=request.user.email).first()
+        super().save_model(request, obj, form, change)
+
+    def _set_status(self, request, queryset, status):
+        updated = queryset.update(status=status)
+        self.message_user(request, f"{updated} theme(s) marked as {status}.")
+
+    @admin.action(description='Mark selected as published')
+    def mark_published(self, request, queryset):
+        self._set_status(request, queryset, ThemeStatus.PUBLISHED)
+
+    @admin.action(description='Mark selected as draft')
+    def mark_draft(self, request, queryset):
+        self._set_status(request, queryset, ThemeStatus.DRAFT)