Source code for beat.web.experiments.admin

#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.web module of the BEAT platform.              #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################

import simplejson as json
from django import forms
from django.contrib import admin
from django.core.files.base import ContentFile
from django.db.models import Count
from django.db.models import Max
from django.urls import reverse
from django.utils.html import format_html
from django.utils.safestring import mark_safe

from ..common.texts import Messages
from ..ui.forms import CodeMirrorJSONCharField
from ..ui.forms import CodeMirrorJSONFileField
from ..ui.forms import CodeMirrorRSTFileField
from ..ui.forms import NameField
from .models import Block as BlockModel
from .models import BlockInput as BlockInputModel
from .models import CachedFile as CachedFileModel
from .models import Experiment as ExperimentModel
from .models import Result as ResultModel
from .models import validate_experiment

# ----------------------------------------------------------


class ExperimentModelForm(forms.ModelForm):

    name = NameField(
        widget=forms.TextInput(attrs=dict(size=80)), help_text=Messages["name"],
    )

    declaration_file = CodeMirrorJSONFileField(
        label="Declaration", help_text=Messages["json"],
    )

    description_file = CodeMirrorRSTFileField(
        label="Description",
        required=False,
        allow_empty_file=True,
        help_text=Messages["description"],
    )

    class Meta:
        model = ExperimentModel
        exclude = []
        widgets = {
            "short_description": forms.TextInput(attrs=dict(size=100),),
        }

    def clean_declaration_file(self):
        """Cleans-up the declaration_file data, make sure it is really new"""

        new_declaration = self.cleaned_data["declaration_file"].read()
        old_declaration = ""

        if self.instance and self.instance.declaration_file.name is not None:
            old_declaration = self.instance.declaration_string
            if new_declaration == old_declaration:
                self.changed_data.remove("declaration_file")
                content_file = ContentFile(old_declaration)
                content_file.name = self.instance.declaration_file.name
                return content_file

        try:
            core_experiment, errors = validate_experiment(
                json.loads(new_declaration), self.cleaned_data["toolchain"].declaration
            )
        except SyntaxError as e:
            raise forms.ValidationError(str(e))

        if errors:
            all_errors = [forms.ValidationError(k) for k in errors]
            raise forms.ValidationError(all_errors)

        self.cleaned_data["declaration_file"].seek(0)  # reset ContentFile readout
        return self.cleaned_data["declaration_file"]

    def clean(self):
        """Cleans-up the input data, make sure it overall validates"""

        if "declaration_file" in self.data and isinstance(
            self.data["declaration_file"], str
        ):
            mutable_data = self.data.copy()
            mutable_data["declaration_file"] = ContentFile(
                self.data["declaration_file"], name="unsaved"
            )
            self.data = mutable_data


class BlockInline(admin.TabularInline):

    model = BlockModel
    extra = 0

    readonly_fields = ["execution_order", "link", "algorithm", "analyzer", "status"]
    ordering = ["execution_order"]
    fields = readonly_fields

    def link(self, obj):
        url = reverse("admin:experiments_block_change", args=(obj.pk,))
        return mark_safe('<a href="%s">%s</a>' % (url, obj.name))  # nosec

    link.short_description = "name"

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request, obj=None):
        return False


# ----------------------------------------------------------


def reset_experiment(modeladmin, request, queryset):
    for q in queryset:
        q.reset()


reset_experiment.short_description = "Reset selected experiments"


def cancel_experiment(modeladmin, request, queryset):
    for q in queryset:
        q.cancel()


cancel_experiment.short_description = "Cancel selected experiments"


def rehash_experiment(modeladmin, request, queryset):
    for q in queryset:
        q.save()


rehash_experiment.short_description = "Rehash selected experiments"


class Experiment(admin.ModelAdmin):

    list_display = (
        "id",
        "author",
        "toolchain",
        "name",
        "creation_date",
        "start_date",
        "end_date",
        "status",
        "sharing",
    )
    search_fields = [
        "author__username",
        "toolchain__name",
        "toolchain__author__username",
        "name",
        "short_description",
    ]
    readonly_fields = (
        "hash",
        "referenced_datasets",
        "referenced_algorithms",
        "short_description",
    )
    list_display_links = ("id",)

    actions = [
        rehash_experiment,
        reset_experiment,
        cancel_experiment,
    ]

    form = ExperimentModelForm

    filter_horizontal = ["shared_with", "shared_with_team"]

    inlines = [
        BlockInline,
    ]

    fieldsets = (
        (None, dict(fields=("name", "author", "toolchain"),),),
        (
            "Status and dates",
            dict(classes=("collapse",), fields=("start_date", "end_date", "status"),),
        ),
        (
            "Documentation",
            dict(
                classes=("collapse",),
                fields=("short_description", "description_file",),
            ),
        ),
        (
            "References (read-only)",
            dict(
                classes=("collapse",),
                fields=("referenced_datasets", "referenced_algorithms",),
            ),
        ),
        (
            "Sharing",
            dict(
                classes=("collapse",),
                fields=("sharing", "shared_with", "shared_with_team"),
            ),
        ),
        ("Source code", dict(fields=("hash", "declaration_file"),),),
    )


admin.site.register(ExperimentModel, Experiment)

# ----------------------------------------------------------


class BlockInputInline(admin.TabularInline):

    model = BlockInputModel
    verbose_name = "Input"
    verbose_name_plural = "Inputs"
    extra = 0
    ordering = ["database", "cache"]
    readonly_fields = ["input", "channel"]
    fields = readonly_fields

    def input(self, obj):
        if obj.database:
            url = reverse(
                "admin:databases_databaseset_change", args=(obj.database.set.pk,)
            )
            text = "%s (%s)" % (obj.database, obj.database.hash)
            what = "Dataset Output"
        else:
            url = reverse("admin:experiments_cachedfile_change", args=(obj.cache.pk,))
            text = obj.cache.hash
            what = "Cached File"
        return mark_safe('%s: <a href="%s">%s</a>' % (what, url, text))  # nosec

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request, obj=None):
        return False


class CachedFileInline(admin.TabularInline):

    model = CachedFileModel.blocks.through
    verbose_name = "Output"
    verbose_name_plural = "Outputs"
    extra = 0

    readonly_fields = ["output"]
    fields = readonly_fields

    def output(self, obj):
        url = reverse("admin:experiments_cachedfile_change", args=(obj.cachedfile.pk,))
        text = obj.cachedfile.hash
        what = "Cached File"
        return mark_safe('%s: <a href="%s">%s</a>' % (what, url, text))  # nosec

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request, obj=None):
        return False


class BlockDependentsInline(admin.TabularInline):

    model = BlockModel.dependencies.through
    verbose_name = "Dependent"
    verbose_name_plural = "Dependents"
    fk_name = "to_block"
    extra = 0

    readonly_fields = ["order", "name", "algorithm", "analyzer", "status"]
    ordering = ["id"]
    fields = readonly_fields

    def order(self, obj):
        return obj.from_block.execution_order

    def name(self, obj):
        url = reverse("admin:experiments_block_change", args=(obj.from_block.pk,))
        return mark_safe('<a href="%s">%s</a>' % (url, obj.from_block.name))  # nosec

    def algorithm(self, obj):
        return obj.from_block.algorithm

    def analyzer(self, obj):
        return obj.from_block.analyzer

    analyzer.boolean = True

    def status(self, obj):
        return obj.from_block.get_status_display()

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request, obj=None):
        return False


class BlockDependenciesInline(admin.TabularInline):

    model = BlockModel.dependencies.through
    verbose_name = "Dependency"
    verbose_name_plural = "Dependencies"
    fk_name = "from_block"
    extra = 0

    readonly_fields = ["order", "name", "algorithm", "analyzer", "status"]
    ordering = ["id"]
    fields = readonly_fields

    def order(self, obj):
        return obj.to_block.execution_order

    def name(self, obj):
        url = reverse("admin:experiments_block_change", args=(obj.to_block.pk,))
        return mark_safe('<a href="%s">%s</a>' % (url, obj.to_block.name))  # nosec

    def algorithm(self, obj):
        return obj.to_block.algorithm

    def analyzer(self, obj):
        return obj.to_block.analyzer

    analyzer.boolean = True

    def status(self, obj):
        return obj.to_block.get_status_display()

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request, obj=None):
        return False


class BlockModelForm(forms.ModelForm):

    command = CodeMirrorJSONCharField(help_text=Messages["json"], readonly=True,)

    class Meta:
        model = BlockModel
        exclude = []


class Block(admin.ModelAdmin):

    list_display = (
        "id",
        "author",
        "toolchain",
        "xp",
        "execution_order",
        "name",
        "algorithm",
        "analyzer",
        "status",
        "ins",
        "outs",
        "environment",
        "q",
    )

    search_fields = [
        "name",
        "experiment__author__username",
        "experiment__toolchain__author__username",
        "experiment__toolchain__name",
        "experiment__name",
        "algorithm__author__username",
        "algorithm__name",
        "environment__name",
        "environment__version",
    ]

    list_display_links = ("id", "name")

    inlines = [
        BlockDependenciesInline,
        BlockInputInline,
        CachedFileInline,
        BlockDependentsInline,
    ]

    exclude = ["dependencies"]

    def get_queryset(self, request):
        qs = super(Block, self).get_queryset(request)
        return qs.annotate(Count("outputs"))

    def author(self, obj):
        return obj.experiment.author

    def toolchain(self, obj):
        return obj.experiment.toolchain

    def xp(self, obj):
        return obj.experiment.name

    xp.short_description = "experiment"

    def ins(self, obj):
        return obj.inputs.count()

    def outs(self, obj):
        return obj.outputs__count

    outs.admin_order_field = "outputs__count"

    def q(self, obj):
        if obj.queue:
            return obj.queue.name
        return None

    q.short_description = "queue"

    def get_readonly_fields(self, request, obj=None):
        return list(self.readonly_fields) + [
            field.name for field in obj._meta.fields if field.name != "command"
        ]

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request, obj=None):
        return False

    form = BlockModelForm

    fieldsets = (
        (None, dict(fields=("id", "name", "experiment"),),),
        (
            "Status and dates",
            dict(fields=("creation_date", "start_date", "end_date", "status"),),
        ),
        ("Code", dict(classes=("collapse",), fields=("algorithm", "analyzer",),),),
        (
            "Backend",
            dict(
                classes=("collapse",),
                fields=("environment", "queue", "required_slots", "channel"),
            ),
        ),
        ("Command", dict(classes=("collapse",), fields=("command",),),),
    )


admin.site.register(BlockModel, Block)


# ----------------------------------------------------------


class Result(admin.ModelAdmin):

    list_display = ("id", "cache", "name", "type", "primary", "data_value")

    search_fields = [
        "name",
        "cache__hash",
    ]

    list_display_links = ("id", "name")

    list_select_related = ("cache",)

    def get_readonly_fields(self, request, obj=None):
        return list(self.readonly_fields) + [field.name for field in obj._meta.fields]

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request, obj=None):
        return False


admin.site.register(ResultModel, Result)


# ----------------------------------------------------------


def delete_file_on_fs(modeladmin, request, queryset):
    """
    Delete the files contained in the cache
    """

    for obj in queryset:
        obj.delete_files()


delete_file_on_fs.short_description = "Delete files from the cache"


def cascading_delete_file_on_fs(modeladmin, request, queryset):
    """
    Delete the files contained in the cache
    """

    for obj in queryset:
        for block in obj.blocks.all():
            experiment = block.experiment
            for exp_block in experiment.blocks.all():
                for result in exp_block.results.all():
                    if result.cache is not None:
                        result.cache.delete_files()
                for input_ in exp_block.inputs.all():
                    if input_.cache is not None:
                        input_.cache.delete_files()


cascading_delete_file_on_fs.short_description = (
    "Delete files from the " "selected and related caches"
)


class CachedFile(admin.ModelAdmin):

    search_fields = [
        "hash",
        "blocks__name",
        "blocks__experiment__name",
    ]

    list_display = (
        "id",
        "hash",
        "status",
        "date",
        "blocks_url",
    )

    list_display_links = ("id", "hash")

    list_filter = ("status",)

    # to avoid very slow loading of cached files
    raw_id_fields = ("blocks",)

    actions = [delete_file_on_fs, cascading_delete_file_on_fs]

    def get_queryset(self, request):
        qs = super(CachedFile, self).get_queryset(request)
        return qs.annotate(date=Max("blocks__start_date"))

    def get_actions(self, request):
        actions = super(CachedFile, self).get_actions(request)
        if "delete_selected" in actions:
            del actions["delete_selected"]
        return actions

    def date(self, obj):
        return obj.date

    date.admin_order_field = "-date"

    def blocks_url(self, obj):
        retval = "<ul>"
        for block in obj.blocks.all():
            retval += format_html(
                "<li><a href='{block_url}'>{block_name}</a> @ <a href='{experiment_url}'>{experiment_name}</a> ({block_status})</li>",
                experiment_url=reverse(
                    "admin:experiments_experiment_change", args=(block.experiment.id,)
                ),
                experiment_name=block.experiment.fullname(),
                block_url=reverse("admin:experiments_block_change", args=(block.id,)),
                block_name=block.name,
                block_status=block.get_status_display(),
            )
        return retval + "</ul>"

    blocks_url.short_description = "Blocks"
    blocks_url.allow_tags = True

    fieldsets = (
        (None, dict(fields=("hash", "status", "blocks",)),),
        ("Logging", dict(fields=("error_report", "stderr", "stdout"),),),
        (
            "Performance",
            dict(
                classes=("collapse",),
                fields=(
                    "linear_execution_time",
                    "speed_up_real",
                    "speed_up_maximal",
                    "cpu_time",
                    "max_memory",
                    "queuing_time",
                    "data_read_time",
                    "data_read_size",
                    "data_read_nb_blocks",
                    "data_written_time",
                    "data_written_size",
                    "data_written_nb_blocks",
                ),
            ),
        ),
    )

    readonly_fields = ["blocks"]

    def get_readonly_fields(self, request, obj=None):
        return list(self.readonly_fields) + [field.name for field in obj._meta.fields]

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request, obj=None):
        return False


admin.site.register(CachedFileModel, CachedFile)