Add decrypt PDF API endpoint, add decryption to approve process

froide/foirequest/api_views/attachment.py

@@ -14,7 +14,7 @@
 from froide.campaign.models import Campaign
 from froide.document.api_views import DocumentSerializer
 from froide.foirequest.models.message import FoiMessage
-from froide.foirequest.utils import find_attachment_name
+from froide.foirequest.utils import create_decrypted_attachment, find_attachment_name
 from froide.helper.auth import is_crew
 from froide.helper.storage import make_unique_filename
 from froide.helper.text_utils import slugify
@@ -28,11 +28,15 @@
 from ..models import FoiAttachment, FoiEvent
 from ..permissions import WriteFoiRequestPermission
 from ..serializers import (
+    DecryptAttachmentSerializer,
     FoiAttachmentSerializer,
     FoiAttachmentTusSerializer,
     ImageAttachmentConverterSerializer,
 )
-from ..tasks import convert_images_to_pdf_api_task, move_upload_to_attachment
+from ..tasks import (
+    convert_images_to_pdf_api_task,
+    move_upload_to_attachment,
+)
 
 User = get_user_model()
 
@@ -337,3 +341,34 @@ def convert_to_pdf(self, request, pk=None):
             )
         else:
             return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+    @extend_schema(
+        responses={status.HTTP_201_CREATED: FoiAttachmentSerializer},
+        operation_id="decrypt_pdf",
+    )
+    @action(
+        detail=True,
+        methods=["post"],
+        url_path="decrypt-pdf",
+        serializer_class=DecryptAttachmentSerializer,
+    )
+    def decrypt_pdf(self, request, pk=None):
+        attachment = self.get_object()
+        if not attachment.is_pdf:
+            return Response(
+                {"detail": _("Attachment is not a PDF.")},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+        serializer = self.get_serializer(data=request.data)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        data = serializer.validated_data
+        decrypted_att = create_decrypted_attachment(
+            attachment, data["password"], approved=data["approved"]
+        )
+
+        return Response(
+            FoiAttachmentSerializer(decrypted_att, context={"request": request}).data,
+            status=status.HTTP_201_CREATED,
+        )

froide/foirequest/serializers.py

@@ -534,6 +534,11 @@ class ImageAttachmentConverterSerializer(serializers.Serializer):
     message = FoiMessageRelatedField()
 
 
+class DecryptAttachmentSerializer(serializers.Serializer):
+    password = serializers.CharField(required=True)
+    approved = serializers.BooleanField(default=False, required=False)
+
+
 def optimize_message_queryset(request, qs):
     atts = get_read_foiattachment_queryset(
         request, queryset=FoiAttachment.objects.filter(belongs_to__in=qs)

froide/foirequest/tasks.py

@@ -3,6 +3,7 @@
 from collections.abc import Collection
 from datetime import timedelta
 from functools import partial
+from typing import Optional
 
 from django.conf import settings
 from django.core.files.base import ContentFile
@@ -15,7 +16,7 @@
 
 from froide.celery import app as celery_app
 from froide.foirequest.models.event import FoiEvent
-from froide.foirequest.utils import find_attachment_name
+from froide.foirequest.utils import make_decrypted_attachment
 from froide.proof.models import Proof
 from froide.publicbody.models import PublicBody
 from froide.upload.models import Upload
@@ -546,7 +547,9 @@ def warn_on_unprocessed_mail():
 @celery_app.task(
     name="froide.foirequest.tasks.decrypt_pdf_attachment_task", time_limit=600
 )
-def decrypt_pdf_attachment_task(att_id, password):
+def decrypt_pdf_attachment_task(
+    att_id: int, decrypted_id: Optional[int] = None, password: str = ""
+):
     from filingcabinet.pdf_utils import decrypt_pdf
     from filingcabinet.utils import get_local_file
 
@@ -561,24 +564,35 @@ def decrypt_pdf_attachment_task(att_id, password):
     with get_local_file(att.file.path) as file_path:
         output_bytes = decrypt_pdf(file_path, password)
 
+    if decrypted_id is not None:
+        try:
+            decrypted_att = FoiAttachment.objects.get(pk=decrypted_id)
+        except FoiAttachment.DoesNotExist:
+            return
+    else:
+        decrypted_att = make_decrypted_attachment(
+            att,
+            att_kwargs={
+                "approved": False,
+            },
+        )
+
     if output_bytes is None:
+        # Decryption failed, let's delete the decrypted attachment if already saved
+        if decrypted_att.pk is not None:
+            decrypted_att.delete()
+
         return
 
-    name, ext = os.path.splitext(att.name)
-    name = _("{name}_decrypted{ext}").format(name=name, ext=".pdf")
-    name = find_attachment_name(name, att.belongs_to)
-
-    decrypted_att = FoiAttachment.objects.create(
-        name=name,
-        belongs_to=att.belongs_to,
-        approved=False,
-        filetype="application/pdf",
-        is_converted=True,
-        can_approve=att.can_approve,
-    )
     new_file = ContentFile(output_bytes)
     decrypted_att.size = new_file.size
+    decrypted_att.pending = False
     decrypted_att.file.save(att.name, new_file, save=True)
 
-    att.converted = decrypted_att
-    att.save()
+    if att.converted != decrypted_att:
+        if att.converted is not None:
+            # Previously converted version is no longer needed
+            att.converted.remove_file_and_delete()
+
+        att.converted = decrypted_att
+        att.save()

froide/foirequest/templates/foirequest/redact.html

@@ -22,9 +22,7 @@ <h2>{% blocktrans with name=attachment.name %}Redact "{{ name }}"{% endblocktran
         <form id="redaction-form" method="post" style="display:none">
             {% csrf_token %}
         </form>
-        <pdf-redaction id="redact" pdf-path="{{ attachment_url }}" attachment-id="{{ attachment.id }}" attachment-url="{{ attachment.get_anchor_url }}" :redact-regex="{{ foirequest.get_redaction_regexes }}"
-        {% if not attachment.redacted and not attachment.is_redacted %}can-publish="true"{% endif %}
-        :config="{{ config }}" :bottom-toolbar="true">
+        <pdf-redaction id="redact" pdf-path="{{ attachment_url }}" attachment-id="{{ attachment.id }}" attachment-url="{{ attachment.get_anchor_url }}" attachment-id="{{ attachment.id }}" :redact-regex="{{ foirequest.get_redaction_regexes }}" :config="{{ config }}" :bottom-toolbar="true">
         <div class="text-center">
             <h3>{% trans "Redaction tool is loading..." %}</h3>
             <div class="spinner-border" role="status">

froide/foirequest/tests/test_api_attachment.py

@@ -8,6 +8,7 @@
 
 from froide.document.factories import DocumentFactory
 from froide.foirequest.models.event import FoiEvent
+from froide.foirequest.models import FoiAttachment
 from froide.foirequest.models.message import MessageKind
 from froide.foirequest.tests import factories
 from froide.upload.factories import UploadFactory
@@ -387,3 +388,95 @@ def test_retrieve_attachment_on_draft(client: Client, user):
     data = response.json()
     assert len(data["objects"]) == 1
     assert data["objects"][0]["id"] == attachment.pk
+
+
+@pytest.mark.django_db(transaction=True)
+def test_decrypt_attachment(client: Client, user, monkeypatch):
+    request = factories.FoiRequestFactory.create(user=user)
+    message = factories.FoiMessageFactory.create(request=request, kind=MessageKind.POST)
+
+    encrypted = factories.FoiAttachmentFactory.create(
+        belongs_to=message,
+        filetype="application/pdf",
+    )
+
+    data = {
+        "bad": "wrong",
+        "appoved": True,
+    }
+    url = reverse("api:attachment-decrypt-pdf", kwargs={"pk": encrypted.pk})
+
+    # needs to be logged in
+    response = client.post(
+        url,
+        data=data,
+        content_type="application/json",
+    )
+    assert response.status_code == 401
+
+    # wrong user
+    other_user = factories.UserFactory.create()
+    assert client.login(email=other_user.email, password="froide")
+    response = client.post(
+        url,
+        data=data,
+        content_type="application/json",
+    )
+    assert response.status_code == 403
+
+    # bad data
+    assert client.login(email=user.email, password="froide")
+    response = client.post(
+        url,
+        data=data,
+        content_type="application/json",
+    )
+    assert response.status_code == 400
+
+    def mock_decrypt(path, password):
+        if password == "test":
+            return b"1"
+        return None
+
+    import filingcabinet.pdf_utils
+
+    monkeypatch.setattr(filingcabinet.pdf_utils, "decrypt_pdf", mock_decrypt)
+
+    # wrong password
+    data = {
+        "password": "wrong",
+        "approved": True,
+    }
+    assert client.login(email=user.email, password="froide")
+    response = client.post(
+        url,
+        data=data,
+        content_type="application/json",
+    )
+    # Attachment gets created but in failed decryption task deleted
+    assert response.status_code == 201
+    result = response.json()
+    assert not FoiAttachment.objects.filter(id=result["id"]).exists()
+
+    # everything good
+    data["password"] = "test"
+    assert client.login(email=user.email, password="froide")
+    response = client.post(
+        url,
+        data=data,
+        content_type="application/json",
+    )
+    assert response.status_code == 201
+    result = response.json()
+
+    assert result["name"].endswith("-decrypted.pdf")
+    assert result["filetype"] == "application/pdf"
+    assert result["is_converted"] is True
+
+    decrypted = FoiAttachment.objects.get(id=result["id"])
+    assert decrypted.belongs_to == message
+    assert decrypted.approved
+    assert decrypted.is_converted
+
+    encrypted.refresh_from_db()
+    assert encrypted.converted_id == decrypted.id

froide/foirequest/utils.py

@@ -1,9 +1,11 @@
 import datetime
 import json
+import os
 import re
 import zipfile
 from dataclasses import dataclass
 from datetime import timedelta
+from functools import partial
 from io import BytesIO
 from pathlib import PurePath
 from typing import TYPE_CHECKING, Iterator, List, Optional, Tuple, Union
@@ -14,6 +16,7 @@
 from django.core.files import File
 from django.core.mail import mail_managers
 from django.core.validators import validate_email
+from django.db import transaction
 from django.template.loader import render_to_string
 from django.urls import reverse
 from django.utils import timezone
@@ -998,3 +1001,55 @@ def find_attachment_name(name: str, message_id: int) -> str:
         )
     )
     return make_unique_filename(name, attachment_names)
+
+
+def make_converted_attachment(
+    att: FoiAttachment, postfix: str, att_kwargs: dict = None
+) -> FoiAttachment:
+    if att_kwargs is None:
+        att_kwargs = {}
+    name, ext = os.path.splitext(att.name)
+    name = "{name}_{postfix}{ext}".format(name=name, postfix=postfix, ext=ext)
+    name = find_attachment_name(name, att.belongs_to)
+
+    return FoiAttachment(
+        name=name,
+        belongs_to=att.belongs_to,
+        is_converted=True,
+        can_approve=att.can_approve,
+        **att_kwargs,
+    )
+
+
+def make_decrypted_attachment(att: FoiAttachment, att_kwargs=None) -> FoiAttachment:
+    # Translators: filename postfix for decrypted file
+    postfix = _("decrypted")
+    if att_kwargs is None:
+        att_kwargs = {}
+    att_kwargs.setdefault("filetype", "application/pdf")
+    return make_converted_attachment(att, postfix, att_kwargs=att_kwargs)
+
+
+def create_decrypted_attachment(
+    attachment: FoiAttachment, password: str, approved: bool = True
+) -> FoiAttachment:
+    from .tasks import decrypt_pdf_attachment_task
+
+    decrypted_att = make_decrypted_attachment(
+        attachment,
+        att_kwargs={
+            "pending": True,
+            "approved": approved,
+        },
+    )
+    decrypted_att.save()
+
+    transaction.on_commit(
+        partial(
+            decrypt_pdf_attachment_task.delay,
+            attachment.id,
+            decrypted_id=decrypted_att.id,
+            password=password,
+        )
+    )
+    return decrypted_att

froide/foirequest/views/attachment.py

@@ -15,6 +15,7 @@
 
 from crossdomainmedia import CrossDomainMediaMixin
 
+from froide.foirequest.utils import create_decrypted_attachment
 from froide.helper.auth import is_crew
 from froide.helper.utils import is_ajax, render_400, render_403
 
@@ -78,7 +79,22 @@ def approve_attachment(request, foirequest, attachment_id):
 
     # hard guard against publishing of non publishable requests
     if not foirequest.not_publishable:
-        att.approve_and_save()
+        if att.redacted:
+            # if there are redacted versions, delete them
+            # as we are approving the original
+            redacted = att.redacted
+            redacted.attachment_deleted.send(
+                sender=redacted,
+                user=request.user,
+            )
+            redacted.remove_file_and_delete()
+
+        if att.is_pdf and request.POST.get("password") is not None:
+            att = create_decrypted_attachment(
+                att, request.POST["password"], approved=True
+            )
+        else:
+            att.approve_and_save()
         att.attachment_approved.send(
             sender=att,
             user=request.user,

froide/helper/redaction.py

@@ -77,11 +77,7 @@ def try_redacting_file(pdf_file, outpath, instructions):
     tries = 0
     while True:
         try:
-            rewritten_pdf_file = rewrite_pdf(pdf_file, instructions)
-            if rewritten_pdf_file is None:
-                # Possibly encrypted with password, let's just try it anyway
-                rewritten_pdf_file = pdf_file
-            return _redact_file(rewritten_pdf_file, outpath, instructions)
+            return _redact_file(pdf_file, outpath, instructions)
         except PDFException as e:
             tries += 1
             if tries > 2: