diff --git a/froide/foirequest/api_views/attachment.py b/froide/foirequest/api_views/attachment.py index f0610650e..192198016 100644 --- a/froide/foirequest/api_views/attachment.py +++ b/froide/foirequest/api_views/attachment.py @@ -14,7 +14,7 @@ from froide.campaign.models import Campaign from froide.document.api_views import DocumentSerializer from froide.foirequest.models.message import FoiMessage -from froide.foirequest.utils import find_attachment_name +from froide.foirequest.utils import create_decrypted_attachment, find_attachment_name from froide.helper.auth import is_crew from froide.helper.storage import make_unique_filename from froide.helper.text_utils import slugify @@ -28,11 +28,15 @@ from ..models import FoiAttachment, FoiEvent from ..permissions import WriteFoiRequestPermission from ..serializers import ( + DecryptAttachmentSerializer, FoiAttachmentSerializer, FoiAttachmentTusSerializer, ImageAttachmentConverterSerializer, ) -from ..tasks import convert_images_to_pdf_api_task, move_upload_to_attachment +from ..tasks import ( + convert_images_to_pdf_api_task, + move_upload_to_attachment, +) User = get_user_model() @@ -337,3 +341,34 @@ def convert_to_pdf(self, request, pk=None): ) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + @extend_schema( + responses={status.HTTP_201_CREATED: FoiAttachmentSerializer}, + operation_id="decrypt_pdf", + ) + @action( + detail=True, + methods=["post"], + url_path="decrypt-pdf", + serializer_class=DecryptAttachmentSerializer, + ) + def decrypt_pdf(self, request, pk=None): + attachment = self.get_object() + if not attachment.is_pdf: + return Response( + {"detail": _("Attachment is not a PDF.")}, + status=status.HTTP_400_BAD_REQUEST, + ) + serializer = self.get_serializer(data=request.data) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + data = serializer.validated_data + decrypted_att = create_decrypted_attachment( + attachment, data["password"], approved=data["approved"] + ) + + return Response( + FoiAttachmentSerializer(decrypted_att, context={"request": request}).data, + status=status.HTTP_201_CREATED, + ) diff --git a/froide/foirequest/serializers.py b/froide/foirequest/serializers.py index c6f80029b..9c2727b6a 100644 --- a/froide/foirequest/serializers.py +++ b/froide/foirequest/serializers.py @@ -534,6 +534,11 @@ class ImageAttachmentConverterSerializer(serializers.Serializer): message = FoiMessageRelatedField() +class DecryptAttachmentSerializer(serializers.Serializer): + password = serializers.CharField(required=True) + approved = serializers.BooleanField(default=False, required=False) + + def optimize_message_queryset(request, qs): atts = get_read_foiattachment_queryset( request, queryset=FoiAttachment.objects.filter(belongs_to__in=qs) diff --git a/froide/foirequest/tasks.py b/froide/foirequest/tasks.py index b85c777b4..96e5c5473 100644 --- a/froide/foirequest/tasks.py +++ b/froide/foirequest/tasks.py @@ -3,6 +3,7 @@ from collections.abc import Collection from datetime import timedelta from functools import partial +from typing import Optional from django.conf import settings from django.core.files.base import ContentFile @@ -15,7 +16,7 @@ from froide.celery import app as celery_app from froide.foirequest.models.event import FoiEvent -from froide.foirequest.utils import find_attachment_name +from froide.foirequest.utils import make_decrypted_attachment from froide.proof.models import Proof from froide.publicbody.models import PublicBody from froide.upload.models import Upload @@ -546,7 +547,9 @@ def warn_on_unprocessed_mail(): @celery_app.task( name="froide.foirequest.tasks.decrypt_pdf_attachment_task", time_limit=600 ) -def decrypt_pdf_attachment_task(att_id, password): +def decrypt_pdf_attachment_task( + att_id: int, decrypted_id: Optional[int] = None, password: str = "" +): from filingcabinet.pdf_utils import decrypt_pdf from filingcabinet.utils import get_local_file @@ -561,24 +564,35 @@ def decrypt_pdf_attachment_task(att_id, password): with get_local_file(att.file.path) as file_path: output_bytes = decrypt_pdf(file_path, password) + if decrypted_id is not None: + try: + decrypted_att = FoiAttachment.objects.get(pk=decrypted_id) + except FoiAttachment.DoesNotExist: + return + else: + decrypted_att = make_decrypted_attachment( + att, + att_kwargs={ + "approved": False, + }, + ) + if output_bytes is None: + # Decryption failed, let's delete the decrypted attachment if already saved + if decrypted_att.pk is not None: + decrypted_att.delete() + return - name, ext = os.path.splitext(att.name) - name = _("{name}_decrypted{ext}").format(name=name, ext=".pdf") - name = find_attachment_name(name, att.belongs_to) - - decrypted_att = FoiAttachment.objects.create( - name=name, - belongs_to=att.belongs_to, - approved=False, - filetype="application/pdf", - is_converted=True, - can_approve=att.can_approve, - ) new_file = ContentFile(output_bytes) decrypted_att.size = new_file.size + decrypted_att.pending = False decrypted_att.file.save(att.name, new_file, save=True) - att.converted = decrypted_att - att.save() + if att.converted != decrypted_att: + if att.converted is not None: + # Previously converted version is no longer needed + att.converted.remove_file_and_delete() + + att.converted = decrypted_att + att.save() diff --git a/froide/foirequest/templates/foirequest/redact.html b/froide/foirequest/templates/foirequest/redact.html index 972b15ee5..25b9374c5 100644 --- a/froide/foirequest/templates/foirequest/redact.html +++ b/froide/foirequest/templates/foirequest/redact.html @@ -22,9 +22,7 @@

{% blocktrans with name=attachment.name %}Redact "{{ name }}"{% endblocktran - +

{% trans "Redaction tool is loading..." %}

diff --git a/froide/foirequest/tests/test_api_attachment.py b/froide/foirequest/tests/test_api_attachment.py index d5a825885..fbe752db4 100644 --- a/froide/foirequest/tests/test_api_attachment.py +++ b/froide/foirequest/tests/test_api_attachment.py @@ -8,6 +8,7 @@ from froide.document.factories import DocumentFactory from froide.foirequest.models.event import FoiEvent +from froide.foirequest.models import FoiAttachment from froide.foirequest.models.message import MessageKind from froide.foirequest.tests import factories from froide.upload.factories import UploadFactory @@ -387,3 +388,95 @@ def test_retrieve_attachment_on_draft(client: Client, user): data = response.json() assert len(data["objects"]) == 1 assert data["objects"][0]["id"] == attachment.pk + + +@pytest.mark.django_db(transaction=True) +def test_decrypt_attachment(client: Client, user, monkeypatch): + request = factories.FoiRequestFactory.create(user=user) + message = factories.FoiMessageFactory.create(request=request, kind=MessageKind.POST) + + encrypted = factories.FoiAttachmentFactory.create( + belongs_to=message, + filetype="application/pdf", + ) + + data = { + "bad": "wrong", + "appoved": True, + } + url = reverse("api:attachment-decrypt-pdf", kwargs={"pk": encrypted.pk}) + + # needs to be logged in + response = client.post( + url, + data=data, + content_type="application/json", + ) + assert response.status_code == 401 + + # wrong user + other_user = factories.UserFactory.create() + assert client.login(email=other_user.email, password="froide") + response = client.post( + url, + data=data, + content_type="application/json", + ) + assert response.status_code == 403 + + # bad data + assert client.login(email=user.email, password="froide") + response = client.post( + url, + data=data, + content_type="application/json", + ) + assert response.status_code == 400 + + def mock_decrypt(path, password): + if password == "test": + return b"1" + return None + + import filingcabinet.pdf_utils + + monkeypatch.setattr(filingcabinet.pdf_utils, "decrypt_pdf", mock_decrypt) + + # wrong password + data = { + "password": "wrong", + "approved": True, + } + assert client.login(email=user.email, password="froide") + response = client.post( + url, + data=data, + content_type="application/json", + ) + # Attachment gets created but in failed decryption task deleted + assert response.status_code == 201 + result = response.json() + assert not FoiAttachment.objects.filter(id=result["id"]).exists() + + # everything good + data["password"] = "test" + assert client.login(email=user.email, password="froide") + response = client.post( + url, + data=data, + content_type="application/json", + ) + assert response.status_code == 201 + result = response.json() + + assert result["name"].endswith("-decrypted.pdf") + assert result["filetype"] == "application/pdf" + assert result["is_converted"] is True + + decrypted = FoiAttachment.objects.get(id=result["id"]) + assert decrypted.belongs_to == message + assert decrypted.approved + assert decrypted.is_converted + + encrypted.refresh_from_db() + assert encrypted.converted_id == decrypted.id diff --git a/froide/foirequest/utils.py b/froide/foirequest/utils.py index 0399c6346..a1c49f6f6 100644 --- a/froide/foirequest/utils.py +++ b/froide/foirequest/utils.py @@ -1,9 +1,11 @@ import datetime import json +import os import re import zipfile from dataclasses import dataclass from datetime import timedelta +from functools import partial from io import BytesIO from pathlib import PurePath from typing import TYPE_CHECKING, Iterator, List, Optional, Tuple, Union @@ -14,6 +16,7 @@ from django.core.files import File from django.core.mail import mail_managers from django.core.validators import validate_email +from django.db import transaction from django.template.loader import render_to_string from django.urls import reverse from django.utils import timezone @@ -998,3 +1001,55 @@ def find_attachment_name(name: str, message_id: int) -> str: ) ) return make_unique_filename(name, attachment_names) + + +def make_converted_attachment( + att: FoiAttachment, postfix: str, att_kwargs: dict = None +) -> FoiAttachment: + if att_kwargs is None: + att_kwargs = {} + name, ext = os.path.splitext(att.name) + name = "{name}_{postfix}{ext}".format(name=name, postfix=postfix, ext=ext) + name = find_attachment_name(name, att.belongs_to) + + return FoiAttachment( + name=name, + belongs_to=att.belongs_to, + is_converted=True, + can_approve=att.can_approve, + **att_kwargs, + ) + + +def make_decrypted_attachment(att: FoiAttachment, att_kwargs=None) -> FoiAttachment: + # Translators: filename postfix for decrypted file + postfix = _("decrypted") + if att_kwargs is None: + att_kwargs = {} + att_kwargs.setdefault("filetype", "application/pdf") + return make_converted_attachment(att, postfix, att_kwargs=att_kwargs) + + +def create_decrypted_attachment( + attachment: FoiAttachment, password: str, approved: bool = True +) -> FoiAttachment: + from .tasks import decrypt_pdf_attachment_task + + decrypted_att = make_decrypted_attachment( + attachment, + att_kwargs={ + "pending": True, + "approved": approved, + }, + ) + decrypted_att.save() + + transaction.on_commit( + partial( + decrypt_pdf_attachment_task.delay, + attachment.id, + decrypted_id=decrypted_att.id, + password=password, + ) + ) + return decrypted_att diff --git a/froide/foirequest/views/attachment.py b/froide/foirequest/views/attachment.py index 12b90add2..5b380f44b 100644 --- a/froide/foirequest/views/attachment.py +++ b/froide/foirequest/views/attachment.py @@ -15,6 +15,7 @@ from crossdomainmedia import CrossDomainMediaMixin +from froide.foirequest.utils import create_decrypted_attachment from froide.helper.auth import is_crew from froide.helper.utils import is_ajax, render_400, render_403 @@ -78,7 +79,22 @@ def approve_attachment(request, foirequest, attachment_id): # hard guard against publishing of non publishable requests if not foirequest.not_publishable: - att.approve_and_save() + if att.redacted: + # if there are redacted versions, delete them + # as we are approving the original + redacted = att.redacted + redacted.attachment_deleted.send( + sender=redacted, + user=request.user, + ) + redacted.remove_file_and_delete() + + if att.is_pdf and request.POST.get("password") is not None: + att = create_decrypted_attachment( + att, request.POST["password"], approved=True + ) + else: + att.approve_and_save() att.attachment_approved.send( sender=att, user=request.user, diff --git a/froide/helper/redaction.py b/froide/helper/redaction.py index 0922132de..5320715d4 100644 --- a/froide/helper/redaction.py +++ b/froide/helper/redaction.py @@ -77,11 +77,7 @@ def try_redacting_file(pdf_file, outpath, instructions): tries = 0 while True: try: - rewritten_pdf_file = rewrite_pdf(pdf_file, instructions) - if rewritten_pdf_file is None: - # Possibly encrypted with password, let's just try it anyway - rewritten_pdf_file = pdf_file - return _redact_file(rewritten_pdf_file, outpath, instructions) + return _redact_file(pdf_file, outpath, instructions) except PDFException as e: tries += 1 if tries > 2: diff --git a/frontend/javascript/components/redaction/pdf-redaction.vue b/frontend/javascript/components/redaction/pdf-redaction.vue index 61c2a0214..c3eb35600 100644 --- a/frontend/javascript/components/redaction/pdf-redaction.vue +++ b/frontend/javascript/components/redaction/pdf-redaction.vue @@ -193,28 +193,15 @@
-
+