Export by identity

Description

Extends the exporting features to export all submissions for an ID, globally or within an assignment.

Additions

• A new endpoint adding an identity filter to the already existing per-assignment submission endpoint
• A new endpoint accepting an identifier instead of an assignment ID to export everything linked to that identity
• A findByIdentifierEquals function for IdentityRepository

Changes

(in order of appearance)

• Renamed AssignmentExportController to SubmissionExportController since that reflects the meaning a bit better
• Assignment names are now filtered for all invalid filename characters instead of just quotes
• The assignment ID is also included in the export
• To generalize the code as much as possible, assignment and aid parameters are now nullable in many places
• The API docs table of contents is now scrollable

Test and Review

Closes #171 (closed)

To be filled in by the reviewers

• All of the methods are commented to expectation

• The methods are tested to satisfaction

• There are no unnecessary files present in the MR

• The continuous integration has no problems with the MR

• The MR is filled in as requested (including labels, milestones, and reviewers)

• The documentation is up-to-date

• All nullable parameters are marked as such

core/src/main/java/nl/tudelft/ewi/auta/core/controller/SubmissionExportController.java

@@ -13,6 +13,7 @@ import nl.tudelft.ewi.auta.core.model.Submission;
 import nl.tudelft.ewi.auta.core.response.exception.InvalidFormatException;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
+import org.jetbrains.annotations.Contract;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
@@ -34,18 +35,35 @@ import java.time.format.DateTimeFormatter;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Consumer;
+import java.util.regex.Pattern;
 
 /**
  * A controller exporting submissions for an assignment.
  */
 @Controller
-public class AssignmentExportController {
+public class SubmissionExportController {
     /**
      * The datetime formatter for timestamps included in filenames.
      */
     private static final DateTimeFormatter FILENAME_TIMESTAMP_FORMATTER =
             DateTimeFormatter.ofPattern("yyyy-MM-dd--HH-mm-ssxx");
 
+    /**
+     * A pattern matching characters that are forbidden in filenames on most common platforms.
+     */
+    private static final Pattern FORBIDDEN_FILENAME_CHARACTERS_PATTERN =
+            Pattern.compile("[<>:\"/\\\\|?*]");
+
+    /**
+     * The default output format.
+     */
+    private static final String DEFAULT_FORMAT = "json";
+
+    /**
+     * The default CSV flavor.
+     */
+    private static final String DEFAULT_CSV_FORMAT = "excel";
+
     /**
      * The header all CSV files are given.
      *
@@ -54,6 +72,7 @@ public class AssignmentExportController {
      * by JSON exports.
      */
     private static final String[] CSV_HEADER = {
+            "assignment",
             "identity",
             "submitted",
             "dispatched",
@@ -108,7 +127,7 @@ public class AssignmentExportController {
      * @param gson the GSON instance to serialize responses to JSON with
      * @param mongo the Mongo template instance complex queries are executed on
      */
-    public AssignmentExportController(
+    public SubmissionExportController(
             final Repositories repositories,
             final Gson gson,
             final MongoTemplate mongo
@@ -133,38 +152,141 @@ public class AssignmentExportController {
     @GetMapping("/api/v1/assignment/{aid}/export")
     public ResponseEntity<StreamingResponseBody> exportAllSubmissionsAction(
             final @PathVariable("aid") String aid,
-            final @RequestParam(value = "format", defaultValue = "json") String format,
-            final @RequestParam(value = "csvformat", defaultValue = "excel") String csvFormat
+            final @RequestParam(value = "format", defaultValue = DEFAULT_FORMAT) String format,
+            final @RequestParam(value = "csvformat", defaultValue = DEFAULT_CSV_FORMAT) String
+                    csvFormat
+    ) {
+        return this.runExport(aid, null, format, csvFormat);
+    }
+
+    /**
+     * Exports all submissions for an assignment by a given identity.
+     *
+     * @param aid the ID of the assignment to export for
+     * @param iid the identity to filter by
+     * @param format the file format to output
+     * @param csvFormat the CSV flavor to output if the format is CSV
+     *
+     * @return a response streaming the exported submissions
+     *
+     * @see #exportAllSubmissionsAction(String, String, String) for details
+     */
+    @GetMapping("/api/v1/assignment/{aid}/export/{iid}")
+    public ResponseEntity<StreamingResponseBody> exportAllSubmissionsInAssignmentForIdentityAction(
+            final @PathVariable("aid") String aid,
+            final @PathVariable("iid") String iid,
+            final @RequestParam(value = "format", defaultValue = DEFAULT_FORMAT) String format,
+            final @RequestParam(value = "csvformat", defaultValue = DEFAULT_CSV_FORMAT) String
+                    csvFormat
+    ) {
+        return this.runExport(aid, iid, format, csvFormat);
+    }
+
+    /**
+     * Exports all submissions by a given identity.
+     *
+     * @param iid the identity to filter by
+     * @param format the file format to output
+     * @param csvFormat the CSV flavor to output if the format is CSV
+     *
+     * @return a response streaming the exported submissions
+     */
+    @GetMapping("/api/v1/identity/{iid}/export")
+    public ResponseEntity<StreamingResponseBody> exportAllSubmissionsForIdentityAction(
+            final @PathVariable("iid") String iid,
+            final @RequestParam(value = "format", defaultValue = DEFAULT_FORMAT) String format,
+            final @RequestParam(value = "csvformat", defaultValue = DEFAULT_CSV_FORMAT) String
+                    csvFormat
+    ) {
+        return this.runExport(null, iid, format, csvFormat);
+    }
+
+    /**
+     * Runs an export.
+     *
+     * @param aid the ID of the assignment to export for
+     * @param iid the identity to filter on, if any
+     * @param format the file format to output
+     * @param csvFormat the CSV flavor to use when exporting CSV
+     *
+     * @return a response streaming the exported submissions
+     */
+    private ResponseEntity<StreamingResponseBody> runExport(
+            final @Nullable String aid,
+            final @Nullable String iid,
+            final String format,
+            final String csvFormat
     ) {
-        final var assignmentRepository = this.repositories.getAssignmentRepository();
-        final var assignment = assignmentRepository.findExisting(aid);
+        @Nullable
+        final var assignment = this.findAssignment(aid);
 
         switch (format.toLowerCase()) {
             case "json":
-                return this.exportAllSubmissionsAsJsonAction(assignment);
+                return this.exportAllSubmissionsAsJsonAction(assignment, iid);
             case "csv":
-                return this.exportAllSubmissionsAsCsvAction(assignment, csvFormat);
+                return this.exportAllSubmissionsAsCsvAction(assignment, iid, csvFormat);
             default:
                 throw new InvalidFormatException(format + " is not a known export format");
         }
     }
 
+    /**
+     * Returns the assignment with the given ID.
+     *
+     * @param aid the assignment to look for
+     *
+     * @return the assignment or {@code null} if the ID was {@code null}
+     *
+     * @throws nl.tudelft.ewi.auta.core.response.exception.NoSuchAssignmentException if the
+     *         assignment does not exist
+     */
+    @Contract("null -> null; !null -> !null")
+    @Nullable
+    private Assignment findAssignment(final @Nullable String aid) {
+        if (aid != null) {
+            final var assignmentRepository = this.repositories.getAssignmentRepository();
+            return assignmentRepository.findExisting(aid);
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * Returns the ID of an assignment.
+     *
+     * @param assignment the assignment to get the ID from
+     *
+     * @return the ID or {@code null} if the assignment was {@code null}
+     */
+    @Contract("null -> null; !null -> !null")
+    @Nullable
+    private String getAssignmentId(final @Nullable Assignment assignment) {
+        if (assignment == null) {
+            return null;
+        } else {
+            return assignment.getId();
+        }
+    }
+
     /**
      * Exports all submissions as a JSON array.
      *
      * @param assignment the assignment to export for
+     * @param iid the identity the export is for
      *
      * @return a response streaming JSON
      */
     private ResponseEntity<StreamingResponseBody> exportAllSubmissionsAsJsonAction(
-            final Assignment assignment
+            final @Nullable Assignment assignment, final @Nullable String iid
     ) {
-        return this.generateDownloadResponse(assignment, "full", "json", out -> {
+        final var kind = this.getSubmissionKind(iid);
+
+        return this.generateDownloadResponse(assignment, kind, "json", out -> {
             final var writer = this.gson.newJsonWriter(
                     new OutputStreamWriter(out, StandardCharsets.UTF_8)
             ).beginArray();
 
-            this.exportToJsonArrayElements(assignment.getId(), writer);
+            this.exportToJsonArrayElements(this.getAssignmentId(assignment), iid, writer);
 
             // noinspection resource: Closing would close the stream, which is not ours to close
             writer.endArray().flush();
@@ -175,20 +297,35 @@ public class AssignmentExportController {
      * Exports all submissions as elements of a JSON array.
      *
      * @param aid the assignment to export for
+     * @param iid the identity the export is for
      * @param out the JSON writer to write elements to
      */
-    private void exportToJsonArrayElements(final String aid, final JsonWriter out) {
+    private void exportToJsonArrayElements(
+            final @Nullable String aid, final @Nullable String iid, final JsonWriter out
+    ) {
         this.forEachSubmission(
                 aid,
-                submission -> this.gson.toJson(this.writeSubmissionToJson(submission), out)
+                iid,
+                submission -> this.gson.toJson(this.writeSubmissionToJson(submission, iid), out)
         );
     }
 
-    private JsonObject writeSubmissionToJson(final Submission submission) {
+    /**
+     * Writes a single submission to JSON.
+     *
+     * @param submission the submission to write
+     * @param iid the identity the submission is from, if any
+     *
+     * @return the serialized JSON object
+     */
+    private JsonObject writeSubmissionToJson(
+            final Submission submission, final @Nullable String iid
+    ) {
         final var log = submission.getPipelineLog();
 
         final var obj = new JsonObject();
-        obj.addProperty("identity", this.getIdentity(submission));
+        obj.addProperty("assignment", submission.getAssignmentId());
+        obj.addProperty("identity", this.getIdentity(submission, iid));
         obj.addProperty("submitted", this.instantToString(log.getSubmitted()));
         obj.addProperty("dispatched", this.instantToString(log.getDispatched()));
         obj.addProperty("analyzed", this.instantToString(log.getAnalysisDone()));
@@ -235,25 +372,34 @@ public class AssignmentExportController {
      * in a way humans can understand.
      *
      * @param assignment the assignment to export for
+     * @param iid the identity the export is for, if any
      * @param csvFormat the specific CSV format to export
      *
      * @return a response streaming CSV
      */
     private ResponseEntity<StreamingResponseBody> exportAllSubmissionsAsCsvAction(
-            final Assignment assignment, final String csvFormat
+            final @Nullable Assignment assignment,
+            final @Nullable String iid,
+            final String csvFormat
     ) {
+        @Nullable
+        final String aid = this.getAssignmentId(assignment);
+
+        final var kind = this.getSubmissionKind(iid);
 
         final var format = CSV_FORMAT_MAP.get(csvFormat.toLowerCase());
         if (format == null) {
             throw new InvalidFormatException(csvFormat + " is not a known CSV format");
         }
 
-        return this.generateDownloadResponse(assignment, "full", "csv", out -> {
+        return this.generateDownloadResponse(assignment, kind, "csv", out -> {
             try (var writer = format.withHeader(CSV_HEADER).print(
                     new OutputStreamWriter(out, StandardCharsets.UTF_8)
             )) {
-                this.forEachSubmission(assignment.getId(),
-                        s -> this.writeSubmissionToCsv(s, writer)
+                this.forEachSubmission(
+                        aid,
+                        iid,
+                        s -> this.writeSubmissionToCsv(s, iid, writer)
                 );
             }
         });
@@ -266,17 +412,24 @@ public class AssignmentExportController {
      * do not match up, an exception is thrown.
      *
      * @param submission the submission write
+     * @param iid the identity the submission is from, if {@code null} a database lookup
+     *            is performed
      * @param writer the writer to write it to
      *
      * @throws AssertionError if the length of the header and the length of the record are different
      * @throws CheckedExceptionTunnel&lt;IOException&gt; if an I/O error occurs
      */
-    private void writeSubmissionToCsv(final Submission submission, final CSVPrinter writer) {
+    private void writeSubmissionToCsv(
+            final Submission submission,
+            final @Nullable String iid,
+            final CSVPrinter writer
+    ) {
         final var log = submission.getPipelineLog();
 
         try {
             final var record = new Object[] {
-                    this.getIdentity(submission),
+                    submission.getAssignmentId(),
+                    this.getIdentity(submission, iid),
                     log.getSubmitted(),
                     log.getDispatched(),
                     log.getAnalysisDone(),
@@ -300,11 +453,16 @@ public class AssignmentExportController {
      * Returns the identity associated with the submission.
      *
      * @param submission the submission to get the identity for
+     * @param iid if not {@code null}, this string is returned and no database lookups are performed
      *
      * @return the identity ID or {@code null} if the submission was anonymous
      */
     @Nullable
-    private String getIdentity(final Submission submission) {
+    private String getIdentity(final Submission submission, final @Nullable String iid) {
+        if (iid != null) {
+            return iid;
+        }
+
         final var identityRepository = this.repositories.getIdentityRepository();
         return identityRepository.findById(submission.getId())
                        .map(IdentityContainer::getIdentifier)
@@ -328,21 +486,89 @@ public class AssignmentExportController {
                 .toString();
     }
 
+    /**
+     * Executes a lambda for each submission.
+     *
+     * @param aid the assignment to get the submissions of
+     * @param iid the identity to filter by, if any
+     * @param consumer the consumer taking submissions
+     */
+    private void forEachSubmission(
+            final @Nullable String aid,
+            final @Nullable String iid,
+            final Consumer<Submission> consumer
+    ) {
+        assert aid != null || iid != null
+                : "aid = null -> iid != null violated, don't know what to export";
+
+        if (aid == null) {
+            this.forEachSubmissionByIdentity(iid, consumer);
+        } else if (iid == null) {
+            this.forEachSubmissionInAssignment(aid, consumer);
+        } else {
+            this.forEachSubmissionInAssignmentByIdentity(aid, iid, consumer);
+        }
+    }
+
     /**
      * Executes a lambda for each submission of an assignment.
      *
      * @param aid the assignment to get the submissions of
      * @param consumer the consumer taking submissions
      */
-    private void forEachSubmission(final String aid, final Consumer<Submission> consumer) {
-        try (var submissions = this.mongo.stream(
-                new Query(Criteria.where("assignmentId").is(aid)),
-                Submission.class
-        )) {
+    private void forEachSubmissionInAssignment(
+            final String aid,
+            final Consumer<Submission> consumer
+    ) {
+        final var query = new Query(Criteria.where("assignmentId").is(aid));
+
+        try (var submissions = this.mongo.stream(query, Submission.class)) {
             submissions.forEachRemaining(consumer);
         }
     }
 
+    /**
+     * Executes a lambda for each submission for an assignment by an identity.
+     *
+     * @param aid the assignment to get the submissions of
+     * @param iid the identity to filter by
+     * @param consumer the consumer taking submissions
+     */
+    private void forEachSubmissionInAssignmentByIdentity(
+            final String aid,
+            final String iid,
+            final Consumer<Submission> consumer
+    ) {
+        final var submissionRepository = this.repositories.getSubmissionRepository();
+        final var query = new Query(Criteria.where("identifier").is(iid));
+        try (var links = this.mongo.stream(query, IdentityContainer.class)) {
+            links.forEachRemaining(ic -> submissionRepository
+                    .findByAssignmentIdEqualsAndIdEquals(aid, ic.getSubmissionId())
+                    .ifPresent(consumer)
+            );
+        }
+    }
+
+    /**
+     * Executes a lambda for each submission by an identity.
+     *
+     * @param iid the identity to export for
+     * @param consumer the consumer taking submissions
+     */
+    private void forEachSubmissionByIdentity(
+            final String iid,
+            final Consumer<Submission> consumer
+    ) {
+        final var submissionRepository = this.repositories.getSubmissionRepository();
+        final var query = new Query(Criteria.where("identifier").is(iid));
+        try (var links = this.mongo.stream(query, IdentityContainer.class)) {
+            links.forEachRemaining(ic -> submissionRepository
+                    .findById(ic.getSubmissionId())
+                    .ifPresent(consumer)
+            );
+        }
+    }
+
     /**
      * Creates a response entity meant for downloading (as opposed to in-browser viewing).
      *
@@ -357,7 +583,7 @@ public class AssignmentExportController {
      * @return the generated response entity
      */
     private ResponseEntity<StreamingResponseBody> generateDownloadResponse(
-            final Assignment assignment,
+            final @Nullable Assignment assignment,
             final String kind,
             final String format,
             final StreamingResponseBody generator
@@ -369,6 +595,23 @@ public class AssignmentExportController {
                        .body(generator);
     }
 
+    /**
+     * Generates an export kind for the given export settings.
+     *
+     * If no restrictions are set, {@code "full"} is returned.
+     *
+     * @param iid the identity the export is for, if any
+     *
+     * @return the kind
+     */
+    private String getSubmissionKind(final @Nullable String iid) {
+        if (iid == null) {
+            return "full";
+        }
+
+        return "for-" + iid;
+    }
+
     /**
      * Generates a filename for the exported file.
      *
@@ -386,10 +629,17 @@ public class AssignmentExportController {
      * @return the name
      */
     private String generateFileName(
-            final Assignment assignment, final String kind, final String format
+            final @Nullable Assignment assignment, final String kind, final String format
     ) {
-        return assignment.getName().replace('"', '-')
-                       + "-export"
+        final String prefix;
+        if (assignment != null) {
+            prefix = FORBIDDEN_FILENAME_CHARACTERS_PATTERN.matcher(assignment.getName())
+                    .replaceAll("-") + '-';
+        } else {
+            prefix = "";
+        }
+
+        return prefix + "export"
                        + '-' + kind
                        + '-' + ZonedDateTime.now().format(FILENAME_TIMESTAMP_FORMATTER)
                        + '.' + format;