Merge branch 'development' into 'master'

Release 2.3.2

See merge request !161

.gitlab-ci.yml

@@ -69,64 +69,64 @@ variables:
 stages:
   - build
   - test
-  - review
+#  - review
 #  - dast
-  - staging
-  - canary
+#  - staging
+#  - canary
   - jar
   - publish
-  - production
-  - incremental rollout 10%
-  - incremental rollout 25%
-  - incremental rollout 50%
-  - incremental rollout 100%
-  - performance
-  - cleanup
+#  - production
+#  - incremental rollout 10%
+#  - incremental rollout 25%
+#  - incremental rollout 50%
+#  - incremental rollout 100%
+#  - performance
+#  - cleanup
 
 include:
-  - template: Jobs/Build.gitlab-ci.yml
-  - template: Jobs/Test.gitlab-ci.yml
+#  - template: Jobs/Build.gitlab-ci.yml
+#  - template: Jobs/Test.gitlab-ci.yml
 #  - template: Jobs/Code-Quality.gitlab-ci.yml
-  - template: Jobs/Deploy.gitlab-ci.yml
-  - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml
+#  - template: Jobs/Deploy.gitlab-ci.yml
+#  - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml
 #  - template: Security/DAST.gitlab-ci.yml #re-enable when testing website
 #  - template: Security/Container-Scanning.gitlab-ci.yml
 #  - template: Security/Dependency-Scanning.gitlab-ci.yml #Wait for GitLab 12.0
-  - template: License-Management.gitlab-ci.yml
+#  - template: License-Management.gitlab-ci.yml
 #  - template: Security/SAST.gitlab-ci.yml
 
-trampoline:
-  image:
-    name: "timvanderhorst/trampoline:auta"
-    entrypoint: [""]
-  stage: test
-  allow_failure: true
-  script:
-    - cd /home/trampoline ; python3 run.py
-  artifacts:
-    paths:
-      - report.json
-      - report.html
-    expire_in: 1 week
-
-trampoline-feedback:
-  image:
-    name: "timvanderhorst/trampoline:auta"
-    entrypoint: [""]
-  variables:
-    AUTA_ASSIGNMENT_ID: 5cda623347fc2b00068befa1
-  stage: test
-  allow_failure: true
-  script:
-    - cd /home/trampoline ; python3 run.py
-  artifacts:
-    paths:
-      - report.json
-      - report.html
-      - benchmark_CYCLOMATIC_COMPLEXITY_METHOD.html
-      - benchmark_METHOD_EFFECTIVE_LOC_METHOD.html
-      - benchmark_PARAMETER_COUNT_METHOD.html
-    expire_in: 1 week
+#trampoline:
+#  image:
+#    name: "timvanderhorst/trampoline:auta"
+#    entrypoint: [""]
+#  stage: test
+#  allow_failure: true
+#  script:
+#    - cd /home/trampoline ; python3 run.py
+#  artifacts:
+#    paths:
+#      - report.json
+#      - report.html
+#    expire_in: 1 week
+#
+#trampoline-feedback:
+#  image:
+#    name: "timvanderhorst/trampoline:auta"
+#    entrypoint: [""]
+#  variables:
+#    AUTA_ASSIGNMENT_ID: 5cda623347fc2b00068befa1
+#  stage: test
+#  allow_failure: true
+#  script:
+#    - cd /home/trampoline ; python3 run.py
+#  artifacts:
+#    paths:
+#      - report.json
+#      - report.html
+#      - benchmark_CYCLOMATIC_COMPLEXITY_METHOD.html
+#      - benchmark_METHOD_EFFECTIVE_LOC_METHOD.html
+#      - benchmark_PARAMETER_COUNT_METHOD.html
+#    expire_in: 1 week
 
 checkstyle:
 #  image: $CI_REGISTRY/$CI_PROJECT_PATH/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHA
@@ -165,11 +165,12 @@ jar-worker:
     expire_in: 1h
 
 publish-core:
-  image: registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable
+  image: docker:19.03.13
   variables:
     DOCKER_TLS_CERTDIR: ""
+    DOCKER_HOST: tcp://docker:2375
   services:
-    - docker:19.03.5-dind
+    - docker:19.03.13-dind
   only:
     - branches
   stage: publish
@@ -177,14 +178,15 @@ publish-core:
     - jar-core
   script:
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - devops/build-slim-image core $CI_COMMIT_REF_NAME
+    - sh devops/build-slim-image core $CI_COMMIT_REF_NAME
 
 publish-worker:
-  image: registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable
+  image: docker:19.03.13
   variables:
     DOCKER_TLS_CERTDIR: ""
+    DOCKER_HOST: tcp://docker:2375
   services:
-    - docker:19.03.5-dind
+    - docker:19.03.13-dind
   only:
     - branches
   stage: publish
@@ -192,10 +194,11 @@ publish-worker:
     - jar-worker
   script:
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - devops/build-slim-image worker $CI_COMMIT_REF_NAME
+    - sh devops/build-slim-image worker $CI_COMMIT_REF_NAME
 
 # Override DAST job to exclude master branch
 #dast:
 #  except:
 #    refs:
 #      - master
+

build.gradle

@@ -26,7 +26,7 @@ allprojects {
     apply plugin: ScompPlugin
 
     group 'nl.tudelft.ewi'
-    version = '2.3.1'
+    version = '2.3.2'
 
     sourceCompatibility = '1.11'
     targetCompatibility = '1.11'

core/src/main/java/nl/tudelft/ewi/auta/core/authentication/TokenAuthenticationFilter.java

@@ -2,7 +2,12 @@ package nl.tudelft.ewi.auta.core.authentication;
 
 import java.io.IOException;
 import java.sql.SQLException;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
+import javax.annotation.Nullable;
 import javax.servlet.FilterChain;
 import javax.servlet.ServletException;
 import javax.servlet.ServletRequest;
@@ -10,9 +15,11 @@ import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServletRequest;
 
 import nl.tudelft.ewi.auta.core.authentication.database.DatabaseConnector;
+import nl.tudelft.ewi.auta.core.response.exception.InvalidTokenException;
 import nl.tudelft.ewi.auta.core.response.exception.MissingUserException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpHeaders;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Service;
 import org.springframework.web.filter.GenericFilterBean;
@@ -24,6 +31,11 @@ import org.springframework.web.filter.GenericFilterBean;
 public class TokenAuthenticationFilter extends GenericFilterBean {
     private static final Logger logger = LoggerFactory.getLogger(TokenAuthenticationFilter.class);
 
+    /**
+     * The prefix of AuTA authorization headers.
+     */
+    private static final String AUTH_PREFIX = "AutaToken ";
+
     /**
      * The authentication database to query.
      */
@@ -53,8 +65,13 @@ public class TokenAuthenticationFilter extends GenericFilterBean {
             final FilterChain chain)
             throws IOException, ServletException {
         var httpRequest = (HttpServletRequest) request;
-        var token = httpRequest.getHeader("Auth-Token");
-        if (token == null && request.getParameter("auth-token") != null) {
+
+        @Nullable
+        var token = this.getAuthToken(httpRequest);
+        if (token == null) {
+            token = httpRequest.getHeader("Auth-Token");
+        }
+        if (token == null) {
             token = request.getParameter("auth-token");
         }
 
@@ -65,6 +82,38 @@ public class TokenAuthenticationFilter extends GenericFilterBean {
         chain.doFilter(request, response);
     }
 
+    /**
+     * Returns the authentication token of the Authorization header.
+     *
+     * This only recognizes the Auta-Token type.
+     *
+     * @param req the request containing the Authorization header
+     *
+     * @return the token, or {@code null} if no token is present
+     *
+     * @throws InvalidTokenException if more than one token was present
+     */
+    @Nullable
+    private String getAuthToken(final HttpServletRequest req) {
+        final var tokens = StreamSupport.stream(
+                Spliterators.spliteratorUnknownSize(
+                        req.getHeaders(HttpHeaders.AUTHORIZATION).asIterator(), Spliterator.ORDERED
+                ), false
+        ).filter(h -> h.startsWith(AUTH_PREFIX))
+         .map(h -> h.substring(AUTH_PREFIX.length()))
+         .collect(Collectors.toList());
+
+        if (tokens.isEmpty()) {
+            return null;
+        }
+
+        if (tokens.size() > 1) {
+            throw new InvalidTokenException("More than one token was passed");
+        }
+
+        return tokens.get(0);
+    }
+
     /**
      * Sets the token for the current session.
      * @param token the token string

core/src/main/java/nl/tudelft/ewi/auta/core/controller/SubmissionController.java

@@ -12,7 +12,7 @@ import nl.tudelft.ewi.auta.core.model.Submission;
 import nl.tudelft.ewi.auta.core.report.HtmlReportGenerator;
 import nl.tudelft.ewi.auta.core.report.ReportAuthorizationFilter;
 import nl.tudelft.ewi.auta.core.response.Response;
-import nl.tudelft.ewi.auta.core.response.exception.InvalidFileTypeException;
+import nl.tudelft.ewi.auta.core.response.exception.InvalidFileNameException;
 import nl.tudelft.ewi.auta.core.response.exception.InvalidSubmissionNameException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -341,7 +341,7 @@ public class SubmissionController extends ControllerBase {
 
 
     /**
-     * Gets the extension of a filename
+     * Gets the extension of a filename.
      *
      * @param filename the filename
      * @return the extension of the file
@@ -376,28 +376,37 @@ public class SubmissionController extends ControllerBase {
             final @PathVariable String aid,
             final @RequestParam(value = "file") MultipartFile file,
             final @RequestParam String directUploadId,
+            final @Nullable @RequestParam(value = "identity", required = false) String reqIdentity,
             final @Nullable @RequestParam(value = "identifier", required = false) String identifier
     ) throws IOException, URISyntaxException {
+        @Nullable
+        final String identity;
+        if (reqIdentity != null) {
+            identity = reqIdentity;
+        } else {
+            identity = identifier;
+        }
+
         final var res = new Response();
         //final var commit = this.getString(req, "commit", res);
-        logger.debug("Received trampoline message, assignment id {}, directUploadId {}, "
-                        + "identifier {}", aid, directUploadId, identifier);
+        logger.debug("Received direct upload, assignment id {}, directUploadId {}, "
+                        + "identity {}", aid, directUploadId, identity);
 
-        // Check if the original filename has an extension
+        @Nullable
         final var originalFilename = file.getOriginalFilename();
-        if (originalFilename == null || this.getExtension(originalFilename).isEmpty()) {
-            throw new InvalidFileTypeException("File has no extension: " + originalFilename);
+        if (originalFilename == null || originalFilename.isEmpty()) {
+            throw new InvalidFileNameException("The file has no name");
         }
+
         final var submissionRepository = this.repositories.getSubmissionRepository();
         final var assignmentRepository = this.repositories.getAssignmentRepository();
 
         // Delete data associated with submission that has name direct-upload:COMMIT_HASH
-        final var newSubmissionName = "direct-upload:" + directUploadId;
         var assignment = assignmentRepository.findExisting(aid);
-        this.deleteSubmissionData(newSubmissionName, aid);
+        this.deleteSubmissionData(directUploadId, aid);
 
         // Move file in request to the temp folders directory.
-        final String extension = this.getExtension(originalFilename);
+        final var extension = this.getExtension(originalFilename);
         final String filename = directUploadId + extension;
 
         final var temp = Files.createTempFile("auta-upload-", extension);
@@ -407,7 +416,7 @@ public class SubmissionController extends ControllerBase {
         // Create a new submission with the new name, add to assignment and identityContainer and
         // add to queue.
         var submission = new Submission();
-        submission.setName(newSubmissionName);
+        submission.setName(directUploadId);
         submission.setAssignmentId(aid);
         submission.setContents("/" + filename);
         submission.getPipelineLog().setSubmitted(Instant.now());
@@ -420,12 +429,12 @@ public class SubmissionController extends ControllerBase {
         assignment = assignmentRepository.save(assignment);
         logger.debug("Created new submission for direct-upload submission, submission id {}", sid);
 
-        if (identifier == null) {
+        if (identity == null) {
             logger.warn("Submission id {} does not belong to an identity", sid);
         } else {
-            final var identityContainer = new IdentityContainer(sid, identifier);
+            final var identityContainer = new IdentityContainer(sid, identity);
             this.repositories.getIdentityRepository().save(identityContainer);
-            logger.debug("Submission id {} belongs to {}", sid, identifier);
+            logger.debug("Submission id {} belongs to {}", sid, identity);
         }
 
         this.queue.add(new Job(assignment, submission));

core/src/main/java/nl/tudelft/ewi/auta/core/plugin/PluginLoader.java

@@ -22,6 +22,7 @@ import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.stream.Collectors;
 
 /**
@@ -92,6 +93,11 @@ public class PluginLoader {
                 this.settings.get("plugins directory", String.class)
         );
 
+        if (Files.notExists(dir)) {
+            logger.warn("Plugins directory {} does not exist, skipping plugins", dir);
+            return Collections.emptyList();
+        }
+
         // Find all loadable jars in that directory
         final Collection<URL> urls;
         try (var files = Files.list(dir)) {

core/src/main/java/nl/tudelft/ewi/auta/core/response/ErrorCode.java

@@ -173,7 +173,12 @@ public enum ErrorCode {
     /**
      * There is no worker with the given name.
      */
-    NO_SUCH_WORKER(HttpStatus.NOT_FOUND);
+    NO_SUCH_WORKER(HttpStatus.NOT_FOUND),
+
+    /**
+     * The submitted file has an invalid name.
+     */
+    INVALID_FILE_NAME(HttpStatus.UNPROCESSABLE_ENTITY);
 
     /**
      * The associated HTTP status code.

core/src/main/java/nl/tudelft/ewi/auta/core/response/exception/InvalidFileNameException.java

+package nl.tudelft.ewi.auta.core.response.exception;
+
+import nl.tudelft.ewi.auta.core.response.ErrorCode;
+
+/**
+ * An exception indicating that analysis was requested for a file with no or otherwise invalid name.
+ */
+public class InvalidFileNameException extends ApiException {
+    /**
+     * Creates a new invalid file name exception.
+     */
+    public InvalidFileNameException() {
+        super();
+    }
+
+    /**
+     * Creates a new invalid file name exception.
+     *
+     * @param message the message explaining what caused the exception
+     */
+    public InvalidFileNameException(final String message) {
+        super(message);
+    }
+
+    /**
+     * Creates a new invalid file name exception.
+     *
+     * @param cause the exception that caused this exception
+     */
+    public InvalidFileNameException(final Throwable cause) {
+        super(cause);
+    }
+
+    /**
+     * Creates a new invalid file name exception.
+     *
+     * @param message the message explaining what caused the exception
+     * @param cause   the exception that caused this exception
+     */
+    public InvalidFileNameException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * Returns the error code that represents the exception.
+     *
+     * @return the error code
+     */
+    @Override
+    public ErrorCode getErrorCode() {
+        return ErrorCode.INVALID_FILE_NAME;
+    }
+}

core/src/main/java/nl/tudelft/ewi/auta/core/response/exception/InvalidTokenException.java

+package nl.tudelft.ewi.auta.core.response.exception;
+
+import nl.tudelft.ewi.auta.core.response.ErrorCode;
+
+public class InvalidTokenException extends ApiException {
+    /**
+     * Creates a new invalid token exception.
+     */
+    public InvalidTokenException() {
+        super();
+    }
+
+    /**
+     * Creates a new invalid token exception.
+     *
+     * @param message the message explaining what caused the exception
+     */
+    public InvalidTokenException(final String message) {
+        super(message);
+    }
+
+    /**
+     * Creates a new invalid token exception.
+     *
+     * @param cause the exception that caused this exception
+     */
+    public InvalidTokenException(final Throwable cause) {
+        super(cause);
+    }
+
+    /**
+     * Creates a new invalid token exception.
+     *
+     * @param message the message explaining what caused the exception
+     * @param cause   the exception that caused this exception
+     */
+    public InvalidTokenException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+
+    @Override
+    public ErrorCode getErrorCode() {
+        return ErrorCode.INVALID_TOKEN;
+    }
+}

core/src/test/java/nl/tudelft/ewi/auta/core/controller/SubmissionControllerTest.java

@@ -374,8 +374,8 @@ public class SubmissionControllerTest {
     }
 
     @Test
-    public void testDirectUploadBadFilename() throws Exception {
-        var mockMultipartFile = new MockMultipartFile("file", "bad-filename",
+    public void testDirectUploadWithoutExtension() throws Exception {
+        var mockMultipartFile = new MockMultipartFile("file", "no-extension",
                 null, "contents".getBytes());
 
         var builder = MockMvcRequestBuilders.multipart(
@@ -383,8 +383,7 @@ public class SubmissionControllerTest {
                         + "/direct-upload"
         ).file(mockMultipartFile)
                 .param("directUploadId", "commit_id");
-        this.mvc.perform(builder).andExpect(status().isUnprocessableEntity());
-        Mockito.verifyZeroInteractions(this.fileStore);
+        this.mvc.perform(builder).andExpect(status().isCreated());
     }
 
     @Test
@@ -404,7 +403,7 @@ public class SubmissionControllerTest {
 
     @Test
     public void testUploadNormalFile() throws Exception {
-        Mockito.when(this.submissionStore.findByNameAndAssignment("direct-upload:commit_id",
+        Mockito.when(this.submissionStore.findByNameAndAssignment("commit_id",
                 UNIQUE_ASSIGNMENT_ID))
                 .thenReturn(List.of(this.submission));
         var mockMultipartFile = new MockMultipartFile("file", "submission.zip",

worker/src/main/java/nl/tudelft/ewi/auta/worker/tool/python/PythonDetector.java

@@ -166,7 +166,7 @@ public class PythonDetector {
         final var matcher = VERSION_PATTERN.matcher(ver);
         if (!matcher.matches()) {
             logger.warn("{} does not match version pattern {}, skipping",
-                    VERSION_PATTERN.pattern(), ver
+                    ver, VERSION_PATTERN.pattern()
             );
             return null;
         }