Merge branch 'docker-resuscitation' into 'development'

Docker resuscitation

See merge request eip/code-measures!105

core/src/main/java/nl/tudelft/ewi/auta/core/controller/AssignmentController.java

@@ -203,12 +203,24 @@ public class AssignmentController extends ControllerBase {
 
         assignment.setName(name);
         assignment.setAllowedLanguages(Collections.singleton(lang));
-        assignment.setMetricSettings(stat.stream().map(o -> new MetricSettings(
+        assignment.setMetricSettings(stat.stream().map(o -> {
+            if (MetricName.valueOf((String) o.get("name")) == MetricName.DOCKER_LOGS) {
+                return new MetricSettings(
+                        MetricName.valueOf((String) o.get("name")),
+                        (String) o.get("script"),
+                        (String) o.get("formatter"),
+                        0,
+                        0,
+                        (String) o.get("dockerfile")
+                );
+            }
+            return new MetricSettings(
                         MetricName.valueOf((String) o.get("name")),
                         (String) o.get("script"),
                         (String) o.get("formatter"),
                         0,
                         0
-        )).collect(Collectors.toUnmodifiableSet()));
+                );
+        }).collect(Collectors.toUnmodifiableSet()));
     }
 }

src/main/java/nl/tudelft/ewi/auta/common/model/metric/MetricSettings.java

@@ -44,6 +44,13 @@ public class MetricSettings {
      */
     private Integer maxFailures = 0;
 
+    /**
+     * A dockerfile.
+     */
+    @Language("Dockerfile")
+    @Nullable
+    private String dockerfile;
+
     /**
      * Creates a new set of metric settings.
      *
@@ -71,6 +78,37 @@ public class MetricSettings {
         }
     }
 
+    /**
+     * Creates a new set of metric settings.
+     *
+     * @param metric the name of the metric the settings apply to
+     * @param passingScript the script used to check the results
+     * @param formattingScript the script used to format the results
+     * @param maxWarnings the maximum number of warnings
+     * @param maxFailures the maximum number of failures
+     * @param dockerfile a dockerfile
+     */
+    public MetricSettings(
+            final @Nullable MetricName metric,
+            final @Nullable @Language("ECMAScript 6") String passingScript,
+            final @Nullable @Language("ECMAScript 6") String formattingScript,
+            final @Nullable Integer maxWarnings,
+            final @Nullable Integer maxFailures,
+            final @Nullable @Language("Dockerfile") String dockerfile
+    ) {
+        this.metric = metric;
+        this.passingScript = passingScript;
+        this.formattingScript = formattingScript;
+        if (maxWarnings != null) {
+            this.maxWarnings = maxWarnings;
+        }
+        if (maxFailures != null) {
+            this.maxFailures = maxFailures;
+        }
+        this.dockerfile = dockerfile;
+    }
+
+
     /**
      * Creates a new set of metric settings.
      *
@@ -168,6 +206,17 @@ public class MetricSettings {
         return this.maxFailures;
     }
 
+    /**
+     * Returns this metric's dockerfile.
+     *
+     * @return a dockerfile
+     */
+    @Language("Dockerfile")
+    @Nullable
+    public String getDockerfile() {
+        return this.dockerfile;
+    }
+
     @Override
     public final boolean equals(final Object obj) {
         if (!(obj instanceof MetricSettings)) {
@@ -180,13 +229,14 @@ public class MetricSettings {
                 && Objects.equals(this.passingScript, other.passingScript)
                 && Objects.equals(this.formattingScript, other.formattingScript)
                 && Objects.equals(this.maxFailures, other.maxFailures)
-                && Objects.equals(this.maxWarnings, other.maxWarnings);
+                && Objects.equals(this.maxWarnings, other.maxWarnings)
+                && Objects.equals(this.dockerfile, other.dockerfile);
     }
 
     @Override
     public final int hashCode() {
         return Objects.hash(this.metric, this.passingScript, this.formattingScript,
-                this.maxWarnings, this.maxFailures
+                this.maxWarnings, this.maxFailures, this.dockerfile
         );
     }
 }

worker/src/main/java/nl/tudelft/ewi/auta/checker/generic/docker/DockerRunner.java

@@ -21,7 +21,6 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -72,7 +71,7 @@ public class DockerRunner extends JobAnalyzer {
         var dfArchive = this.archiveDockerfile(victim);
         this.createImage(victim, dfArchive);
 
-        var imageName = (String) victim.getOptions().get("assignment name");
+        var imageName = (String) victim.getId();
         var logs = this.start(this.createContainer(victim, imageName.toLowerCase()));
 
         logger.info("Retrieved logs from container!");
@@ -120,7 +119,6 @@ public class DockerRunner extends JobAnalyzer {
         config.put("HostConfig", Map.of("Binds",
                 List.of(job.getDir().toAbsolutePath() + ":/var/auta/submission")
                 ));
-        config.put("WorkingDir", "/var/auta/submission");
         final var container = this.api.createContainer(config);
 
         logger.info("Created container with ID {}", container);
@@ -134,7 +132,7 @@ public class DockerRunner extends JobAnalyzer {
      * @param job the job for this image
      */
     private void createImage(final Job job, final File file) {
-        var name = (String) job.getOptions().get("assignment name");
+        var name = job.getId();
         var imageName = name.toLowerCase();
         logger.info("Creating new image for assignment {}.", name);
 
@@ -162,10 +160,12 @@ public class DockerRunner extends JobAnalyzer {
      * @throws IOException on exceptions when writing to the archive
      */
     private File archiveDockerfile(final Job job) throws IOException {
-        var dynamic = (ArrayList<Map>) job.getOptions().get("dynamic");
-        var df = dynamic.stream().filter(elem -> elem.containsKey("dockerfile")).findFirst();
+        var df = job.getMetricSettings().stream()
+                    .filter(settings -> settings
+                            .getMetric() == MetricName.DOCKER_LOGS)
+                    .findFirst();
 
-        if (!df.isPresent()) {
+        if (df.isEmpty() || (df.get().getDockerfile() == null)) {
             throw new MissingDockerfileException("Dockerfile not found");
         }
         var dockerfile = Files.createTempFile(
@@ -179,10 +179,10 @@ public class DockerRunner extends JobAnalyzer {
                 ".tar.gz"
         ).toFile();
 
-        var dfString = (String) df.get().get("dockerfile");
-
         try (
-                var in = new ByteArrayInputStream(dfString.getBytes(StandardCharsets.UTF_8));
+                var in = new ByteArrayInputStream(
+                        df.get().getDockerfile().getBytes(StandardCharsets.UTF_8)
+                );
                 var dout = new FileOutputStream(dockerfile);
                 var out = new TarArchiveOutputStream(new GZIPOutputStream(
                         new BufferedOutputStream(new FileOutputStream(dockerfileArchive))
@@ -193,7 +193,7 @@ public class DockerRunner extends JobAnalyzer {
             try {
                 entry = out.createArchiveEntry(dockerfile, "Dockerfile");
                 out.putArchiveEntry(entry);
-                out.write(dfString.getBytes(StandardCharsets.UTF_8));
+                out.write(df.get().getDockerfile().getBytes(StandardCharsets.UTF_8));
             } finally {
                 out.closeArchiveEntry();
             }

worker/src/main/java/nl/tudelft/ewi/auta/checker/generic/docker/api/DockerUriBuilder.java

 package nl.tudelft.ewi.auta.checker.generic.docker.api;
 
+import nl.tudelft.ewi.auta.worker.config.WorkerSettings;
+
 /**
  * Builds URIs for the nl.tudelft.ewi.auta.checker.generic.docker API.
  *
@@ -19,11 +21,13 @@ public class DockerUriBuilder {
 
     /**
      * Creates a new Docker URI builder
-     * generating URIs for a daemon listening on http://localhost:2376/.
+     * generating URIs for a daemon listening on the docker address.
+     *
+     * @param settings the worker settings
      */
-    public DockerUriBuilder() {
+    public DockerUriBuilder(final WorkerSettings settings) {
         this.protocol = "http";
-        this.hostname = "localhost:2376";
+        this.hostname = settings.getDockerAddress();
     }
 
     /**

worker/src/main/java/nl/tudelft/ewi/auta/worker/Job.java

@@ -111,7 +111,7 @@ public class Job implements AutoCloseable {
         return ((Collection<Map<String, Object>>) this.options.get("metrics")).stream()
                 .map(m ->
                         new MetricSettings(MetricName.valueOf((String) m.get("metric")),
-                                null, null, null, null
+                                null, null, null, null, (String) m.get("dockerfile")
                         ))
                 .collect(Collectors.toList());
     }

worker/src/main/java/nl/tudelft/ewi/auta/worker/config/WorkerSettings.java

@@ -93,6 +93,12 @@ public class WorkerSettings {
      */
     private final String authToken;
 
+    /**
+     * The address to reach the docker API on.
+     */
+    private final String dockerAddress;
+
+
     /**
      * Creates a new worker settings repository.
      *
@@ -121,6 +127,7 @@ public class WorkerSettings {
 
         this.encryptionKeyLength =
                 (int) Float.parseFloat(this.getOne(args, "key-length", "256.0"));
+        this.dockerAddress = this.getOne(args, "docker-address", "172.17.0.1:2376");
     }
 
     /**
@@ -254,6 +261,15 @@ public class WorkerSettings {
         return this.authToken;
     }
 
+    /**
+     * Returns the address to reach the docker api on.
+     *
+     * @return the address to reach the docker api on
+     */
+    public String getDockerAddress() {
+        return this.dockerAddress;
+    }
+
     /**
      * Returns a new randomly chosen name from the list.
      * appended with the current date and time to try and ensure uniqueness

worker/src/main/java/nl/tudelft/ewi/auta/worker/tool/python/PythonDetector.java

@@ -44,7 +44,7 @@ public class PythonDetector {
     /**
      * The minimum required version for worker tools.
      */
-    private static final Version TARGET_VERSION = new Version(3, 6, 0);
+    private static final Version TARGET_VERSION = new Version(3, 5, 0);
 
     /**
      * Valid Python executable names.

worker/src/test/java/nl/tudelft/ewi/auta/checker/generic/docker/DockerTest.java

-package nl.tudelft.ewi.auta.checker.generic.docker;
-
-import nl.tudelft.ewi.auta.checker.generic.docker.api.DockerApi;
-import nl.tudelft.ewi.auta.common.model.entity.ProjectEntity;
-import nl.tudelft.ewi.auta.worker.Job;
-import nl.tudelft.ewi.auta.worker.config.WorkerSettings;
-import nl.tudelft.ewi.auta.worker.jobconfig.JobConfig;
-import org.assertj.core.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
-public class DockerTest {
-
-    private DockerRunner runner;
-    private DockerApi api;
-
-    @BeforeEach
-    public void setup() {
-        this.api = Mockito.mock(DockerApi.class);
-        final var settings = Mockito.mock(WorkerSettings.class);
-
-        Mockito.when(
-                settings.getTemp()).thenReturn(Paths.get(System.getProperty("java.io.tmpdir"))
-        );
-        this.runner = new DockerRunner(this.api, settings);
-
-        Mockito.when(this.api.getContainers()).thenReturn("");
-        Mockito.when(this.api.createContainer(Mockito.anyMap())).thenReturn("test");
-        Mockito.when(this.api.getLogs(Mockito.anyString())).thenReturn("BUILD FAILED");
-    }
-
-    @Test
-    public void testJob() throws IOException {
-        Mockito.when(this.api.getImages()).thenReturn("test");
-
-        final var project = new ProjectEntity();
-        final var tempDir = Files.createTempDirectory("test");
-        final var checkList = new ArrayList<Map<String, String>>();
-        checkList.add(Map.of("dockerfile", "FROM test"));
-        final var job = new Job(
-                project,
-                "java",
-                Map.of("dynamic", checkList, "assignment name", "test"),
-                "test",
-                tempDir,
-                JobConfig.DEFAULT
-        );
-        this.runner.analyze(job, new HashMap<>());
-        final var message = job.getProject().getMetrics().iterator().next().getValue();
-        assertThat(message).isEqualTo("BUILD FAILED");
-    }
-
-    @Test
-    public void testMissingDockerfile() {
-        final var job = new Job(
-                null, null, Map.of("dynamic", new ArrayList<>()), null, null, JobConfig.DEFAULT
-        );
-        assertThrows(MissingDockerfileException.class,
-                () -> this.runner.analyze(job, new HashMap<>()));
-    }
-
-    @Test
-    public void testCreateImage() throws Exception {
-        Mockito.when(this.api.getImages()).thenReturn("");
-
-        final var project = new ProjectEntity();
-        final var tempDir = Files.createTempDirectory("test");
-        final var checkList = new ArrayList<Map<String, String>>();
-        checkList.add(Map.of("dockerfile", "FROM test"));
-        final var job = new Job(
-                project,
-                "java",
-                Map.of("dynamic", checkList, "assignment name", "test"),
-                "test",
-                tempDir,
-                JobConfig.DEFAULT
-        );
-        new Thread(() -> {
-            try {
-                this.runner.analyze(job, "");
-            } catch (IOException e) {
-                Assertions.fail("Caught IOException when analyzing");
-            }
-        }).start();
-        Thread.sleep(100);
-
-        Mockito.verify(this.api, Mockito.times(1))
-                .createImage(Mockito.anyString(), Mockito.any());
-    }
-}

worker/src/test/java/nl/tudelft/ewi/auta/checker/generic/docker/DockerURIBuilderTest.java

 package nl.tudelft.ewi.auta.checker.generic.docker;
 
 import nl.tudelft.ewi.auta.checker.generic.docker.api.DockerUriBuilder;
+import nl.tudelft.ewi.auta.worker.config.WorkerSettings;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
 
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 
@@ -12,18 +14,21 @@ public class DockerURIBuilderTest {
 
     @BeforeEach
     public void setup() {
-        this.builder = new DockerUriBuilder();
+        final var settings = Mockito.mock(WorkerSettings.class);
+        Mockito.when(settings.getDockerAddress()).thenReturn("172.17.0.1:2376");
+        this.builder = new DockerUriBuilder(settings);
     }
 
     @Test
     public void simpleTest() {
         var uri = this.builder.build("v1.37", "/containers/create");
-        assertThat(uri).isEqualTo("http://localhost:2376/v1.37/containers/create");
+        assertThat(uri).isEqualTo("http://172.17.0.1:2376/v1.37/containers/create");
     }
 
     @Test
     public void customPathTest() {
-        var uri = new DockerUriBuilder("http", "localhost:2376").build("v1.37", "/containers/logs");
+        var uri = new DockerUriBuilder("http", "172.17.0.1:2376").build("v1.37", "/containers"
+                + "/logs");
         var uriSimple = this.builder.build("v1.37", "/containers/logs");
         assertThat(uri).isEqualTo(uriSimple);
     }