Automatically extract downloaded blobs

6 files changed, 213 insertions, 30 deletions

diff --git a/src/main/java/io/github/jshipit/BlobDownloader.java b/src/main/java/io/github/jshipit/BlobDownloader.java
index 6a17057..91f126e 100644
--- a/src/main/java/io/github/jshipit/BlobDownloader.java
+++ b/src/main/java/io/github/jshipit/BlobDownloader.java
@@ -1,11 +1,10 @@
 package io.github.jshipit;
 
 import me.tongfei.progressbar.ProgressBar;
+import org.apache.commons.compress.archivers.tar.*;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
 import java.net.*;
 
 public class BlobDownloader extends Thread {
@@ -14,15 +13,17 @@ public class BlobDownloader extends Thread {
     private final String digest;
     private final String[][] headers;
     private final String tmpdir;
+    private final boolean extract;
 
-    public BlobDownloader(String url, String digest, String[][] headers, String tmpdir) {
+    public BlobDownloader(String url, String digest, String[][] headers, String tmpdir, boolean extract) {
         this.url = url;
         this.digest = digest;
         this.headers = headers;
         this.tmpdir = tmpdir;
+        this.extract = extract;
     }
 
-    public void run() throws RuntimeException {
+    public void run() {
         URL url_obj = null;
         try {
             url_obj = new URI(this.url).toURL();
@@ -60,7 +61,7 @@ public class BlobDownloader extends Thread {
         int fileSize = con.getContentLength();
 
         try (InputStream in = con.getInputStream();
-             OutputStream out = new FileOutputStream(tmpdir + "/" + digest.replace("sha256:", "") + "layer.tar")) {
+             OutputStream out = new FileOutputStream(tmpdir + "/" + digest.replace("sha256:", "") + ".tar.gz")) {
             byte[] buffer = new byte[4096];
             int bytesRead;
             try (ProgressBar pb = new ProgressBar("Download blob "+digest.replace("sha256:", ""), fileSize)) {
@@ -73,5 +74,25 @@ public class BlobDownloader extends Thread {
             System.out.println("Failed to download blob "+digest);
             e.printStackTrace();
         }
+
+        File extracted = new File(tmpdir + "/" + digest.replace("sha256:", ""));
+        if (extracted.exists()) {
+            extracted.delete();
+        }
+
+        if (!extract) {
+            return;
+        }
+        TarManager tarManager = new TarManager();
+        try {
+            tarManager.untar(tmpdir + "/" + digest.replace("sha256:", "") + ".tar.gz", extracted);
+        } catch (IOException e) {
+            System.out.println("Failed to extract blob "+digest);
+            e.printStackTrace();
+        }
+        File tar = new File(tmpdir + "/" + digest.replace("sha256:", "") + ".tar.gz");
+        if (tar.exists()) {
+            tar.delete();
+        }
     }
 }
diff --git a/src/main/java/io/github/jshipit/ContainerManager.java b/src/main/java/io/github/jshipit/ContainerManager.java
new file mode 100644
index 0000000..e8e2dad
--- /dev/null
+++ b/src/main/java/io/github/jshipit/ContainerManager.java
@@ -0,0 +1,56 @@
+package io.github.jshipit;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.UUID;
+
+public class ContainerManager {
+    private String containerID;
+    private String containerName;
+    private String containerRepo;
+    private String containerImage;
+    private String containerTag;
+    private String containerApiRepo;
+    private String entryCommand;
+    private OCIDataStore dataStore;
+
+    public ContainerManager(String containerName, String containerImage, String containerTag, String entryCommand, OCIDataStore dataStore) {
+        this.containerName = containerName;
+        this.containerImage = containerImage;
+        this.containerTag = containerTag;
+        this.entryCommand = entryCommand;
+        this.dataStore = dataStore;
+    }
+
+    public void createContainer() {
+        System.out.println("Creating container");
+
+        if (!Files.isDirectory(Paths.get(dataStore.getPath() + this.containerImage + "/" + this.containerTag))) {
+            System.out.println("Image does not exist, pulling now");
+            dataStore.createImage(this.containerApiRepo, this.containerRepo, this.containerImage, this.containerTag);
+        }
+
+        if (this.containerID == null) {
+            this.containerID = genContainerID();
+        }
+
+        String containerDirectory = dataStore.createContainerDirectory(this.containerImage, this.containerTag, this.containerName, this.containerID);
+
+        System.out.println("Container directory: " + containerDirectory);
+        System.out.println("Container ID: " + this.containerID);
+
+        new File(containerDirectory + "/containerOverlay").mkdirs();
+        new File(containerDirectory + "/root").mkdirs();
+
+
+
+
+    }
+
+    public String genContainerID() {
+        System.out.println("Generating container ID");
+        return UUID.randomUUID().toString();
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/io/github/jshipit/DockerAPIHelper.java b/src/main/java/io/github/jshipit/DockerAPIHelper.java
index fd601c6..d11f581 100755
--- a/src/main/java/io/github/jshipit/DockerAPIHelper.java
+++ b/src/main/java/io/github/jshipit/DockerAPIHelper.java
@@ -138,10 +138,10 @@ public class DockerAPIHelper {
         }
     }
 
-    public void fetchBlob(String digest, String tmpdir) throws IOException, RuntimeException {
+    public void fetchBlob(String digest, String tmpdir, boolean extract) throws IOException, RuntimeException {
         String url = "https://" + this.apiRepo + "/v2/" + this.repository + "/" + this.image + "/blobs/"+digest;
         String[][] headers = {{"Authorization", "Bearer "+this.apiToken}, {"Accept", "application/vnd/docker.distribution.manifest.v2+json"}};
-        BlobDownloader downloader = new BlobDownloader(url, digest, headers, tmpdir);
+        BlobDownloader downloader = new BlobDownloader(url, digest, headers, tmpdir, extract);
         downloader.start();
     }
 
diff --git a/src/main/java/io/github/jshipit/ImageBuilder.java b/src/main/java/io/github/jshipit/ImageBuilder.java
deleted file mode 100755
index 743ff1a..0000000
--- a/src/main/java/io/github/jshipit/ImageBuilder.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package io.github.jshipit;

-

-public class ImageBuilder {

-

-    private String[] layers;

-

-    public ImageBuilder(String[] layers) {

-        this.layers = layers;

-    }

-

-    /*

-        * Assembles a tarball from the layers

-     */

-    public void assemble() {

-

-        System.out.println("Assembling image");

-    }

-

-}

diff --git a/src/main/java/io/github/jshipit/OCIDataStore.java b/src/main/java/io/github/jshipit/OCIDataStore.java
index d2c3a25..3e0b9f3 100755
--- a/src/main/java/io/github/jshipit/OCIDataStore.java
+++ b/src/main/java/io/github/jshipit/OCIDataStore.java
@@ -2,6 +2,7 @@ package io.github.jshipit;
 
 import com.fasterxml.jackson.databind.JsonNode;
 
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.nio.file.FileAlreadyExistsException;
@@ -37,7 +38,6 @@ public class OCIDataStore {
             if (!(e instanceof FileAlreadyExistsException)) {
                 System.out.println("Failed to create directory: " + path);
                 e.printStackTrace();
-                return;
             }
         }
     }
@@ -52,6 +52,7 @@ public class OCIDataStore {
                 statement.setQueryTimeout(30);  // set timeout to 30 sec.
 
                 statement.executeUpdate("CREATE TABLE IF NOT EXISTS blobs (id INTEGER PRIMARY KEY AUTOINCREMENT, digest TEXT, path TEXT)");
+                statement.executeUpdate("CREATE TABLE IF NOT EXISTS containers (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, id TEXT, path TEXT)");
                 System.out.println("A new database has been created.");
             }
 
@@ -94,6 +95,35 @@ public class OCIDataStore {
         return false;
     }
 
+    public void addContainerToDatabase(String path, String name, String id) {
+        String url = "jdbc:sqlite:" + this.databasePath;
+
+        try (Connection conn = DriverManager.getConnection(url)) {
+            if (conn != null) {
+                Statement statement = conn.createStatement();
+                statement.setQueryTimeout(30);  // set timeout to 30 sec.
+
+                statement.executeUpdate("INSERT INTO containers (name, id, path) VALUES ('" + name + "', '" + id + "', '" + path + "')");
+            }
+
+        } catch (SQLException e) {
+            System.out.println(e.getMessage());
+        }
+    }
+
+    public String createContainerDirectory(String image, String tag, String name, String id) {
+        Path containerPath = Path.of(this.path+"/"+image+"/"+tag+"/"+name+"_"+id);
+        try {
+            Files.createDirectory(containerPath);
+        } catch (IOException e) {
+            System.out.println("Failed to create directory: " + containerPath);
+            e.printStackTrace();
+            return null;
+        }
+        addContainerToDatabase(containerPath.toString(), name, id);
+        return containerPath.toString();
+    }
+
     public void createImage(String apiRepo, String repo, String image, String tag) {
 
         Path imgPath = Path.of(this.path+"/"+image);
@@ -153,7 +183,7 @@ public class OCIDataStore {
 
             try {
                 if (!isBlobInDatabase(layer.get("digest").asText())) {
-                    api.fetchBlob(layer.get("digest").asText(), layerpath);
+                    api.fetchBlob(layer.get("digest").asText(), layerpath, true);
                     addBlobToDatabase(layer.get("digest").asText());
                 } else {
                     System.out.println("Blob already in database: " + layer.get("digest").asText());
@@ -164,6 +194,14 @@ public class OCIDataStore {
             }
         }
 
+        try {
+            api.fetchBlob(manifest.get("config").get("digest").asText(), this.path+"/"+image+"/"+tag, false);
+            File config = new File(this.path+"/"+image+"/"+tag+manifest.get("config").get("digest").asText().replace("sha256:", ""));
+            config.renameTo(new File(this.path+"/"+image+"/"+tag+"/config"));
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
         FileWriter writer = null;
         try {
             writer = new FileWriter(this.path+"/"+image+"/"+tag+"/layers");
@@ -174,4 +212,19 @@ public class OCIDataStore {
         }
     }
 
+    public String getPath() {
+        return path;
+    }
+
+    public void setPath(String path) {
+        this.path = path;
+    }
+
+    public String getDatabasePath() {
+        return databasePath;
+    }
+
+    public void setDatabasePath(String databasePath) {
+        this.databasePath = databasePath;
+    }
 }
diff --git a/src/main/java/io/github/jshipit/TarManager.java b/src/main/java/io/github/jshipit/TarManager.java
new file mode 100644
index 0000000..40db71d
--- /dev/null
+++ b/src/main/java/io/github/jshipit/TarManager.java
@@ -0,0 +1,72 @@
+package io.github.jshipit;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.utils.IOUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+public class TarManager {
+
+    public TarManager() {
+
+    }
+
+    public void tar(String name, File... files) throws IOException {
+        try (TarArchiveOutputStream out = getTarArchiveOutputStream(name)){
+            for (File file : files){
+                addToArchive(out, file, ".");
+            }
+        }
+    }
+
+    public void untar(String in, File out) throws IOException {
+        try (TarArchiveInputStream fin = new TarArchiveInputStream(new GzipCompressorInputStream(new FileInputStream(in)))){
+            TarArchiveEntry entry;
+            while ((entry = fin.getNextTarEntry()) != null) {
+                if (entry.isDirectory()) {
+                    continue;
+                }
+                File curfile = new File(out, entry.getName());
+                File parent = curfile.getParentFile();
+                if (!parent.exists()) {
+                    parent.mkdirs();
+                }
+                IOUtils.copy(fin, new FileOutputStream(curfile));
+            }
+        }
+    }
+
+    private TarArchiveOutputStream getTarArchiveOutputStream(String name) throws IOException {
+        TarArchiveOutputStream taos = new TarArchiveOutputStream(new FileOutputStream(name));
+        taos.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
+        taos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
+        taos.setAddPaxHeadersForNonAsciiNames(true);
+        return taos;
+    }
+
+    private void addToArchive(TarArchiveOutputStream out, File file, String dir) throws IOException {
+        String entry = dir + File.separator + file.getName();
+        if (file.isFile()){
+            out.putArchiveEntry(new TarArchiveEntry(file, entry));
+            try (FileInputStream in = new FileInputStream(file)){
+                IOUtils.copy(in, out);
+            }
+            out.closeArchiveEntry();
+        } else if (file.isDirectory()) {
+            File[] children = file.listFiles();
+            if (children != null){
+                for (File child : children){
+                    addToArchive(out, child, entry);
+                }
+            }
+        } else {
+            System.out.println(file.getName() + " is not supported");
+        }
+    }
+}
\ No newline at end of file