Add blob deduplication

Adds installed blobs to a database and skips downloading them if it finds them in the database

3 files changed, 183 insertions, 135 deletions

diff --git a/src/main/java/io/github/jshipit/DockerAPIHelper.java b/src/main/java/io/github/jshipit/DockerAPIHelper.java
index 56bfeca..fd601c6 100755
--- a/src/main/java/io/github/jshipit/DockerAPIHelper.java
+++ b/src/main/java/io/github/jshipit/DockerAPIHelper.java
@@ -35,7 +35,6 @@ public class DockerAPIHelper {
             getAuthenticationUrl();
             apiToken = generateAPIToken();
         } catch (IOException | RuntimeException e) {
-            System.out.println("IOException | RuntimeException");
             e.printStackTrace();
         }
     }
@@ -55,7 +54,11 @@ public class DockerAPIHelper {
         con.connect();
         if (con.getResponseCode() == 401 ) {
             Map<String, List<String>> headers = con.getHeaderFields();
-            List<String> authenticate = headers.get("Www-Authenticate");
+            List<String> authenticate = headers.get("www-authenticate");
+            if (authenticate == null) {
+                authenticate = headers.get("Www-Authenticate"); // Some registries (registry.getcryst.al) do this for some reason
+            }
+            assert authenticate != null;
             this.authURL = authenticate.get(0).replace("Bearer realm=", "").replace("\"", "").split(",")[0];
             this.authService = authenticate.get(0).replace("service=", "").replace("\"", "").split(",")[1];
         } else {
diff --git a/src/main/java/io/github/jshipit/JshipIT.java b/src/main/java/io/github/jshipit/JshipIT.java
index 165d0bd..7784128 100755
--- a/src/main/java/io/github/jshipit/JshipIT.java
+++ b/src/main/java/io/github/jshipit/JshipIT.java
@@ -43,6 +43,6 @@ public class JshipIT {
          */
 
         OCIDataStore dataStore = new OCIDataStore("./tmp");
-        dataStore.createImage("registry.getcryst.al","crystal/misc", "docker", "latest");
+        dataStore.createImage("registry.docker.io","library", "bash", "devel-alpine3.18");
     }
 }
\ No newline at end of file
diff --git a/src/main/java/io/github/jshipit/OCIDataStore.java b/src/main/java/io/github/jshipit/OCIDataStore.java
index eaceff4..d2c3a25 100755
--- a/src/main/java/io/github/jshipit/OCIDataStore.java
+++ b/src/main/java/io/github/jshipit/OCIDataStore.java
@@ -1,132 +1,177 @@
-package io.github.jshipit;

-

-import com.fasterxml.jackson.databind.JsonNode;

-

-import java.io.FileWriter;

-import java.io.IOException;

-import java.nio.file.FileAlreadyExistsException;

-import java.nio.file.Files;

-import java.nio.file.Path;

-import java.sql.*;

-import java.util.ArrayList;

-import java.util.List;

-

-public class OCIDataStore {

-

-    private String path;

-    private String databasePath;

-

-    public OCIDataStore(String path) {

-        this.path = path;

-        this.databasePath = path + "/datstore.db";

-        createStore();

-        try {

-            createStoreDatabase();

-        } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {

-            throw new RuntimeException(e);

-        }

-

-    }

-

-    private void createStore() {

-        System.out.println("Creating OCI Data Store");

-        Path path = Path.of(this.path);

-        try {

-            Files.createDirectory(path);

-        } catch (IOException e) {

-            System.out.println("Failed to create directory: " + path);

-            e.printStackTrace();

-        }

-    }

-

-    private void createStoreDatabase() throws ClassNotFoundException, InstantiationException, IllegalAccessException {

-        String url = "jdbc:sqlite:" + this.databasePath;

-

-        try (Connection conn = DriverManager.getConnection(url)) {

-            if (conn != null) {

-                DatabaseMetaData meta = conn.getMetaData();

-                Statement statement = conn.createStatement();

-                statement.setQueryTimeout(30);  // set timeout to 30 sec.

-

-                statement.executeUpdate("CREATE TABLE IF NOT EXISTS blobs (id INTEGER PRIMARY KEY AUTOINCREMENT, digest TEXT, path TEXT)");

-                System.out.println("A new database has been created.");

-            }

-

-        } catch (SQLException e) {

-            System.out.println(e.getMessage());

-        }

-    }

-

-    public void createImage(String apiRepo, String repo, String image, String tag) {

-

-        Path imgPath = Path.of(this.path+"/"+image);

-        try {

-            Files.createDirectory(imgPath);

-        } catch (IOException e) {

-            System.out.println("Failed to create directory: " + imgPath);

-        }

-

-        Path tagPath = Path.of(this.path+"/"+image+"/"+tag);

-        try {

-            Files.createDirectory(tagPath);

-        } catch (IOException e) {

-            System.out.println("Failed to create directory: " + tagPath);

-            e.printStackTrace();

-            return;

-        }

-

-        DockerAPIHelper api = new DockerAPIHelper(apiRepo, repo, image, tag);

-        JsonNode manifest = null;

-

-        try {

-            manifest = api.fetchManifestJson();

-        } catch (IOException ignored) {} // Proper error handling is bloat

-

-        System.out.println("Manifest: " + manifest);

-

-        Path path = Path.of(this.path+"/"+api.getImage()+"/"+api.getTag());

-        try {

-            Files.createDirectory(path);

-        } catch (IOException e) {

-            if (!(e instanceof FileAlreadyExistsException)) {

-                System.out.println("Failed to create directory: " + path);

-                e.printStackTrace();

-                return;

-            }

-        }

-

-        assert manifest != null;

-        JsonNode layers = manifest.get("layers");

-        List<String> digests = new ArrayList<String>();

-        String layerpath = this.path+"/blobs";

-        try {

-            Files.createDirectory(Path.of(layerpath));

-        } catch (IOException e) {

-            if (!(e instanceof FileAlreadyExistsException)) {

-                System.out.println("Failed to create directory: " + layerpath);

-                e.printStackTrace();

-                return;

-            }

-        }

-        for (JsonNode layer : layers) {

-            System.out.println("Layer: " + layer);

-

-            try {

-                api.fetchBlob(layer.get("digest").asText(), layerpath);

-                digests.add(layer.get("digest").asText());

-            } catch (IOException e) {

-                e.printStackTrace();

-            }

-        }

-

-        FileWriter writer = null;

-        try {

-            writer = new FileWriter(this.path+"/"+image+"/"+tag+"/layers");

-            writer.write(String.join("\n", digests));

-            writer.close();

-        } catch (IOException e) {

-            throw new RuntimeException(e);

-        }

-    }

-

-}

+package io.github.jshipit;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.List;
+
+public class OCIDataStore {
+
+    private String path;
+    private String databasePath;
+
+    public OCIDataStore(String path) {
+        this.path = path;
+        this.databasePath = path + "/datastore.db";
+        createStore();
+        try {
+            createStoreDatabase();
+        } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
+    private void createStore() {
+        System.out.println("Creating OCI Data Store");
+        Path path = Path.of(this.path);
+        try {
+            Files.createDirectory(path);
+        } catch (IOException e) {
+            if (!(e instanceof FileAlreadyExistsException)) {
+                System.out.println("Failed to create directory: " + path);
+                e.printStackTrace();
+                return;
+            }
+        }
+    }
+
+    private void createStoreDatabase() throws ClassNotFoundException, InstantiationException, IllegalAccessException {
+        String url = "jdbc:sqlite:" + this.databasePath;
+
+        try (Connection conn = DriverManager.getConnection(url)) {
+            if (conn != null) {
+                DatabaseMetaData meta = conn.getMetaData();
+                Statement statement = conn.createStatement();
+                statement.setQueryTimeout(30);  // set timeout to 30 sec.
+
+                statement.executeUpdate("CREATE TABLE IF NOT EXISTS blobs (id INTEGER PRIMARY KEY AUTOINCREMENT, digest TEXT, path TEXT)");
+                System.out.println("A new database has been created.");
+            }
+
+        } catch (SQLException e) {
+            System.out.println(e.getMessage());
+        }
+    }
+
+    public void addBlobToDatabase(String blob) {
+        String url = "jdbc:sqlite:" + this.databasePath;
+
+        try (Connection conn = DriverManager.getConnection(url)) {
+            if (conn != null) {
+                Statement statement = conn.createStatement();
+                statement.setQueryTimeout(30);  // set timeout to 30 sec.
+
+                statement.executeUpdate("INSERT INTO blobs (digest, path) VALUES ('" + blob + "', '" + this.path + "/blobs/" + blob + "')");
+            }
+
+        } catch (SQLException e) {
+            System.out.println(e.getMessage());
+        }
+    }
+
+    public boolean isBlobInDatabase(String blob) {
+        String url = "jdbc:sqlite:" + this.databasePath;
+
+        try (Connection conn = DriverManager.getConnection(url)) {
+            if (conn != null) {
+                Statement statement = conn.createStatement();
+                statement.setQueryTimeout(30);  // set timeout to 30 sec.
+
+                ResultSet rs = statement.executeQuery("SELECT * FROM blobs WHERE digest = '" + blob + "'");
+                return rs.next();
+            }
+
+        } catch (SQLException e) {
+            System.out.println(e.getMessage());
+        }
+        return false;
+    }
+
+    public void createImage(String apiRepo, String repo, String image, String tag) {
+
+        Path imgPath = Path.of(this.path+"/"+image);
+        try {
+            Files.createDirectory(imgPath);
+        } catch (IOException e) {
+            System.out.println("Failed to create directory: " + imgPath);
+        }
+
+        Path tagPath = Path.of(this.path+"/"+image+"/"+tag);
+        try {
+            Files.createDirectory(tagPath);
+        } catch (IOException e) {
+            System.out.println("Failed to create directory: " + tagPath);
+            e.printStackTrace();
+            return;
+        }
+
+        DockerAPIHelper api = new DockerAPIHelper(apiRepo, repo, image, tag);
+
+        System.out.println("API Token: " + api.getApiToken());
+
+        JsonNode manifest = null;
+
+        try {
+            manifest = api.fetchManifestJson();
+        } catch (IOException ignored) {} // Proper error handling is bloat
+
+        System.out.println("Manifest: " + manifest);
+
+        Path path = Path.of(this.path+"/"+api.getImage()+"/"+api.getTag());
+        try {
+            Files.createDirectory(path);
+        } catch (IOException e) {
+            if (!(e instanceof FileAlreadyExistsException)) {
+                System.out.println("Failed to create directory: " + path);
+                e.printStackTrace();
+                return;
+            }
+        }
+
+        assert manifest != null;
+        JsonNode layers = manifest.get("layers");
+        List<String> digests = new ArrayList<String>();
+        String layerpath = this.path+"/blobs";
+        try {
+            Files.createDirectory(Path.of(layerpath));
+        } catch (IOException e) {
+            if (!(e instanceof FileAlreadyExistsException)) {
+                System.out.println("Failed to create directory: " + layerpath);
+                e.printStackTrace();
+                return;
+            }
+        }
+        for (JsonNode layer : layers) {
+            System.out.println("Layer: " + layer);
+
+            try {
+                if (!isBlobInDatabase(layer.get("digest").asText())) {
+                    api.fetchBlob(layer.get("digest").asText(), layerpath);
+                    addBlobToDatabase(layer.get("digest").asText());
+                } else {
+                    System.out.println("Blob already in database: " + layer.get("digest").asText());
+                }
+                digests.add(layer.get("digest").asText());
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+
+        FileWriter writer = null;
+        try {
+            writer = new FileWriter(this.path+"/"+image+"/"+tag+"/layers");
+            writer.write(String.join("\n", digests));
+            writer.close();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}