From 732147566e83c1e160154b18c33618a36fac30e0 Mon Sep 17 00:00:00 2001 From: axtloss Date: Wed, 24 May 2023 00:40:33 +0200 Subject: Add blob deduplication Adds installed blobs to a database and skips downloading them if it finds them in the database --- .../java/io/github/jshipit/DockerAPIHelper.java | 7 +- src/main/java/io/github/jshipit/JshipIT.java | 2 +- src/main/java/io/github/jshipit/OCIDataStore.java | 309 ++++++++++++--------- 3 files changed, 183 insertions(+), 135 deletions(-) (limited to 'src') diff --git a/src/main/java/io/github/jshipit/DockerAPIHelper.java b/src/main/java/io/github/jshipit/DockerAPIHelper.java index 56bfeca..fd601c6 100755 --- a/src/main/java/io/github/jshipit/DockerAPIHelper.java +++ b/src/main/java/io/github/jshipit/DockerAPIHelper.java @@ -35,7 +35,6 @@ public class DockerAPIHelper { getAuthenticationUrl(); apiToken = generateAPIToken(); } catch (IOException | RuntimeException e) { - System.out.println("IOException | RuntimeException"); e.printStackTrace(); } } @@ -55,7 +54,11 @@ public class DockerAPIHelper { con.connect(); if (con.getResponseCode() == 401 ) { Map> headers = con.getHeaderFields(); - List authenticate = headers.get("Www-Authenticate"); + List authenticate = headers.get("www-authenticate"); + if (authenticate == null) { + authenticate = headers.get("Www-Authenticate"); // Some registries (registry.getcryst.al) do this for some reason + } + assert authenticate != null; this.authURL = authenticate.get(0).replace("Bearer realm=", "").replace("\"", "").split(",")[0]; this.authService = authenticate.get(0).replace("service=", "").replace("\"", "").split(",")[1]; } else { diff --git a/src/main/java/io/github/jshipit/JshipIT.java b/src/main/java/io/github/jshipit/JshipIT.java index 165d0bd..7784128 100755 --- a/src/main/java/io/github/jshipit/JshipIT.java +++ b/src/main/java/io/github/jshipit/JshipIT.java @@ -43,6 +43,6 @@ public class JshipIT { */ OCIDataStore dataStore = new OCIDataStore("./tmp"); - dataStore.createImage("registry.getcryst.al","crystal/misc", "docker", "latest"); + dataStore.createImage("registry.docker.io","library", "bash", "devel-alpine3.18"); } } \ No newline at end of file diff --git a/src/main/java/io/github/jshipit/OCIDataStore.java b/src/main/java/io/github/jshipit/OCIDataStore.java index eaceff4..d2c3a25 100755 --- a/src/main/java/io/github/jshipit/OCIDataStore.java +++ b/src/main/java/io/github/jshipit/OCIDataStore.java @@ -1,132 +1,177 @@ -package io.github.jshipit; - -import com.fasterxml.jackson.databind.JsonNode; - -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.*; -import java.util.ArrayList; -import java.util.List; - -public class OCIDataStore { - - private String path; - private String databasePath; - - public OCIDataStore(String path) { - this.path = path; - this.databasePath = path + "/datstore.db"; - createStore(); - try { - createStoreDatabase(); - } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { - throw new RuntimeException(e); - } - - } - - private void createStore() { - System.out.println("Creating OCI Data Store"); - Path path = Path.of(this.path); - try { - Files.createDirectory(path); - } catch (IOException e) { - System.out.println("Failed to create directory: " + path); - e.printStackTrace(); - } - } - - private void createStoreDatabase() throws ClassNotFoundException, InstantiationException, IllegalAccessException { - String url = "jdbc:sqlite:" + this.databasePath; - - try (Connection conn = DriverManager.getConnection(url)) { - if (conn != null) { - DatabaseMetaData meta = conn.getMetaData(); - Statement statement = conn.createStatement(); - statement.setQueryTimeout(30); // set timeout to 30 sec. - - statement.executeUpdate("CREATE TABLE IF NOT EXISTS blobs (id INTEGER PRIMARY KEY AUTOINCREMENT, digest TEXT, path TEXT)"); - System.out.println("A new database has been created."); - } - - } catch (SQLException e) { - System.out.println(e.getMessage()); - } - } - - public void createImage(String apiRepo, String repo, String image, String tag) { - - Path imgPath = Path.of(this.path+"/"+image); - try { - Files.createDirectory(imgPath); - } catch (IOException e) { - System.out.println("Failed to create directory: " + imgPath); - } - - Path tagPath = Path.of(this.path+"/"+image+"/"+tag); - try { - Files.createDirectory(tagPath); - } catch (IOException e) { - System.out.println("Failed to create directory: " + tagPath); - e.printStackTrace(); - return; - } - - DockerAPIHelper api = new DockerAPIHelper(apiRepo, repo, image, tag); - JsonNode manifest = null; - - try { - manifest = api.fetchManifestJson(); - } catch (IOException ignored) {} // Proper error handling is bloat - - System.out.println("Manifest: " + manifest); - - Path path = Path.of(this.path+"/"+api.getImage()+"/"+api.getTag()); - try { - Files.createDirectory(path); - } catch (IOException e) { - if (!(e instanceof FileAlreadyExistsException)) { - System.out.println("Failed to create directory: " + path); - e.printStackTrace(); - return; - } - } - - assert manifest != null; - JsonNode layers = manifest.get("layers"); - List digests = new ArrayList(); - String layerpath = this.path+"/blobs"; - try { - Files.createDirectory(Path.of(layerpath)); - } catch (IOException e) { - if (!(e instanceof FileAlreadyExistsException)) { - System.out.println("Failed to create directory: " + layerpath); - e.printStackTrace(); - return; - } - } - for (JsonNode layer : layers) { - System.out.println("Layer: " + layer); - - try { - api.fetchBlob(layer.get("digest").asText(), layerpath); - digests.add(layer.get("digest").asText()); - } catch (IOException e) { - e.printStackTrace(); - } - } - - FileWriter writer = null; - try { - writer = new FileWriter(this.path+"/"+image+"/"+tag+"/layers"); - writer.write(String.join("\n", digests)); - writer.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - -} +package io.github.jshipit; + +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +public class OCIDataStore { + + private String path; + private String databasePath; + + public OCIDataStore(String path) { + this.path = path; + this.databasePath = path + "/datastore.db"; + createStore(); + try { + createStoreDatabase(); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { + throw new RuntimeException(e); + } + + } + + private void createStore() { + System.out.println("Creating OCI Data Store"); + Path path = Path.of(this.path); + try { + Files.createDirectory(path); + } catch (IOException e) { + if (!(e instanceof FileAlreadyExistsException)) { + System.out.println("Failed to create directory: " + path); + e.printStackTrace(); + return; + } + } + } + + private void createStoreDatabase() throws ClassNotFoundException, InstantiationException, IllegalAccessException { + String url = "jdbc:sqlite:" + this.databasePath; + + try (Connection conn = DriverManager.getConnection(url)) { + if (conn != null) { + DatabaseMetaData meta = conn.getMetaData(); + Statement statement = conn.createStatement(); + statement.setQueryTimeout(30); // set timeout to 30 sec. + + statement.executeUpdate("CREATE TABLE IF NOT EXISTS blobs (id INTEGER PRIMARY KEY AUTOINCREMENT, digest TEXT, path TEXT)"); + System.out.println("A new database has been created."); + } + + } catch (SQLException e) { + System.out.println(e.getMessage()); + } + } + + public void addBlobToDatabase(String blob) { + String url = "jdbc:sqlite:" + this.databasePath; + + try (Connection conn = DriverManager.getConnection(url)) { + if (conn != null) { + Statement statement = conn.createStatement(); + statement.setQueryTimeout(30); // set timeout to 30 sec. + + statement.executeUpdate("INSERT INTO blobs (digest, path) VALUES ('" + blob + "', '" + this.path + "/blobs/" + blob + "')"); + } + + } catch (SQLException e) { + System.out.println(e.getMessage()); + } + } + + public boolean isBlobInDatabase(String blob) { + String url = "jdbc:sqlite:" + this.databasePath; + + try (Connection conn = DriverManager.getConnection(url)) { + if (conn != null) { + Statement statement = conn.createStatement(); + statement.setQueryTimeout(30); // set timeout to 30 sec. + + ResultSet rs = statement.executeQuery("SELECT * FROM blobs WHERE digest = '" + blob + "'"); + return rs.next(); + } + + } catch (SQLException e) { + System.out.println(e.getMessage()); + } + return false; + } + + public void createImage(String apiRepo, String repo, String image, String tag) { + + Path imgPath = Path.of(this.path+"/"+image); + try { + Files.createDirectory(imgPath); + } catch (IOException e) { + System.out.println("Failed to create directory: " + imgPath); + } + + Path tagPath = Path.of(this.path+"/"+image+"/"+tag); + try { + Files.createDirectory(tagPath); + } catch (IOException e) { + System.out.println("Failed to create directory: " + tagPath); + e.printStackTrace(); + return; + } + + DockerAPIHelper api = new DockerAPIHelper(apiRepo, repo, image, tag); + + System.out.println("API Token: " + api.getApiToken()); + + JsonNode manifest = null; + + try { + manifest = api.fetchManifestJson(); + } catch (IOException ignored) {} // Proper error handling is bloat + + System.out.println("Manifest: " + manifest); + + Path path = Path.of(this.path+"/"+api.getImage()+"/"+api.getTag()); + try { + Files.createDirectory(path); + } catch (IOException e) { + if (!(e instanceof FileAlreadyExistsException)) { + System.out.println("Failed to create directory: " + path); + e.printStackTrace(); + return; + } + } + + assert manifest != null; + JsonNode layers = manifest.get("layers"); + List digests = new ArrayList(); + String layerpath = this.path+"/blobs"; + try { + Files.createDirectory(Path.of(layerpath)); + } catch (IOException e) { + if (!(e instanceof FileAlreadyExistsException)) { + System.out.println("Failed to create directory: " + layerpath); + e.printStackTrace(); + return; + } + } + for (JsonNode layer : layers) { + System.out.println("Layer: " + layer); + + try { + if (!isBlobInDatabase(layer.get("digest").asText())) { + api.fetchBlob(layer.get("digest").asText(), layerpath); + addBlobToDatabase(layer.get("digest").asText()); + } else { + System.out.println("Blob already in database: " + layer.get("digest").asText()); + } + digests.add(layer.get("digest").asText()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + FileWriter writer = null; + try { + writer = new FileWriter(this.path+"/"+image+"/"+tag+"/layers"); + writer.write(String.join("\n", digests)); + writer.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} -- cgit v1.2.3