aboutsummaryrefslogtreecommitdiff
path: root/src/main/java/io/github
diff options
context:
space:
mode:
Diffstat (limited to 'src/main/java/io/github')
-rw-r--r--src/main/java/io/github/jshipit/BlobDownloader.java38
-rw-r--r--src/main/java/io/github/jshipit/ContainerManager.java45
-rwxr-xr-xsrc/main/java/io/github/jshipit/DockerAPIHelper.java27
-rwxr-xr-xsrc/main/java/io/github/jshipit/JshipIT.java35
-rwxr-xr-xsrc/main/java/io/github/jshipit/OCIDataStore.java122
-rw-r--r--src/main/java/io/github/jshipit/SysUtils.java38
6 files changed, 227 insertions, 78 deletions
diff --git a/src/main/java/io/github/jshipit/BlobDownloader.java b/src/main/java/io/github/jshipit/BlobDownloader.java
index ea8b09c..f860a93 100644
--- a/src/main/java/io/github/jshipit/BlobDownloader.java
+++ b/src/main/java/io/github/jshipit/BlobDownloader.java
@@ -1,8 +1,6 @@
package io.github.jshipit;
import me.tongfei.progressbar.ProgressBar;
-import org.apache.commons.compress.archivers.tar.*;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import java.io.*;
import java.net.*;
@@ -27,6 +25,9 @@ public class BlobDownloader extends Thread {
this.renameTo = renameTo;
}
+ /*
+ * Download a blob
+ */
public void run() {
URL url_obj = null;
try {
@@ -59,12 +60,12 @@ public class BlobDownloader extends Thread {
}
- int fileSize = con.getContentLength();
+ int fileSize = con.getContentLength(); // We get the file size to display a progress bar
try (InputStream in = con.getInputStream();
OutputStream out = new FileOutputStream(tmpdir + "/" + digest.replace("sha256:", "") + ".tar.gz")) {
- byte[] buffer = new byte[4096];
- int bytesRead;
+ byte[] buffer = new byte[4096]; // Always read in chunks of 4096 bytes
+ int bytesRead; // How many bytes were read, used to update the progress bar
try (ProgressBar pb = new ProgressBar("Download blob "+digest.replace("sha256:", ""), fileSize)) {
while ((bytesRead = in.read(buffer)) != -1) {
out.write(buffer, 0, bytesRead);
@@ -90,19 +91,22 @@ public class BlobDownloader extends Thread {
return;
}
- if (!extract) {
- return;
- }
- File extracted = new File(tmpdir + "/" + digest.replace("sha256:", ""));
- if (extracted.exists()) {
- extracted.delete();
- }
- SysUtils tarManager = new SysUtils();
- tarManager.untar(tmpdir + "/" + digest.replace("sha256:", "") + ".tar.gz", tmpdir + "/" + digest.replace("sha256:", ""));
- File tar = new File(tmpdir + "/" + digest.replace("sha256:", "") + ".tar.gz");
- if (tar.exists()) {
- tar.delete();
+ if (extract) {
+
+ File extracted = new File(tmpdir + "/" + digest.replace("sha256:", ""));
+ if (extracted.exists()) {
+ extracted.delete();
+ }
+ SysUtils tarManager = new SysUtils();
+
+ // Extract the downloaded tar.gz
+ tarManager.untar(tmpdir + "/" + digest.replace("sha256:", "") + ".tar.gz", tmpdir + "/" + digest.replace("sha256:", ""));
+
+ File tar = new File(tmpdir + "/" + digest.replace("sha256:", "") + ".tar.gz");
+ if (tar.exists()) {
+ tar.delete();
+ }
}
}
}
diff --git a/src/main/java/io/github/jshipit/ContainerManager.java b/src/main/java/io/github/jshipit/ContainerManager.java
index 92482df..07dae9c 100644
--- a/src/main/java/io/github/jshipit/ContainerManager.java
+++ b/src/main/java/io/github/jshipit/ContainerManager.java
@@ -39,6 +39,9 @@ public class ContainerManager {
this.dataStore = dataStore;
}
+ /*
+ * Create a container directory
+ */
public void createContainer() {
System.out.println("Creating container");
@@ -54,11 +57,18 @@ public class ContainerManager {
String containerDirectory = dataStore.createContainerDirectory(this.containerImage, this.containerTag, this.containerName, this.containerApiRepo, this.containerRepo);
- new File(containerDirectory + "/containerOverlay").mkdirs();
- new File(containerDirectory + "/root").mkdirs();
- new File(containerDirectory+"/work").mkdirs();
+ new File(containerDirectory + "/containerOverlay").mkdirs(); // The upper directory of the overlay mount, user data and any changes to root will be stored here
+ new File(containerDirectory + "/root").mkdirs(); // The root directory of the overlay mount
+ new File(containerDirectory+"/work").mkdirs(); // The work directory of the overlay mount
}
+ /*
+ * Start a container
+ *
+ * @param getCommand: If true, return the command to run the container
+ *
+ * @return: The command to run the container. If getCommand is false, return null
+ */
public String startContainer(boolean getCommand) {
String containerDirectory = dataStore.getContainerPath(this.containerName);
@@ -68,7 +78,9 @@ public class ContainerManager {
} catch (IOException e) {
throw new RuntimeException(e);
}
- assert content != null;
+ assert content != null; // assert my beloved
+
+ // We parse the layers file to get the layers of the image
List<String> layers = new ArrayList<>();
for (String line : content) {
layers.add(this.dataStore.getPath() + "/blobs/" + line.replace("sha256:", ""));
@@ -77,15 +89,19 @@ public class ContainerManager {
SysUtils sysUtils = new SysUtils();
String[] diffs = layers.toArray(new String[0]);
+ // Depending on getCommand we either directly mount the layers or return the command to mount the layers
return sysUtils.overlayMount(diffs, containerDirectory + "/containerOverlay", containerDirectory + "/root", containerDirectory + "/work", !getCommand);
}
+ /*
+ * Run a command in a container
+ */
public void runCommand() {
String containerDirectory = dataStore.getContainerPath(this.containerName);
String dataStorePath = dataStore.getPath();
- File configPath = new File(dataStorePath + "/" + this.containerImage + "/" + this.containerTag + "/config");
+ File configPath = new File(dataStorePath + "/" + this.containerImage + "/" + this.containerTag + "/config"); // Path to the config file of the image
String content = null;
try {
content = Files.readAllLines(Paths.get(configPath.getAbsolutePath())).toString();
@@ -99,11 +115,11 @@ public class ContainerManager {
String hostname = null;
try {
JsonNode config = mapper.readTree(content.toString()).get(0).get("config");
- config.get("Env").forEach((JsonNode envVar) -> {
+ config.get("Env").forEach((JsonNode envVar) -> { // Get the environment variables specified in the config file
env.add(envVar.asText());
});
- cmd = config.get("Cmd").get(0).asText();
- hostname = config.get("Hostname").asText();
+ cmd = config.get("Cmd").get(0).asText(); // Get the command specified in the config file
+ hostname = config.get("Hostname").asText(); // Get the hostname specified in the config file
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
@@ -115,13 +131,13 @@ public class ContainerManager {
List<String> bwrapCommand = new ArrayList<>();
for(String envVar : env) {
- bwrapCommand.add("--setenv "+envVar.split("=")[0]+" "+envVar.split("=")[1]);
+ bwrapCommand.add("--setenv "+envVar.split("=")[0]+" "+envVar.split("=")[1]); // Create the flags for bwrap to set the environment variables
}
- bwrapCommand.add("--bind "+containerDirectory+"/root / --chdir /");
- bwrapCommand.add("--share-net");
- bwrapCommand.add("--unshare-uts --hostname "+ (!hostname.isBlank() ? hostname : this.containerName+"-"+this.containerImage));
- bwrapCommand.add("/bin/sh -c '"+(this.containerCommand != null ? this.containerCommand : cmd)+"'");
+ bwrapCommand.add("--bind "+containerDirectory+"/root / --chdir /"); // Bind the root to / inside the bwrap namespace
+ bwrapCommand.add("--share-net"); // Share the network namespace
+ bwrapCommand.add("--unshare-uts --hostname "+ (!hostname.isBlank() ? hostname : this.containerName+"-"+this.containerImage)); // Unshare the UTS namespace and set the hostname
+ bwrapCommand.add("/bin/sh -c '"+(this.containerCommand != null ? this.containerCommand : cmd)+"'"); // Run the command specified in the config file or the command specified in the constructor
SysUtils sysUtils = new SysUtils();
String bwrapCMD = sysUtils.execInBwrap(bwrapCommand.toArray(new String[0]), false);
String mountCMD = startContainer(true);
@@ -137,6 +153,9 @@ public class ContainerManager {
}
}
+ /*
+ * Delete a container
+ */
public void deleteContainer() {
String containerDirectory = dataStore.getContainerPath(this.containerName);
try {
diff --git a/src/main/java/io/github/jshipit/DockerAPIHelper.java b/src/main/java/io/github/jshipit/DockerAPIHelper.java
index ae0e536..b7c472c 100755
--- a/src/main/java/io/github/jshipit/DockerAPIHelper.java
+++ b/src/main/java/io/github/jshipit/DockerAPIHelper.java
@@ -38,6 +38,9 @@ public class DockerAPIHelper {
}
}
+ /*
+ * Get the Authentication URL from the registry
+ */
public void getAuthenticationUrl() throws IOException {
URL url_obj = null;
try {
@@ -60,11 +63,16 @@ public class DockerAPIHelper {
this.authURL = authenticate.get(0).replace("Bearer realm=", "").replace("\"", "").split(",")[0];
this.authService = authenticate.get(0).replace("service=", "").replace("\"", "").split(",")[1];
} else {
- this.authURL = "https://auth.docker.io/token";
+ this.authURL = "https://auth.docker.io/token"; // Just default to docker if the registry does not support or need authentication
this.authService = "registry.docker.io";
}
}
+ /*
+ * Generate an API token from the authentication URL
+ *
+ * @return String API token
+ */
public String generateAPIToken() throws IOException, RuntimeException {
URL url_obj = null;
try {
@@ -96,6 +104,11 @@ public class DockerAPIHelper {
}
}
+ /*
+ * Fetch the manifest JSON from the registry
+ *
+ * @return JsonNode Manifest JSON
+ */
public JsonNode fetchManifestJson() throws IOException, RuntimeException {
URL url_obj = null;
try {
@@ -130,6 +143,14 @@ public class DockerAPIHelper {
}
}
+ /*
+ * Fetch a blob from a specific image
+ *
+ * @param digest Digest of the blob to fetch
+ * @param tmpdir Directory to store the blob
+ * @param extract Whether or not to extract the blob
+ * @param renameTo Rename the blob to this name
+ */
public void fetchBlob(String digest, String tmpdir, boolean extract, String renameTo) throws IOException, RuntimeException {
String url = "https://" + this.apiRepo + "/v2/" + this.repository + "/" + this.image + "/blobs/"+digest;
String[][] headers = {{"Authorization", "Bearer "+this.apiToken}, {"Accept", "application/vnd/docker.distribution.manifest.v2+json"}};
@@ -137,10 +158,6 @@ public class DockerAPIHelper {
downloader.start();
}
- public String getApiToken() {
- return apiToken;
- }
-
public String getImage() {
return image;
}
diff --git a/src/main/java/io/github/jshipit/JshipIT.java b/src/main/java/io/github/jshipit/JshipIT.java
index 9ba74d0..76f224b 100755
--- a/src/main/java/io/github/jshipit/JshipIT.java
+++ b/src/main/java/io/github/jshipit/JshipIT.java
@@ -35,6 +35,8 @@ public class JshipIT {
System.out.println("Container name is required");
System.exit(1);
}
+
+ // Parse the image name into the registry, repo, image name and tag
List<String> image = new ArrayList<String>(Arrays.asList(commandCreate.containerImage.split("/")));
String containerImage = image.get(image.size() - 1).split(":")[0];
String apiRepo = image.get(0);
@@ -42,25 +44,16 @@ public class JshipIT {
image.remove(image.size() - 1);
String containerRepo = String.join("/", image);
- switch (apiRepo) {
- case "docker.io":
- apiRepo = "registry.docker.io";
- break;
- case "ghcr.io":
- apiRepo = "ghcr.io";
- break;
- case "quay.io":
- apiRepo = "quay.io";
- break;
- default:
- break;
+ // Convert the registry name to the OCI registry name
+ if (apiRepo == "docker.io") {
+ apiRepo = "registry.docker.io";
}
-
ContainerManager containerManager = new ContainerManager(commandCreate.containerName, containerImage, commandCreate.containerImage.split(":")[1], apiRepo, containerRepo, dataStore);
containerManager.createContainer();
} else if (commands.getParsedCommand().equals("pull")) {
+ // Parse the image name into the registry, repo, image name and tag
List<String> image = new ArrayList<String>(Arrays.asList(commandPull.containerImage.split("/")));
String containerImage = image.get(image.size() - 1).split(":")[0];
String apiRepo = image.get(0);
@@ -68,19 +61,11 @@ public class JshipIT {
image.remove(image.size() - 1);
String containerRepo = String.join("/", image);
- switch (apiRepo) {
- case "docker.io":
- apiRepo = "registry.docker.io";
- break;
- case "ghcr.io":
- apiRepo = "ghcr.io";
- break;
- case "quay.io":
- apiRepo = "quay.io";
- break;
- default:
- break;
+ // Convert the registry name to the OCI registry name
+ if (apiRepo == "docker.io") {
+ apiRepo = "registry.docker.io";
}
+
System.out.println("Pulling image " + containerImage + " from " + apiRepo + "/" + containerRepo);
dataStore.createImage(apiRepo, containerRepo, containerImage, commandPull.containerImage.split(":")[1]);
} else if (commands.getParsedCommand().equals("start")) {
diff --git a/src/main/java/io/github/jshipit/OCIDataStore.java b/src/main/java/io/github/jshipit/OCIDataStore.java
index e399e75..c77f671 100755
--- a/src/main/java/io/github/jshipit/OCIDataStore.java
+++ b/src/main/java/io/github/jshipit/OCIDataStore.java
@@ -10,6 +10,10 @@ import java.sql.*;
import java.util.ArrayList;
import java.util.List;
+/*
+ * OCI Data Store
+ * This class is responsible for managing the OCI data store
+ */
public class OCIDataStore {
private String path;
@@ -18,9 +22,11 @@ public class OCIDataStore {
public OCIDataStore(String path) {
this.path = path;
this.databasePath = path + "/datastore.db";
+ // Create OCI Data Store if it does not exist
if (!Files.isDirectory(Path.of(path))) {
createStore();
}
+ // Create OCI Data Store database if it does not exist
if (!Files.exists(Path.of(this.databasePath))) {
try {
createStoreDatabase();
@@ -30,6 +36,9 @@ public class OCIDataStore {
}
}
+ /*
+ * Creates the OCI Data Store
+ */
private void createStore() {
System.out.println("Creating OCI Data Store");
Path path = Path.of(this.path);
@@ -43,6 +52,9 @@ public class OCIDataStore {
}
}
+ /*
+ * Creates the OCI Data Store database
+ */
private void createStoreDatabase() throws ClassNotFoundException, InstantiationException, IllegalAccessException {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -52,6 +64,7 @@ public class OCIDataStore {
Statement statement = conn.createStatement();
statement.setQueryTimeout(30); // set timeout to 30 sec.
+ // We use two tables so that we can deduplicate blobs and keep track of containers that the user created
statement.executeUpdate("CREATE TABLE IF NOT EXISTS blobs (id INTEGER PRIMARY KEY AUTOINCREMENT, digest TEXT, path TEXT)");
statement.executeUpdate("CREATE TABLE IF NOT EXISTS containers (containerID INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, path TEXT, image TEXT, tag TEXT, apiRepo TEXT, repo TEXT)");
System.out.println("A new database has been created.");
@@ -62,6 +75,11 @@ public class OCIDataStore {
}
}
+ /*
+ * Adds a downloaded blob to the database
+ *
+ * @param blob The blob to add
+ */
public void addBlobToDatabase(String blob) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -78,6 +96,13 @@ public class OCIDataStore {
}
}
+ /*
+ * Checks if a blob is in the database
+ *
+ * @param blob The blob to check
+ *
+ * @return True if the blob is in the database, false otherwise
+ */
public boolean isBlobInDatabase(String blob) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -96,6 +121,16 @@ public class OCIDataStore {
return false;
}
+ /*
+ * Adds a container to the database
+ *
+ * @param path The path to the container
+ * @param name The name of the container
+ * @param image The image that the container uses
+ * @param tag The image tag that the container uses
+ * @param apiRepo The API repository of the container
+ * @param repo The repository of the container
+ */
public void addContainerToDatabase(String path, String name, String image, String tag, String apiRepo, String repo) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -112,6 +147,11 @@ public class OCIDataStore {
}
}
+ /*
+ * Deletes a container from the database
+ *
+ * @param name The name of the container to delete
+ */
public void deleteContainerFromDatabase(String name) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -128,6 +168,13 @@ public class OCIDataStore {
}
}
+ /*
+ * Checks if a container exists in the database
+ *
+ * @param name The name of the container to check
+ *
+ * @return True if the container exists, false otherwise
+ */
public boolean containerExists(String name) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -146,6 +193,13 @@ public class OCIDataStore {
return false;
}
+ /*
+ * Gets the path to a container
+ *
+ * @param name The name of the container
+ *
+ * @return The path to the container
+ */
public String getContainerPath(String name) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -164,6 +218,13 @@ public class OCIDataStore {
return null;
}
+ /*
+ * Gets the image of a container
+ *
+ * @param name The name of the container
+ *
+ * @return The image of the container
+ */
public String getContainerImage(String name) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -182,6 +243,14 @@ public class OCIDataStore {
return null;
}
+
+ /*
+ * Gets the tag of a container
+ *
+ * @param name The name of the container
+ *
+ * @return The tag of the container
+ */
public String getContainerTag(String name) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -200,6 +269,14 @@ public class OCIDataStore {
return null;
}
+
+ /*
+ * Gets the API repository of a container
+ *
+ * @param name The name of the container
+ *
+ * @return The API repository of the container
+ */
public String getContainerApiRepo(String name) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -218,6 +295,13 @@ public class OCIDataStore {
return null;
}
+ /*
+ * Gets the repository of a container
+ *
+ * @param name The name of the container
+ *
+ * @return The repository of the container
+ */
public String getContainerRepo(String name) {
String url = "jdbc:sqlite:" + this.databasePath;
@@ -236,6 +320,17 @@ public class OCIDataStore {
return null;
}
+ /*
+ * Creates the directory for a container and adds it to the database
+ *
+ * @param image The image of the container
+ * @param tag The tag of the container
+ * @param name The name of the container
+ * @param apiRepo The API repository of the container
+ * @param repo The repository of the container
+ *
+ * @return The path to the container
+ */
public String createContainerDirectory(String image, String tag, String name, String apiRepo, String repo) {
Path containerPath = Path.of(this.path+"/"+image+"/"+tag+"/"+name);
try {
@@ -249,8 +344,15 @@ public class OCIDataStore {
return containerPath.toString();
}
+ /*
+ * Downlaods an oci-image, extracts it and adds it to the blob database
+ *
+ * @param apiRepo The API repository of the container
+ * @param repo The repository of the container
+ * @param image The image of the container
+ * @param tag The tag of the container
+ */
public void createImage(String apiRepo, String repo, String image, String tag) {
-
Path imgPath = Path.of(this.path+"/"+image);
try {
Files.createDirectory(imgPath);
@@ -298,10 +400,11 @@ public class OCIDataStore {
return;
}
}
- List <String> layerDigests = new ArrayList<>();
+
+ List <String> layerDigests = new ArrayList<>(); // We store the blobs an image uses in a file, so that we know which blobs are used by which images
for (JsonNode layer : layers) {
try {
- if (!isBlobInDatabase(layer.get("digest").asText())) {
+ if (!isBlobInDatabase(layer.get("digest").asText())) { // We only download blobs that are not already in the database
api.fetchBlob(layer.get("digest").asText(), layerpath, true, null);
addBlobToDatabase(layer.get("digest").asText());
layerDigests.add(layer.get("digest").asText());
@@ -319,6 +422,7 @@ public class OCIDataStore {
e.printStackTrace();
}
+ // We download the config file of the image (I was too lazy to make a separate download function for that (having this part async is not good))
try {
api.fetchBlob(manifest.get("config").get("digest").asText(), this.path+"/"+image+"/"+tag, false, this.path+"/"+image+"/"+tag+"/config");
} catch (IOException e) {
@@ -330,16 +434,4 @@ public class OCIDataStore {
public String getPath() {
return path;
}
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public String getDatabasePath() {
- return databasePath;
- }
-
- public void setDatabasePath(String databasePath) {
- this.databasePath = databasePath;
- }
}
diff --git a/src/main/java/io/github/jshipit/SysUtils.java b/src/main/java/io/github/jshipit/SysUtils.java
index 59450ad..ed14612 100644
--- a/src/main/java/io/github/jshipit/SysUtils.java
+++ b/src/main/java/io/github/jshipit/SysUtils.java
@@ -4,10 +4,19 @@ import com.sun.jna.Platform;
import java.io.File;
+/*
+* A class for running system commands
+ */
public class SysUtils {
-
+ /*
+ * Untar a tarball to a directory
+ *
+ * @param in The tarball to untar
+ * @param out The directory to untar to
+ */
public void untar(String in, String out) {
+ // We do not use the Java tar library because it does not care about unix permissions. Common Java L
new File(out).mkdirs();
ProcessBuilder pb = new ProcessBuilder("tar", "-xf", in, "-C", out);
pb.inheritIO();
@@ -19,10 +28,20 @@ public class SysUtils {
}
}
+ /*
+ * Execute a command in a bwrap sandbox
+ *
+ * @param args The arguments to pass to bwrap
+ * @param execute Whether to execute the command or just return the command
+ *
+ * @return The command that was executed. Empty string if execute is true
+ */
public String execInBwrap(String[] args, boolean execute) {
if (!execute) {
return "bwrap "+String.join(" ", args);
}
+ // ProcessBuilder wraps the args in quotes, since all args are passed at once, they are all quoted together
+ // This breaks bwrap (and many other commands), so we use bash -c to execute the command with the entire command as the arg
ProcessBuilder pb = new ProcessBuilder("bash", "-c", "bwrap "+String.join(" ", args));
pb.inheritIO();
try {
@@ -34,12 +53,24 @@ public class SysUtils {
return "";
}
+ /*
+ * Creates an overlay mount
+ *
+ * @param lower The lower directories to use
+ * @param upper The upper directory to use
+ * @param target The target directory to mount to
+ * @param work The work directory to use
+ * @param execute Whether to execute the command or just return the command
+ *
+ * @return The command that was executed. Empty string if execute is true
+ */
public String overlayMount(String[] lower, String upper, String target, String work, boolean execute) {
if (!execute) {
return "mount -t overlay overlay -o lowerdir="+String.join(":", lower)+",upperdir="+upper+",workdir="+work+" "+target;
}
if (Platform.isLinux()) {
-
+ // unshare creates a new user namespace, so we can mount without root
+ // This only works on Linux version 5.11 and above, which everyone should have by now
ProcessBuilder pb = new ProcessBuilder("unshare", "--user", "--map-root-user", "--mount", "mount", "-t", "overlay", "overlay", "-o", "lowerdir="+String.join(":", lower)+",upperdir="+upper+",workdir="+work, target);
pb.inheritIO();
try {
@@ -50,8 +81,9 @@ public class SysUtils {
}
} else {
+ // Since other *nix systems do not have overlayfs or unshare, we just print an error
+ // I would be surprised if you even got this far on Windows
System.out.println("Platform not supported.");
- System.out.println("mount -t overlay overlay -o lowerdir="+String.join(":", lower)+",upperdir="+upper+",workdir="+work+" "+target);
}
return "";
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-16 14:18:59 +0000