From 9736e92073502adcfd466a629f10464f518e2c15 Mon Sep 17 00:00:00 2001 From: axtloss Date: Mon, 15 May 2023 22:36:20 +0200 Subject: multithreaded downloads and non docker registries Runs each layer download on a different thread. Allows using registries other than registry.docker.io --- .../java/io/github/jshipit/DockerAPIHelper.java | 320 ++++++++++----------- 1 file changed, 158 insertions(+), 162 deletions(-) (limited to 'src/main/java/io/github/jshipit/DockerAPIHelper.java') diff --git a/src/main/java/io/github/jshipit/DockerAPIHelper.java b/src/main/java/io/github/jshipit/DockerAPIHelper.java index dca582e..127f90e 100755 --- a/src/main/java/io/github/jshipit/DockerAPIHelper.java +++ b/src/main/java/io/github/jshipit/DockerAPIHelper.java @@ -1,162 +1,158 @@ -package io.github.jshipit; - - - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.*; -import com.fasterxml.jackson.databind.node.ArrayNode; - -import java.io.*; -import java.net.*; -import java.util.Scanner; - -public class DockerAPIHelper { - - private String apiToken; - private String apiRepo; - private String authURL; - private String repository; - private String image; - - private String tag; - - public DockerAPIHelper(String apiRepo, String authURL, String repository, String image, String tag) { - System.out.println("DockerAPIHelper constructor"); - this.apiRepo = apiRepo; - this.repository = repository; - this.authURL = authURL; - this.image = image; - this.tag = tag; - try { - apiToken = generateAPIToken(); - } catch (IOException | RuntimeException e) { - System.out.println("IOException | RuntimeException"); - e.printStackTrace(); - } - } - - public String generateAPIToken() throws IOException, RuntimeException { - URL url_obj = null; - try { - url_obj = new URI(this.authURL + "?scope=repository:" + this.repository + "/" + this.image + ":pull" + "&service=" + this.apiRepo).toURL(); - System.out.println(url_obj.toString()); - } catch (URISyntaxException | MalformedURLException e) { - System.out.println("URISyntaxException | MalformedURLException"); - e.printStackTrace(); - } - - - assert url_obj != null; - HttpURLConnection con = (HttpURLConnection) url_obj.openConnection(); - con.setRequestMethod("GET"); - con.connect(); - if (con.getResponseCode() != 200) { - System.out.println("Error: " + con.getResponseCode()); - throw new RuntimeException("Failed : HTTP error code : " - + con.getResponseCode()); - } else { - System.out.println("Success: " + con.getResponseCode()); - String output = ""; - Scanner scanner = new Scanner(url_obj.openStream()); - while (scanner.hasNext()) { - output += scanner.nextLine(); - } - scanner.close(); - System.out.println(output); - ObjectMapper mapper = new ObjectMapper(); - JsonNode token = mapper.readTree(output); - System.out.println(token.get("token").asText()); - return token.get("token").asText(); - } - } - - public JsonNode fetchManifestJson() throws IOException, RuntimeException { - URL url_obj = null; - try { - String repo = this.apiRepo.replace("registry", "registry-1"); - url_obj = new URI("https://" + repo + "/v2/" + this.repository + "/" + this.image + "/manifests/"+this.tag).toURL(); - System.out.println(url_obj.toString()); - } catch(URISyntaxException | MalformedURLException e) { - System.out.println("URISyntaxException | MalformedURLException"); - e.printStackTrace(); - } - - assert url_obj != null; - HttpURLConnection con = (HttpURLConnection) url_obj.openConnection(); - con.setRequestMethod("GET"); - con.setRequestProperty("Authorization", "Bearer " + this.apiToken); - con.setRequestProperty("Accept", "application/vnd.docker.distribution.manifest.v2+json"); - con.connect(); - - if (con.getResponseCode() != 200) { - System.out.println("Error: " + con.getResponseCode()); - throw new RuntimeException("Failed : HTTP error code : " - + con.getResponseCode()); - } else { - System.out.println("Success: " + con.getResponseCode()); - BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream())); - String inputLine; - StringBuffer content = new StringBuffer(); - while ((inputLine = in.readLine()) != null) { - content.append(inputLine); - } - in.close(); - con.disconnect(); - ObjectMapper mapper = new ObjectMapper(); - JsonNode manifest = mapper.readTree(content.toString()); - return manifest; - } - } - - public void fetchBlob(String digest, String tmpdir) throws IOException, RuntimeException { - URL url_obj = null; - try { - String repo = this.apiRepo.replace("registry", "registry-1"); - url_obj = new URI("https://" + repo + "/v2/" + this.repository + "/" + this.image + "/blobs/"+digest).toURL(); - System.out.println(url_obj.toString()); - } catch(URISyntaxException | MalformedURLException e) { - System.out.println("URISyntaxException | MalformedURLException"); - e.printStackTrace(); - } - - assert url_obj != null; - HttpURLConnection con = (HttpURLConnection) url_obj.openConnection(); - con.setRequestMethod("GET"); - con.setRequestProperty("Authorization", "Bearer " + this.apiToken); - con.setRequestProperty("Accept", "application/vnd.docker.distribution.manifest.v2+json"); - con.connect(); - - if (con.getResponseCode() != 200) { - System.out.println("Error: " + con.getResponseCode()); - throw new RuntimeException("Failed : HTTP error code : " - + con.getResponseCode()); - } else { - System.out.println("Success: " + con.getResponseCode()); - } - // Stream file download and output progress - InputStream inputStream = con.getInputStream(); - FileOutputStream outputStream = new FileOutputStream(tmpdir+"/"+digest.replace("sha256:", "")+"layer.tar"); - - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } - - outputStream.close(); - inputStream.close(); - System.out.println("File downloaded successfully."); - } - - public String getApiToken() { - return apiToken; - } - - public String getImage() { - return image; - } - - public String getTag() { - return tag; - } -} +package io.github.jshipit; + + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import me.tongfei.progressbar.*; + +import java.io.*; +import java.net.*; +import java.sql.Blob; +import java.util.List; +import java.util.Map; +import java.util.Scanner; + +public class DockerAPIHelper { + + private String apiToken; + private final String apiRepo; + private String authURL; + private String authService; + private final String repository; + private final String image; + private final String tag; + + public DockerAPIHelper(String apiRepo, String repository, String image, String tag) { + System.out.println("DockerAPIHelper constructor"); + if (apiRepo.contains("registry.docker.io")) { + apiRepo = apiRepo.replace("registry", "registry-1"); // Docker is just funny like that + } + this.apiRepo = apiRepo; + this.repository = repository; + this.image = image; + this.tag = tag; + this.authURL = "https://auth.docker.io/token"; + this.authService = "registry.docker.io"; + try { + getAuthenticationUrl(); + apiToken = generateAPIToken(); + } catch (IOException | RuntimeException e) { + System.out.println("IOException | RuntimeException"); + e.printStackTrace(); + } + } + + public void getAuthenticationUrl() throws IOException { + URL url_obj = null; + try { + url_obj = new URI("https://"+this.apiRepo+"/v2/").toURL(); + System.out.println(url_obj.toString()); + } catch (URISyntaxException | MalformedURLException e) { + e.printStackTrace(); + } + + assert url_obj != null; + HttpURLConnection con = (HttpURLConnection) url_obj.openConnection(); + con.setRequestMethod("GET"); + con.connect(); + if (con.getResponseCode() == 401 ) { + Map> headers = con.getHeaderFields(); + List authenticate = headers.get("Www-Authenticate"); + this.authURL = authenticate.get(0).replace("Bearer realm=", "").replace("\"", "").split(",")[0]; + this.authService = authenticate.get(0).replace("service=", "").replace("\"", "").split(",")[1]; + } else { + this.authURL = "https://auth.docker.io/token"; + this.authService = "registry.docker.io"; + } + } + + public String generateAPIToken() throws IOException, RuntimeException { + URL url_obj = null; + try { + url_obj = new URI(this.authURL + "?scope=repository:" + this.repository + "/" + this.image + ":pull" + "&service=" + this.authService).toURL(); + System.out.println(url_obj.toString()); + } catch (URISyntaxException | MalformedURLException e) { + System.out.println("URISyntaxException | MalformedURLException"); + e.printStackTrace(); + } + + + assert url_obj != null; + HttpURLConnection con = (HttpURLConnection) url_obj.openConnection(); + con.setRequestMethod("GET"); + con.connect(); + if (con.getResponseCode() != 200) { + System.out.println("Error: " + con.getResponseCode()); + throw new RuntimeException("Failed : HTTP error code : " + + con.getResponseCode()); + } else { + System.out.println("Success: " + con.getResponseCode()); + StringBuilder output = new StringBuilder(); + Scanner scanner = new Scanner(url_obj.openStream()); + while (scanner.hasNext()) { + output.append(scanner.nextLine()); + } + scanner.close(); + System.out.println(output); + ObjectMapper mapper = new ObjectMapper(); + JsonNode token = mapper.readTree(output.toString()); + System.out.println(token.get("token").asText()); + return token.get("token").asText(); + } + } + + public JsonNode fetchManifestJson() throws IOException, RuntimeException { + URL url_obj = null; + try { + url_obj = new URI("https://" + this.apiRepo + "/v2/" + this.repository + "/" + this.image + "/manifests/"+this.tag).toURL(); + System.out.println(url_obj.toString()); + } catch(URISyntaxException | MalformedURLException e) { + System.out.println("URISyntaxException | MalformedURLException"); + e.printStackTrace(); + } + + assert url_obj != null; + HttpURLConnection con = (HttpURLConnection) url_obj.openConnection(); + con.setRequestMethod("GET"); + con.setRequestProperty("Authorization", "Bearer " + this.apiToken); + con.setRequestProperty("Accept", "application/vnd.docker.distribution.manifest.v2+json"); + con.connect(); + + if (con.getResponseCode() != 200) { + System.out.println("Error: " + con.getResponseCode()); + throw new RuntimeException("Failed : HTTP error code : " + + con.getResponseCode()); + } else { + System.out.println("Success: " + con.getResponseCode()); + BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream())); + String inputLine; + StringBuilder content = new StringBuilder(); + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + in.close(); + con.disconnect(); + ObjectMapper mapper = new ObjectMapper(); + return mapper.readTree(content.toString()); + } + } + + public void fetchBlob(String digest, String tmpdir) throws IOException, RuntimeException { + String url = "https://" + this.apiRepo + "/v2/" + this.repository + "/" + this.image + "/blobs/"+digest; + String[][] headers = {{"Authorization", "Bearer "+this.apiToken}, {"Accept", "application/vnd/docker.distribution.manifest.v2+json"}}; + BlobDownloader downloader = new BlobDownloader(url, digest, headers, tmpdir); + downloader.start(); + } + + public String getApiToken() { + return apiToken; + } + + public String getImage() { + return image; + } + + public String getTag() { + return tag; + } +} -- cgit v1.2.3