From 82b84af50c64883ef32585b97638ee055ec74367 Mon Sep 17 00:00:00 2001 From: tonikelope Date: Sat, 28 Oct 2017 02:06:43 +0200 Subject: [PATCH] 2.25 -Upload multi-slot multi-size remove --- src/megabasterd/ChunkDownloader.java | 5 +- src/megabasterd/ChunkDownloaderMono.java | 7 +- src/megabasterd/ChunkUploader.java | 11 ++-- src/megabasterd/ChunkUploaderMono.java | 9 +-- src/megabasterd/ChunkWriter.java | 2 +- src/megabasterd/Download.java | 5 +- src/megabasterd/MainPanel.java | 2 +- src/megabasterd/Transference.java | 3 +- src/megabasterd/Upload.java | 21 +++++- src/megabasterd/UploadMACGenerator.java | 83 +++++++----------------- 10 files changed, 64 insertions(+), 84 deletions(-) diff --git a/src/megabasterd/ChunkDownloader.java b/src/megabasterd/ChunkDownloader.java index 88168edeb..1bc808e67 100644 --- a/src/megabasterd/ChunkDownloader.java +++ b/src/megabasterd/ChunkDownloader.java @@ -93,7 +93,6 @@ public void run() { String worker_url = null; Chunk chunk; int reads, conta_error, http_status; - byte[] buffer = new byte[THROTTLE_SLICE_SIZE]; InputStream is; boolean error; @@ -110,7 +109,7 @@ public void run() { worker_url = _download.getDownloadUrlForWorker(); } - chunk = new Chunk(_download.nextChunkId(), _download.getFile_size(), worker_url, Transference.CHUNK_SIZE_MULTI); + chunk = new Chunk(_download.nextChunkId(), _download.getFile_size(), worker_url, Download.CHUNK_SIZE_MULTI); HttpGet httpget = new HttpGet(new URI(chunk.getUrl())); @@ -131,6 +130,8 @@ public void run() { } else { + byte[] buffer = new byte[THROTTLE_SLICE_SIZE]; + while (!_exit && !_download.isStopped() && !_download.getChunkwriter().isExit() && chunk.getOutputStream().size() < chunk.getSize() && (reads = is.read(buffer)) != -1) { chunk.getOutputStream().write(buffer, 0, reads); diff --git a/src/megabasterd/ChunkDownloaderMono.java b/src/megabasterd/ChunkDownloaderMono.java index 495b58d39..b7376165d 100644 --- a/src/megabasterd/ChunkDownloaderMono.java +++ b/src/megabasterd/ChunkDownloaderMono.java @@ -28,8 +28,7 @@ public ChunkDownloaderMono(Download download) { public void run() { String worker_url = null; Chunk chunk; - int reads, max_reads, conta_error, http_status = 200; - byte[] buffer = new byte[THROTTLE_SLICE_SIZE]; + int reads, conta_error, http_status = 200; boolean error; HttpGet httpget = null; CloseableHttpResponse httpresponse = null; @@ -93,7 +92,9 @@ public void run() { if (!isExit() && !getDownload().isStopped() && is != null) { - while (!getDownload().isStopped() && !getDownload().getChunkwriter().isExit() && chunk.getOutputStream().size() < chunk.getSize() && (reads = is.read(buffer, 0, (max_reads = (int) (chunk.getSize() - chunk.getOutputStream().size())) <= buffer.length ? max_reads : buffer.length)) != -1) { + byte[] buffer = new byte[THROTTLE_SLICE_SIZE]; + + while (!getDownload().isStopped() && !getDownload().getChunkwriter().isExit() && chunk.getOutputStream().size() < chunk.getSize() && (reads = is.read(buffer, 0, Math.min((int) (chunk.getSize() - chunk.getOutputStream().size()), buffer.length))) != -1) { chunk.getOutputStream().write(buffer, 0, reads); getDownload().getPartialProgressQueue().add(reads); diff --git a/src/megabasterd/ChunkUploader.java b/src/megabasterd/ChunkUploader.java index f469927aa..8e6a349ea 100644 --- a/src/megabasterd/ChunkUploader.java +++ b/src/megabasterd/ChunkUploader.java @@ -109,8 +109,7 @@ public void run() { String worker_url = _upload.getUl_url(); Chunk chunk; - int reads, to_read, conta_error, re, http_status, tot_bytes_up; - byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE]; + int reads, conta_error, re, http_status, tot_bytes_up; boolean error; OutputStream out; @@ -119,14 +118,15 @@ public void run() { conta_error = 0; while (!_exit && !_upload.isStopped()) { - chunk = new Chunk(_upload.nextChunkId(), _upload.getFile_size(), worker_url, Transference.CHUNK_SIZE_MULTI); + chunk = new Chunk(_upload.nextChunkId(), _upload.getFile_size(), worker_url); f.seek(chunk.getOffset()); + byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE]; + do { - to_read = chunk.getSize() - chunk.getOutputStream().size() >= buffer.length ? buffer.length : (int) (chunk.getSize() - chunk.getOutputStream().size()); - re = f.read(buffer, 0, to_read); + re = f.read(buffer, 0, Math.min((int) (chunk.getSize() - chunk.getOutputStream().size()), buffer.length)); chunk.getOutputStream().write(buffer, 0, re); @@ -164,6 +164,7 @@ public CloseableHttpResponse call() throws IOException { THREAD_POOL.execute(futureTask); out = new ThrottledOutputStream(pipeout, _upload.getMain_panel().getStream_supervisor()); System.out.println(" Subiendo chunk " + chunk.getId() + " desde worker " + _id + "..."); + System.out.println(chunk.getUrl()); while (!_exit && !_upload.isStopped() && (reads = cis.read(buffer)) != -1) { out.write(buffer, 0, reads); diff --git a/src/megabasterd/ChunkUploaderMono.java b/src/megabasterd/ChunkUploaderMono.java index df45c175b..a1864e38e 100644 --- a/src/megabasterd/ChunkUploaderMono.java +++ b/src/megabasterd/ChunkUploaderMono.java @@ -50,8 +50,7 @@ public void run() { String worker_url = getUpload().getUl_url(); Chunk chunk; - int reads, to_read, conta_error, re, http_status, tot_bytes_up = -1; - byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE]; + int reads, conta_error, re, http_status, tot_bytes_up = -1; boolean error = false; try (CloseableHttpClient httpclient = MiscTools.getApacheKissHttpClient(); RandomAccessFile f = new RandomAccessFile(getUpload().getFile_name(), "r");) { @@ -63,16 +62,18 @@ public void run() { FutureTask futureTask = null; while (!isExit() && !getUpload().isStopped()) { + CloseableHttpResponse httpresponse = null; chunk = new Chunk(getUpload().nextChunkId(), getUpload().getFile_size(), null); f.seek(chunk.getOffset()); + byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE]; + do { - to_read = chunk.getSize() - chunk.getOutputStream().size() >= buffer.length ? buffer.length : (int) (chunk.getSize() - chunk.getOutputStream().size()); - re = f.read(buffer, 0, to_read); + re = f.read(buffer, 0, Math.min((int) (chunk.getSize() - chunk.getOutputStream().size()), buffer.length)); chunk.getOutputStream().write(buffer, 0, re); diff --git a/src/megabasterd/ChunkWriter.java b/src/megabasterd/ChunkWriter.java index ac05b0b49..85d8374b9 100644 --- a/src/megabasterd/ChunkWriter.java +++ b/src/megabasterd/ChunkWriter.java @@ -152,7 +152,7 @@ public void run() { private long calculateLastWrittenChunk(long temp_file_size) { if (temp_file_size > 3584 * 1024) { - return 7 + (long) Math.ceil((temp_file_size - 3584 * 1024) / (1024 * 1024 * Transference.CHUNK_SIZE_MULTI)); + return 7 + (long) Math.ceil((temp_file_size - 3584 * 1024) / (1024 * 1024 * (_download.isUse_slots() ? Download.CHUNK_SIZE_MULTI : 1))); } else { int i = 0, tot = 0; diff --git a/src/megabasterd/Download.java b/src/megabasterd/Download.java index dfe51ca87..21c96622b 100644 --- a/src/megabasterd/Download.java +++ b/src/megabasterd/Download.java @@ -59,6 +59,7 @@ public final class Download implements Transference, Runnable, SecureSingleThrea public static final boolean USE_SLOTS_DEFAULT = false; public static final int WORKERS_DEFAULT = 6; public static final boolean USE_MEGA_ACCOUNT_DOWN = false; + public static final int CHUNK_SIZE_MULTI = 10; private final MainPanel _main_panel; private volatile DownloadView _view = null; //lazy init @@ -629,7 +630,7 @@ public void run() { } else { getView().hideAllExceptStatus(); - exit_message = "OOOPS!! Something (bad) happened but... what?"; + exit_message = "OOOPS!! Something (bad) happened but... what? Progress: " + String.valueOf(_progress) + " File size: " + String.valueOf(_file_size); getView().printStatusError(exit_message); @@ -1162,7 +1163,7 @@ public void emergencyStopDownloader(String reason) { public long calculateMaxTempFileSize(long size) { if (size > 3584 * 1024) { - long reminder = (size - 3584 * 1024) % (1024 * 1024 * Transference.CHUNK_SIZE_MULTI); + long reminder = (size - 3584 * 1024) % (1024 * 1024 * (isUse_slots() ? Download.CHUNK_SIZE_MULTI : 1)); return reminder == 0 ? size : (size - reminder); } else { diff --git a/src/megabasterd/MainPanel.java b/src/megabasterd/MainPanel.java index f91f7422d..1aa45254d 100644 --- a/src/megabasterd/MainPanel.java +++ b/src/megabasterd/MainPanel.java @@ -57,7 +57,7 @@ */ public final class MainPanel { - public static final String VERSION = "2.24"; + public static final String VERSION = "2.25"; public static final int THROTTLE_SLICE_SIZE = 16 * 1024; public static final int DEFAULT_BYTE_BUFFER_SIZE = 16 * 1024; public static final int STREAMER_PORT = 1337; diff --git a/src/megabasterd/Transference.java b/src/megabasterd/Transference.java index 3ca099930..98c2b2127 100644 --- a/src/megabasterd/Transference.java +++ b/src/megabasterd/Transference.java @@ -9,13 +9,12 @@ public interface Transference { int MIN_WORKERS = 2; - int MAX_WORKERS = 20; + int MAX_WORKERS = 30; int MAX_SIM_TRANSFERENCES = 20; int SIM_TRANSFERENCES_DEFAULT = 2; boolean LIMIT_TRANSFERENCE_SPEED_DEFAULT = false; int MAX_TRANSFERENCE_SPEED_DEFAULT = 5; int MAX_WAIT_WORKERS_SHUTDOWN = 15; - int CHUNK_SIZE_MULTI = 10; Integer[] FATAL_ERROR_API_CODES = {-2, -8, -9, -10, -11, -12, -13, -14, -15, -16, 22, 23, 24, 25}; void start(); diff --git a/src/megabasterd/Upload.java b/src/megabasterd/Upload.java index a2b7fdac6..e871df726 100644 --- a/src/megabasterd/Upload.java +++ b/src/megabasterd/Upload.java @@ -381,11 +381,11 @@ public void provisionIt() { String[] fdata = new String(data).split("\\|"); - _last_chunk_id_dispatched = Long.parseLong(fdata[0]); + _progress = Long.parseLong(fdata[0]); - _progress = Long.parseLong(fdata[1]); + _last_chunk_id_dispatched = calculateLastUploadedChunk(_progress); - _saved_file_mac = bin2i32a(BASE642Bin(fdata[2])); + _saved_file_mac = bin2i32a(BASE642Bin(fdata[1])); } else if (temp_file.exists()) { @@ -1044,4 +1044,19 @@ public boolean isStatusError() { return _status_error; } + public long calculateLastUploadedChunk(long bytes_read) { + if (bytes_read > 3584 * 1024) { + return 7 + (long) Math.ceil((bytes_read - 3584 * 1024) / (1024 * 1024)); + } else { + int i = 0, tot = 0; + + while (tot < bytes_read) { + i++; + tot += i * 128 * 1024; + } + + return i; + } + } + } diff --git a/src/megabasterd/UploadMACGenerator.java b/src/megabasterd/UploadMACGenerator.java index 76e253b0c..278735640 100644 --- a/src/megabasterd/UploadMACGenerator.java +++ b/src/megabasterd/UploadMACGenerator.java @@ -105,7 +105,7 @@ public void run() { Chunk chunk; int[] file_iv = bin2i32a(_upload.getByte_file_iv()), int_block, file_mac = _upload.getSaved_file_mac(), mac_iv = CryptTools.AES_ZERO_IV_I32A; int reads; - byte[] byte_block = new byte[16]; + String temp_file_data; boolean new_chunk = false; boolean upload_workers_finish = false; @@ -130,89 +130,50 @@ public void run() { try { - if (chunk.getId() <= 7 || !_upload.isUse_slots()) { - - int[] chunk_mac = {file_iv[0], file_iv[1], file_iv[0], file_iv[1]}; + int[] chunk_mac = {file_iv[0], file_iv[1], file_iv[0], file_iv[1]}; + byte[] byte_block = new byte[16]; - while ((reads = chunk_is.read(byte_block)) != -1) { + while ((reads = chunk_is.read(byte_block)) != -1) { - if (reads < byte_block.length) { - for (int i = reads; i < byte_block.length; i++) { - byte_block[i] = 0; - } + if (reads < byte_block.length) { + for (int i = reads; i < byte_block.length; i++) { + byte_block[i] = 0; } - - int_block = bin2i32a(byte_block); - - for (int i = 0; i < chunk_mac.length; i++) { - chunk_mac[i] ^= int_block[i]; - } - - chunk_mac = bin2i32a(cryptor.doFinal(i32a2bin(chunk_mac))); } - for (int i = 0; i < file_mac.length; i++) { - file_mac[i] ^= chunk_mac[i]; - } - - file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac))); - - _bytes_read += chunk.getSize(); - - } else { - do { - int[] chunk_mac = {file_iv[0], file_iv[1], file_iv[0], file_iv[1]}; - - long chunk_size = 0; - - do { - - if ((reads = chunk_is.read(byte_block)) != -1) { - if (reads < byte_block.length) { - for (int i = reads; i < byte_block.length; i++) { - byte_block[i] = 0; - } - } + int_block = bin2i32a(byte_block); - int_block = bin2i32a(byte_block); - - for (int i = 0; i < chunk_mac.length; i++) { - chunk_mac[i] ^= int_block[i]; - } - - chunk_mac = bin2i32a(cryptor.doFinal(i32a2bin(chunk_mac))); - - chunk_size += reads; - } - } while (reads != -1 && chunk_size < 1024 * 1024); - - for (int i = 0; i < file_mac.length; i++) { - file_mac[i] ^= chunk_mac[i]; - } - - file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac))); + for (int i = 0; i < chunk_mac.length; i++) { + chunk_mac[i] ^= int_block[i]; + } - _bytes_read += chunk_size; + chunk_mac = bin2i32a(cryptor.doFinal(i32a2bin(chunk_mac))); + } - } while (reads != -1); + for (int i = 0; i < file_mac.length; i++) { + file_mac[i] ^= chunk_mac[i]; } + file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac))); + } catch (IOException | IllegalBlockSizeException | BadPaddingException ex) { getLogger(UploadMACGenerator.class.getName()).log(Level.SEVERE, null, ex); } - _chunk_queue.remove(chunk.getId()); + _bytes_read += chunk.getSize(); _last_chunk_id_read = chunk.getId(); + _chunk_queue.remove(chunk.getId()); + new_chunk = true; } if (!upload_workers_finish && new_chunk) { - temp_file_data = (String.valueOf(_last_chunk_id_read) + "|" + String.valueOf(_bytes_read) + "|" + Bin2BASE64(i32a2bin(file_mac))); + temp_file_data = (String.valueOf(_bytes_read) + "|" + Bin2BASE64(i32a2bin(file_mac))); - System.out.println("Macgenerator -> " + temp_file_data); + System.out.println("Macgenerator -> " + temp_file_data + " " + _upload.calculateLastUploadedChunk(_bytes_read) + " " + _last_chunk_id_read); temp_file_out = new FileOutputStream(temp_file);