2.25

-Upload multi-slot multi-size remove
tonikelope · Oct 28, 2017 · 82b84af · 82b84af
1 parent 96520d7
commit 82b84af
Show file tree

Hide file tree

Showing 10 changed files with 64 additions and 84 deletions.
diff --git a/src/megabasterd/ChunkDownloader.java b/src/megabasterd/ChunkDownloader.java
@@ -93,7 +93,6 @@ public void run() {
         String worker_url = null;
         Chunk chunk;
         int reads, conta_error, http_status;
-        byte[] buffer = new byte[THROTTLE_SLICE_SIZE];
         InputStream is;
         boolean error;
 
@@ -110,7 +109,7 @@ public void run() {
                     worker_url = _download.getDownloadUrlForWorker();
                 }
 
-                chunk = new Chunk(_download.nextChunkId(), _download.getFile_size(), worker_url, Transference.CHUNK_SIZE_MULTI);
+                chunk = new Chunk(_download.nextChunkId(), _download.getFile_size(), worker_url, Download.CHUNK_SIZE_MULTI);
 
                 HttpGet httpget = new HttpGet(new URI(chunk.getUrl()));
 
@@ -131,6 +130,8 @@ public void run() {
 
                         } else {
 
+                            byte[] buffer = new byte[THROTTLE_SLICE_SIZE];
+
                             while (!_exit && !_download.isStopped() && !_download.getChunkwriter().isExit() && chunk.getOutputStream().size() < chunk.getSize() && (reads = is.read(buffer)) != -1) {
 
                                 chunk.getOutputStream().write(buffer, 0, reads);

diff --git a/src/megabasterd/ChunkDownloaderMono.java b/src/megabasterd/ChunkDownloaderMono.java
@@ -28,8 +28,7 @@ public ChunkDownloaderMono(Download download) {
     public void run() {
         String worker_url = null;
         Chunk chunk;
-        int reads, max_reads, conta_error, http_status = 200;
-        byte[] buffer = new byte[THROTTLE_SLICE_SIZE];
+        int reads, conta_error, http_status = 200;
         boolean error;
         HttpGet httpget = null;
         CloseableHttpResponse httpresponse = null;
@@ -93,7 +92,9 @@ public void run() {
 
                         if (!isExit() && !getDownload().isStopped() && is != null) {
 
-                            while (!getDownload().isStopped() && !getDownload().getChunkwriter().isExit() && chunk.getOutputStream().size() < chunk.getSize() && (reads = is.read(buffer, 0, (max_reads = (int) (chunk.getSize() - chunk.getOutputStream().size())) <= buffer.length ? max_reads : buffer.length)) != -1) {
+                            byte[] buffer = new byte[THROTTLE_SLICE_SIZE];
+
+                            while (!getDownload().isStopped() && !getDownload().getChunkwriter().isExit() && chunk.getOutputStream().size() < chunk.getSize() && (reads = is.read(buffer, 0, Math.min((int) (chunk.getSize() - chunk.getOutputStream().size()), buffer.length))) != -1) {
                                 chunk.getOutputStream().write(buffer, 0, reads);
 
                                 getDownload().getPartialProgressQueue().add(reads);

diff --git a/src/megabasterd/ChunkUploader.java b/src/megabasterd/ChunkUploader.java
@@ -109,8 +109,7 @@ public void run() {
 
         String worker_url = _upload.getUl_url();
         Chunk chunk;
-        int reads, to_read, conta_error, re, http_status, tot_bytes_up;
-        byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE];
+        int reads, conta_error, re, http_status, tot_bytes_up;
         boolean error;
         OutputStream out;
 
@@ -119,14 +118,15 @@ public void run() {
             conta_error = 0;
 
             while (!_exit && !_upload.isStopped()) {
-                chunk = new Chunk(_upload.nextChunkId(), _upload.getFile_size(), worker_url, Transference.CHUNK_SIZE_MULTI);
+                chunk = new Chunk(_upload.nextChunkId(), _upload.getFile_size(), worker_url);
 
                 f.seek(chunk.getOffset());
 
+                byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE];
+
                 do {
-                    to_read = chunk.getSize() - chunk.getOutputStream().size() >= buffer.length ? buffer.length : (int) (chunk.getSize() - chunk.getOutputStream().size());
 
-                    re = f.read(buffer, 0, to_read);
+                    re = f.read(buffer, 0, Math.min((int) (chunk.getSize() - chunk.getOutputStream().size()), buffer.length));
 
                     chunk.getOutputStream().write(buffer, 0, re);
 
@@ -164,6 +164,7 @@ public CloseableHttpResponse call() throws IOException {
                             THREAD_POOL.execute(futureTask);
                             out = new ThrottledOutputStream(pipeout, _upload.getMain_panel().getStream_supervisor());
                             System.out.println(" Subiendo chunk " + chunk.getId() + " desde worker " + _id + "...");
+                            System.out.println(chunk.getUrl());
                             while (!_exit && !_upload.isStopped() && (reads = cis.read(buffer)) != -1) {
                                 out.write(buffer, 0, reads);
 

diff --git a/src/megabasterd/ChunkUploaderMono.java b/src/megabasterd/ChunkUploaderMono.java
@@ -50,8 +50,7 @@ public void run() {
 
         String worker_url = getUpload().getUl_url();
         Chunk chunk;
-        int reads, to_read, conta_error, re, http_status, tot_bytes_up = -1;
-        byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE];
+        int reads, conta_error, re, http_status, tot_bytes_up = -1;
         boolean error = false;
 
         try (CloseableHttpClient httpclient = MiscTools.getApacheKissHttpClient(); RandomAccessFile f = new RandomAccessFile(getUpload().getFile_name(), "r");) {
@@ -63,16 +62,18 @@ public void run() {
             FutureTask<CloseableHttpResponse> futureTask = null;
 
             while (!isExit() && !getUpload().isStopped()) {
+
                 CloseableHttpResponse httpresponse = null;
 
                 chunk = new Chunk(getUpload().nextChunkId(), getUpload().getFile_size(), null);
 
                 f.seek(chunk.getOffset());
 
+                byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE];
+
                 do {
-                    to_read = chunk.getSize() - chunk.getOutputStream().size() >= buffer.length ? buffer.length : (int) (chunk.getSize() - chunk.getOutputStream().size());
 
-                    re = f.read(buffer, 0, to_read);
+                    re = f.read(buffer, 0, Math.min((int) (chunk.getSize() - chunk.getOutputStream().size()), buffer.length));
 
                     chunk.getOutputStream().write(buffer, 0, re);
 

diff --git a/src/megabasterd/ChunkWriter.java b/src/megabasterd/ChunkWriter.java
@@ -152,7 +152,7 @@ public void run() {
 
     private long calculateLastWrittenChunk(long temp_file_size) {
         if (temp_file_size > 3584 * 1024) {
-            return 7 + (long) Math.ceil((temp_file_size - 3584 * 1024) / (1024 * 1024 * Transference.CHUNK_SIZE_MULTI));
+            return 7 + (long) Math.ceil((temp_file_size - 3584 * 1024) / (1024 * 1024 * (_download.isUse_slots() ? Download.CHUNK_SIZE_MULTI : 1)));
         } else {
             int i = 0, tot = 0;
 

diff --git a/src/megabasterd/Download.java b/src/megabasterd/Download.java
@@ -59,6 +59,7 @@ public final class Download implements Transference, Runnable, SecureSingleThrea
     public static final boolean USE_SLOTS_DEFAULT = false;
     public static final int WORKERS_DEFAULT = 6;
     public static final boolean USE_MEGA_ACCOUNT_DOWN = false;
+    public static final int CHUNK_SIZE_MULTI = 10;
 
     private final MainPanel _main_panel;
     private volatile DownloadView _view = null; //lazy init
@@ -629,7 +630,7 @@ public void run() {
                         } else {
                             getView().hideAllExceptStatus();
 
-                            exit_message = "OOOPS!! Something (bad) happened but... what?";
+                            exit_message = "OOOPS!! Something (bad) happened but... what? Progress: " + String.valueOf(_progress) + " File size: " + String.valueOf(_file_size);
 
                             getView().printStatusError(exit_message);
 
@@ -1162,7 +1163,7 @@ public void emergencyStopDownloader(String reason) {
 
     public long calculateMaxTempFileSize(long size) {
         if (size > 3584 * 1024) {
-            long reminder = (size - 3584 * 1024) % (1024 * 1024 * Transference.CHUNK_SIZE_MULTI);
+            long reminder = (size - 3584 * 1024) % (1024 * 1024 * (isUse_slots() ? Download.CHUNK_SIZE_MULTI : 1));
 
             return reminder == 0 ? size : (size - reminder);
         } else {

diff --git a/src/megabasterd/MainPanel.java b/src/megabasterd/MainPanel.java
@@ -57,7 +57,7 @@
  */
 public final class MainPanel {
 
-    public static final String VERSION = "2.24";
+    public static final String VERSION = "2.25";
     public static final int THROTTLE_SLICE_SIZE = 16 * 1024;
     public static final int DEFAULT_BYTE_BUFFER_SIZE = 16 * 1024;
     public static final int STREAMER_PORT = 1337;

diff --git a/src/megabasterd/Transference.java b/src/megabasterd/Transference.java
@@ -9,13 +9,12 @@
 public interface Transference {
 
     int MIN_WORKERS = 2;
-    int MAX_WORKERS = 20;
+    int MAX_WORKERS = 30;
     int MAX_SIM_TRANSFERENCES = 20;
     int SIM_TRANSFERENCES_DEFAULT = 2;
     boolean LIMIT_TRANSFERENCE_SPEED_DEFAULT = false;
     int MAX_TRANSFERENCE_SPEED_DEFAULT = 5;
     int MAX_WAIT_WORKERS_SHUTDOWN = 15;
-    int CHUNK_SIZE_MULTI = 10;
     Integer[] FATAL_ERROR_API_CODES = {-2, -8, -9, -10, -11, -12, -13, -14, -15, -16, 22, 23, 24, 25};
 
     void start();

diff --git a/src/megabasterd/Upload.java b/src/megabasterd/Upload.java
@@ -381,11 +381,11 @@ public void provisionIt() {
 
                     String[] fdata = new String(data).split("\\|");
 
-                    _last_chunk_id_dispatched = Long.parseLong(fdata[0]);
+                    _progress = Long.parseLong(fdata[0]);
 
-                    _progress = Long.parseLong(fdata[1]);
+                    _last_chunk_id_dispatched = calculateLastUploadedChunk(_progress);
 
-                    _saved_file_mac = bin2i32a(BASE642Bin(fdata[2]));
+                    _saved_file_mac = bin2i32a(BASE642Bin(fdata[1]));
 
                 } else if (temp_file.exists()) {
 
@@ -1044,4 +1044,19 @@ public boolean isStatusError() {
         return _status_error;
     }
 
+    public long calculateLastUploadedChunk(long bytes_read) {
+        if (bytes_read > 3584 * 1024) {
+            return 7 + (long) Math.ceil((bytes_read - 3584 * 1024) / (1024 * 1024));
+        } else {
+            int i = 0, tot = 0;
+
+            while (tot < bytes_read) {
+                i++;
+                tot += i * 128 * 1024;
+            }
+
+            return i;
+        }
+    }
+
 }
diff --git a/src/megabasterd/UploadMACGenerator.java b/src/megabasterd/UploadMACGenerator.java
@@ -105,7 +105,7 @@ public void run() {
             Chunk chunk;
             int[] file_iv = bin2i32a(_upload.getByte_file_iv()), int_block, file_mac = _upload.getSaved_file_mac(), mac_iv = CryptTools.AES_ZERO_IV_I32A;
             int reads;
-            byte[] byte_block = new byte[16];
+
             String temp_file_data;
             boolean new_chunk = false;
             boolean upload_workers_finish = false;
@@ -130,89 +130,50 @@ public void run() {
 
                     try {
 
-                        if (chunk.getId() <= 7 || !_upload.isUse_slots()) {
-
-                            int[] chunk_mac = {file_iv[0], file_iv[1], file_iv[0], file_iv[1]};
+                        int[] chunk_mac = {file_iv[0], file_iv[1], file_iv[0], file_iv[1]};
+                        byte[] byte_block = new byte[16];
 
-                            while ((reads = chunk_is.read(byte_block)) != -1) {
+                        while ((reads = chunk_is.read(byte_block)) != -1) {
 
-                                if (reads < byte_block.length) {
-                                    for (int i = reads; i < byte_block.length; i++) {
-                                        byte_block[i] = 0;
-                                    }
+                            if (reads < byte_block.length) {
+                                for (int i = reads; i < byte_block.length; i++) {
+                                    byte_block[i] = 0;
                                 }
-
-                                int_block = bin2i32a(byte_block);
-
-                                for (int i = 0; i < chunk_mac.length; i++) {
-                                    chunk_mac[i] ^= int_block[i];
-                                }
-
-                                chunk_mac = bin2i32a(cryptor.doFinal(i32a2bin(chunk_mac)));
                             }
 
-                            for (int i = 0; i < file_mac.length; i++) {
-                                file_mac[i] ^= chunk_mac[i];
-                            }
-
-                            file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
-
-                            _bytes_read += chunk.getSize();
-
-                        } else {
-                            do {
-                                int[] chunk_mac = {file_iv[0], file_iv[1], file_iv[0], file_iv[1]};
-
-                                long chunk_size = 0;
-
-                                do {
-
-                                    if ((reads = chunk_is.read(byte_block)) != -1) {
-                                        if (reads < byte_block.length) {
-                                            for (int i = reads; i < byte_block.length; i++) {
-                                                byte_block[i] = 0;
-                                            }
-                                        }
+                            int_block = bin2i32a(byte_block);
 
-                                        int_block = bin2i32a(byte_block);
-
-                                        for (int i = 0; i < chunk_mac.length; i++) {
-                                            chunk_mac[i] ^= int_block[i];
-                                        }
-
-                                        chunk_mac = bin2i32a(cryptor.doFinal(i32a2bin(chunk_mac)));
-
-                                        chunk_size += reads;
-                                    }
-                                } while (reads != -1 && chunk_size < 1024 * 1024);
-
-                                for (int i = 0; i < file_mac.length; i++) {
-                                    file_mac[i] ^= chunk_mac[i];
-                                }
-
-                                file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
+                            for (int i = 0; i < chunk_mac.length; i++) {
+                                chunk_mac[i] ^= int_block[i];
+                            }
 
-                                _bytes_read += chunk_size;
+                            chunk_mac = bin2i32a(cryptor.doFinal(i32a2bin(chunk_mac)));
+                        }
 
-                            } while (reads != -1);
+                        for (int i = 0; i < file_mac.length; i++) {
+                            file_mac[i] ^= chunk_mac[i];
                         }
 
+                        file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
+
                     } catch (IOException | IllegalBlockSizeException | BadPaddingException ex) {
                         getLogger(UploadMACGenerator.class.getName()).log(Level.SEVERE, null, ex);
                     }
 
-                    _chunk_queue.remove(chunk.getId());
+                    _bytes_read += chunk.getSize();
 
                     _last_chunk_id_read = chunk.getId();
 
+                    _chunk_queue.remove(chunk.getId());
+
                     new_chunk = true;
                 }
 
                 if (!upload_workers_finish && new_chunk) {
 
-                    temp_file_data = (String.valueOf(_last_chunk_id_read) + "|" + String.valueOf(_bytes_read) + "|" + Bin2BASE64(i32a2bin(file_mac)));
+                    temp_file_data = (String.valueOf(_bytes_read) + "|" + Bin2BASE64(i32a2bin(file_mac)));
 
-                    System.out.println("Macgenerator -> " + temp_file_data);
+                    System.out.println("Macgenerator -> " + temp_file_data + " " + _upload.calculateLastUploadedChunk(_bytes_read) + " " + _last_chunk_id_read);
 
                     temp_file_out = new FileOutputStream(temp_file);