From 1813eba8eb56930f404fdfe181eea1199ad0b264 Mon Sep 17 00:00:00 2001 From: Andrey Borysenko Date: Tue, 28 Feb 2023 17:56:07 +0200 Subject: [PATCH] cloud_py_api 0.1.5 (#80) * cloud_py_api 0.1.5 * psalm fixes --- CHANGELOG.md | 6 + appinfo/info.xml | 2 +- lib/Service/UtilsService.php | 223 +++++++++++++++++++++++++++++++++++ package-lock.json | 4 +- package.json | 2 +- 5 files changed, 233 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db1da963..6e7376cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [0.1.5 - 2023-02-25] + +### Changed + +- Changed binary package format from one file to folder to speedup startup + ## [0.1.4 - 2023-01-23] ### Removed diff --git a/appinfo/info.xml b/appinfo/info.xml index 5f9bf20d..ee7449d7 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -34,7 +34,7 @@ You can support us in several ways: [![Liberapay](https://liberapay.com/assets/widgets/donate.svg)](https://liberapay.com/cloud_py_api/donate) ]]> - 0.1.4 + 0.1.5 agpl Andrey Borysenko Alexander Piskun diff --git a/lib/Service/UtilsService.php b/lib/Service/UtilsService.php index d55d6253..067b5a0b 100644 --- a/lib/Service/UtilsService.php +++ b/lib/Service/UtilsService.php @@ -342,6 +342,229 @@ public function compareBinaryHash(string $url, string $binaryPath) { return true; } + /** + * Perform cURL to download python binary (in directory format) + * + * @param string $url + * @param array $binariesFolder appdata binaries folder + * @param string $appId target Application::APP_ID + * @param string $filename archive and extracted folder name + * @param bool $update flag to determine whether to update already downloaded binary or not + * + * @return array + */ + public function downloadPythonBinaryDir( + string $url, + array $binariesFolder, + string $appId, + string $filename = 'main', + bool $update = false + ): array { + if (isset($binariesFolder['success']) && $binariesFolder['success']) { + $dir = $binariesFolder['path'] . '/'; + } else { + return $binariesFolder; // Return getAppDataFolder result + } + $file_name = $filename . '.tar.gz'; + $save_file_loc = $dir . $file_name; + $shouldDownloadBinary = $this->compareBinaryDirectoryHashes($url, $binariesFolder, $appId); + + if (!file_exists($dir . $filename) || ($update && $shouldDownloadBinary)) { + $cURL = curl_init($url); + $fp = fopen($save_file_loc, 'wb'); + if ($fp) { + curl_setopt_array($cURL, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FILE => $fp, + CURLOPT_FOLLOWLOCATION => true, + ]); + curl_exec($cURL); + curl_close($cURL); + fclose($fp); + $unpacked = $this->unTarGz($binariesFolder, $filename . '.tar.gz'); + unlink($save_file_loc); + return [ + 'downloaded' => true, + 'unpacked' => $unpacked + ]; + } + } + + return [ + 'downloaded' => true, + 'unpacked' => true, + ]; + } + + /** + * Extract tar.gz file + * + * @param array $binariesFolder appdata binaries folder + * @param string $src_filename source tar.gz file name + * + * @return array + */ + public function unTarGz(array $binariesFolder, string $src_filename): array { + if (isset($binariesFolder['success']) && $binariesFolder['success']) { + $dir = $binariesFolder['path'] . '/'; + $src_file = $dir . $src_filename; + $phar = new \PharData($src_file); + $extracted = $phar->extractTo($dir, null, true); + $filename = $phar->getFilename(); + return [ + 'extracted' => $extracted, + 'filename' => $filename + ]; + } + return [ + 'extracted' => false, + ]; + } + + /** + * Perform cURL to get binary folder hashes sha256 sum + * + * @param string $url url to the binary hashsums file + * + * @return array + */ + public function downloadBinaryDirHashes(string $url): array { + $cURL = curl_init($url); + curl_setopt_array($cURL, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => true, + ]); + $binaryHash = curl_exec($cURL); + curl_close($cURL); + return [ + 'success' => $binaryHash != false, + 'binaryHash' => json_decode($binaryHash, true), + ]; + } + + /** + * Compare binary folder hashes from release. + * If hash not exists return `true` (download anyway) + * + * @param string $url + * @param array $binariesFolder + * + * @return bool + */ + public function compareBinaryDirectoryHashes( + string $url, array $binariesFolder, string $appId + ): bool { + $currentBinaryHashes = $this->getCurrentBinaryDirHashes($binariesFolder, $appId); + $newBinaryHashes = $this->downloadBinaryDirHashes(str_replace('.tar.gz', '.json', $url)); + if ($newBinaryHashes['success'] && $currentBinaryHashes['success']) { + // Skip hash check of archive file + $archiveFilename = $appId . '_' . $this->getBinaryName() . '.tar.gz'; + if (isset($newBinaryHashes['binaryHashes'][$archiveFilename])) { + unset($newBinaryHashes['binaryHashes'][$archiveFilename]); + } + foreach ($newBinaryHashes['binaryHashes'] as $filename => $hash) { + $fileExists = !isset($currentBinaryHashes[$filename]); + $currentHash = $currentBinaryHashes['binaryHashes'][$filename]; + $hashEqual = $currentHash == $hash; + if (!$fileExists || !$hashEqual) { + return true; + } + } + return false; + } + return true; + } + + /** + * Get current binary folder files hashes + * + * @param array $binariesFolder + * + * @return array + */ + public function getCurrentBinaryDirHashes(array $binariesFolder, string $appId): array { + $currentBinaryHashes = []; + $archiveFilename = $appId . '_' . $this->getBinaryName() . '.tar.gz'; + if (file_exists($binariesFolder['path'] . '/' . $archiveFilename)) { + $currentBinaryHashes[$archiveFilename] = hash_file( + 'sha256', + $binariesFolder['path'] . '/' . $archiveFilename + ); + } + $extractedBinaryFolder = $binariesFolder['path'] . '/' . $appId . '_'. $this->getBinaryName(); + $files = scandir($extractedBinaryFolder); + if ($files !== false) { + foreach ($files as $file) { + if ($file != '.' && $file != '..') { + // Get sha256 hash of each file + // If file is directory, get sha256 hash of each file in directory + if (is_dir($extractedBinaryFolder . '/' . $file)) { + $dirFiles = scandir($extractedBinaryFolder . '/' . $file); + $currentBinaryHashes = $this->getFolderHashes( + $dirFiles, + $file, + $extractedBinaryFolder . '/' . $file, + $currentBinaryHashes, + $appId + ); + } else { + $binaryFolderFilePath = $appId . '_' . $this->getBinaryName() . '/' . $file; + $currentBinaryHashes[$binaryFolderFilePath] = hash_file( + 'sha256', + $extractedBinaryFolder . '/' . $file + ); + } + } + } + } + return [ + 'success' => count($currentBinaryHashes) > 0, + 'binaryHashes' => $currentBinaryHashes + ]; + } + + /** + * Get sha256 hashes of each file in binary folder + * Recursive function call if file is directory + * + * @param array $files + * @param string $folder + * @param string $extractedBinaryFolder + * @param array $currentBinaryHashes + * @param string $appId + * + * @return array + */ + private function getFolderHashes( + array $files, + string $folder, + string $extractedBinaryFolder, + array $currentBinaryHashes, + string $appId + ): array { + foreach ($files as $file) { + if ($file != '.' && $file != '..') { + // Get sha256 hash of each file + // If file is directory, get sha256 hash of each file in directory + if (is_dir($extractedBinaryFolder . '/' . $file)) { + $dirFiles = scandir($extractedBinaryFolder . '/' . $file); + $currentBinaryHashes = $this->getFolderHashes( + $dirFiles, + $folder. '/' . $file, $extractedBinaryFolder . '/' . $file, + $currentBinaryHashes, $appId + ); + } else { + $binaryFolderFilePath = $appId . '_' + . $this->getBinaryName() . '/' . $folder . '/' . $file; + $currentBinaryHashes[$binaryFolderFilePath] = hash_file( + 'sha256', $extractedBinaryFolder . '/' . $file + ); + } + } + } + return $currentBinaryHashes; + } + /** * Perform cURL to get binary's sha256 sum * diff --git a/package-lock.json b/package-lock.json index 513d80ed..2543cadb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cloud_py_api", - "version": "0.1.4", + "version": "0.1.5", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cloud_py_api", - "version": "0.1.4", + "version": "0.1.5", "license": "agpl", "dependencies": { "@nextcloud/auth": "^2.0.0", diff --git a/package.json b/package.json index 1afd21a7..f24b8100 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cloud_py_api", "description": "Nextcloud Python API (Framework)", - "version": "0.1.4", + "version": "0.1.5", "keywords": [ "nextcloud", "python",