diff --git a/python-parallel-processing/01_java_vs_python/Fibonacci.java b/python-parallel-processing/01_java_vs_python/Fibonacci.java new file mode 100644 index 0000000000..8abc93b8cc --- /dev/null +++ b/python-parallel-processing/01_java_vs_python/Fibonacci.java @@ -0,0 +1,12 @@ +public class Fibonacci { + public static void main(String[] args) { + int cpus = Runtime.getRuntime().availableProcessors(); + for (int i = 0; i < cpus; i++) { + new Thread(() -> fib(45)).start(); + } + } + private static int fib(int n) { + return n < 2 ? n : fib(n - 2) + fib(n - 1); + } +} + diff --git a/python-parallel-processing/01_java_vs_python/fibonacci.py b/python-parallel-processing/01_java_vs_python/fibonacci.py new file mode 100644 index 0000000000..800e2793f2 --- /dev/null +++ b/python-parallel-processing/01_java_vs_python/fibonacci.py @@ -0,0 +1,10 @@ +import os +import threading + + +def fib(n): + return n if n < 2 else fib(n - 2) + fib(n - 1) + + +for _ in range(os.cpu_count()): + threading.Thread(target=fib, args=(35,)).start() diff --git a/python-parallel-processing/02_process-based_parallelism/01_fibonacci_multiprocessing.py b/python-parallel-processing/02_process-based_parallelism/01_fibonacci_multiprocessing.py new file mode 100644 index 0000000000..3d1cbb8ecc --- /dev/null +++ b/python-parallel-processing/02_process-based_parallelism/01_fibonacci_multiprocessing.py @@ -0,0 +1,10 @@ +import multiprocessing + + +def fib(n): + return n if n < 2 else fib(n - 2) + fib(n - 1) + + +if __name__ == "__main__": + for _ in range(multiprocessing.cpu_count()): + multiprocessing.Process(target=fib, args=(35,)).start() diff --git a/python-parallel-processing/02_process-based_parallelism/02_fibonacci_multiprocessing_pool.py b/python-parallel-processing/02_process-based_parallelism/02_fibonacci_multiprocessing_pool.py new file mode 100644 index 0000000000..cdd6ac8fbe --- /dev/null +++ b/python-parallel-processing/02_process-based_parallelism/02_fibonacci_multiprocessing_pool.py @@ -0,0 +1,12 @@ +import multiprocessing + + +def fib(n): + return n if n < 2 else fib(n - 2) + fib(n - 1) + + +if __name__ == "__main__": + with multiprocessing.Pool(processes=4) as pool: + results = pool.map(fib, range(40)) + for i, result in enumerate(results): + print(f"fib({i}) = {result}") diff --git a/python-parallel-processing/02_process-based_parallelism/03_fibonacci_concurrent_futures_process_pool.py b/python-parallel-processing/02_process-based_parallelism/03_fibonacci_concurrent_futures_process_pool.py new file mode 100644 index 0000000000..83631cd61a --- /dev/null +++ b/python-parallel-processing/02_process-based_parallelism/03_fibonacci_concurrent_futures_process_pool.py @@ -0,0 +1,12 @@ +from concurrent.futures import ProcessPoolExecutor + + +def fib(n): + return n if n < 2 else fib(n - 2) + fib(n - 1) + + +if __name__ == "__main__": + with ProcessPoolExecutor(max_workers=4) as executor: + results = executor.map(fib, range(40)) + for i, result in enumerate(results): + print(f"fib({i}) = {result}") diff --git a/python-parallel-processing/02_process-based_parallelism/04_echo_benchmark.py b/python-parallel-processing/02_process-based_parallelism/04_echo_benchmark.py new file mode 100644 index 0000000000..f42ea3a31c --- /dev/null +++ b/python-parallel-processing/02_process-based_parallelism/04_echo_benchmark.py @@ -0,0 +1,17 @@ +import time +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor + + +def echo(data): + return data + + +if __name__ == "__main__": + data = [complex(i, i) for i in range(15_000_000)] + for executor in ThreadPoolExecutor(), ProcessPoolExecutor(): + t1 = time.perf_counter() + with executor: + future = executor.submit(echo, data) + future.result() + t2 = time.perf_counter() + print(f"{type(executor).__name__:>20s}: {t2 - t1:.2f}s") diff --git a/python-parallel-processing/03_gil-immune_library/numpy_threads.py b/python-parallel-processing/03_gil-immune_library/numpy_threads.py new file mode 100644 index 0000000000..dddcfb4a50 --- /dev/null +++ b/python-parallel-processing/03_gil-immune_library/numpy_threads.py @@ -0,0 +1,5 @@ +import numpy as np + +rng = np.random.default_rng() +matrix = rng.random(size=(5000, 5000)) +matrix @ matrix diff --git a/python-parallel-processing/03_gil-immune_library/requirements.txt b/python-parallel-processing/03_gil-immune_library/requirements.txt new file mode 100644 index 0000000000..b26e81fba4 --- /dev/null +++ b/python-parallel-processing/03_gil-immune_library/requirements.txt @@ -0,0 +1,2 @@ +numpy + diff --git a/python-parallel-processing/04_extension_module/compile.sh b/python-parallel-processing/04_extension_module/compile.sh new file mode 100755 index 0000000000..d19dde69ce --- /dev/null +++ b/python-parallel-processing/04_extension_module/compile.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +gcc $(python3-config --cflags) -shared -fPIC -O3 -o fibmodule.so fibmodule.c + diff --git a/python-parallel-processing/04_extension_module/fibmodule.c b/python-parallel-processing/04_extension_module/fibmodule.c new file mode 100644 index 0000000000..7c003baec0 --- /dev/null +++ b/python-parallel-processing/04_extension_module/fibmodule.c @@ -0,0 +1,37 @@ +#include + +int fib(int n) { + return n < 2 ? n : fib(n - 2) + fib(n - 1); +} + +static PyObject* fibmodule_fib(PyObject* self, PyObject* args) { + int n, result; + + if (!PyArg_ParseTuple(args, "i", &n)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + result = fib(n); + Py_END_ALLOW_THREADS + + return Py_BuildValue("i", result); +} + +static PyMethodDef fib_methods[] = { + {"fib", fibmodule_fib, METH_VARARGS, "Calculate the nth Fibonacci"}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef fibmodule = { + PyModuleDef_HEAD_INIT, + "fibmodule", + "Efficient Fibonacci number calculator", + -1, + fib_methods +}; + +PyMODINIT_FUNC PyInit_fibmodule(void) { + return PyModule_Create(&fibmodule); +} + diff --git a/python-parallel-processing/04_extension_module/fibonacci_ext.py b/python-parallel-processing/04_extension_module/fibonacci_ext.py new file mode 100644 index 0000000000..9a97190cd9 --- /dev/null +++ b/python-parallel-processing/04_extension_module/fibonacci_ext.py @@ -0,0 +1,7 @@ +import os +import threading + +import fibmodule + +for _ in range(os.cpu_count()): + threading.Thread(target=fibmodule.fib, args=(45,)).start() diff --git a/python-parallel-processing/05_extension_module_cython/compile.sh b/python-parallel-processing/05_extension_module_cython/compile.sh new file mode 100755 index 0000000000..89b01bf358 --- /dev/null +++ b/python-parallel-processing/05_extension_module_cython/compile.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cythonize --inplace --annotate -3 fibmodule.py + diff --git a/python-parallel-processing/05_extension_module_cython/fibmodule.py b/python-parallel-processing/05_extension_module_cython/fibmodule.py new file mode 100644 index 0000000000..107519d113 --- /dev/null +++ b/python-parallel-processing/05_extension_module_cython/fibmodule.py @@ -0,0 +1,20 @@ +import cython + + +@cython.ccall +def fib(n: cython.int) -> cython.int: + with cython.nogil: + return _fib(n) + + +@cython.cfunc +@cython.nogil +@cython.exceptval(check=False) +def _fib(n: cython.int) -> cython.int: + return n if n < 2 else _fib(n - 2) + _fib(n - 1) + + +if cython.compiled: + print("Cython compiled this module") +else: + print("Cython didn't compile this module") diff --git a/python-parallel-processing/05_extension_module_cython/fibmodule.pyx b/python-parallel-processing/05_extension_module_cython/fibmodule.pyx new file mode 100644 index 0000000000..0c77bb1151 --- /dev/null +++ b/python-parallel-processing/05_extension_module_cython/fibmodule.pyx @@ -0,0 +1,8 @@ +cpdef int fib(int n): + with nogil: + return _fib(n) + + +cdef int _fib(int n) noexcept nogil: + return n if n < 2 else _fib(n - 2) + _fib(n - 1) + diff --git a/python-parallel-processing/05_extension_module_cython/fibonacci_ext.py b/python-parallel-processing/05_extension_module_cython/fibonacci_ext.py new file mode 100644 index 0000000000..9a97190cd9 --- /dev/null +++ b/python-parallel-processing/05_extension_module_cython/fibonacci_ext.py @@ -0,0 +1,7 @@ +import os +import threading + +import fibmodule + +for _ in range(os.cpu_count()): + threading.Thread(target=fibmodule.fib, args=(45,)).start() diff --git a/python-parallel-processing/05_extension_module_cython/requirements.txt b/python-parallel-processing/05_extension_module_cython/requirements.txt new file mode 100644 index 0000000000..05f41acf6f --- /dev/null +++ b/python-parallel-processing/05_extension_module_cython/requirements.txt @@ -0,0 +1,2 @@ +cython + diff --git a/python-parallel-processing/06_ctypes/compile.sh b/python-parallel-processing/06_ctypes/compile.sh new file mode 100755 index 0000000000..3a05054236 --- /dev/null +++ b/python-parallel-processing/06_ctypes/compile.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +gcc -shared -fPIC -O3 -o fibonacci.so fibonacci.c diff --git a/python-parallel-processing/06_ctypes/fibonacci.c b/python-parallel-processing/06_ctypes/fibonacci.c new file mode 100644 index 0000000000..7255d3fdcd --- /dev/null +++ b/python-parallel-processing/06_ctypes/fibonacci.c @@ -0,0 +1,3 @@ +int fib(int n) { + return n < 2 ? n : fib(n - 2) + fib(n - 1); +} diff --git a/python-parallel-processing/06_ctypes/fibonacci_ctypes.py b/python-parallel-processing/06_ctypes/fibonacci_ctypes.py new file mode 100644 index 0000000000..fc83cd49db --- /dev/null +++ b/python-parallel-processing/06_ctypes/fibonacci_ctypes.py @@ -0,0 +1,12 @@ +import ctypes +import os +import threading + +fibonacci = ctypes.CDLL("./fibonacci.so") + +fib = fibonacci.fib +fib.argtypes = (ctypes.c_int,) +fib.restype = ctypes.c_int + +for _ in range(os.cpu_count()): + threading.Thread(target=fib, args=(45,)).start() diff --git a/python-parallel-processing/07_image_processing/image_processing.py b/python-parallel-processing/07_image_processing/image_processing.py new file mode 100644 index 0000000000..6875634c9b --- /dev/null +++ b/python-parallel-processing/07_image_processing/image_processing.py @@ -0,0 +1,126 @@ +# image_processing.py + +import argparse +import pathlib +import time +import tkinter as tk +import tkinter.ttk as ttk + +import numpy as np +import PIL.Image +import PIL.ImageTk + +import parallel + + +class AppWindow(tk.Tk): + def __init__(self, image: PIL.Image.Image) -> None: + super().__init__() + + # Main window + self.title("Exposure and Gamma Correction") + self.resizable(False, False) + + # Parameters frame + self.frame = ttk.LabelFrame(self, text="Parameters") + self.frame.pack(fill=tk.X, padx=10, pady=10) + self.frame.columnconfigure(0, weight=0) + self.frame.columnconfigure(1, weight=1) + + # EV slider + self.var_ev = tk.DoubleVar(value=0) + ev_label = ttk.Label(self.frame, text="Exposure:") + ev_label.grid(row=0, column=0, sticky=tk.W, padx=10, pady=10) + ev_slider = ttk.Scale( + self.frame, + from_=-1, + to=1, + orient=tk.HORIZONTAL, + variable=self.var_ev, + ) + ev_slider.bind("", self.on_slide) + ev_slider.grid(row=0, column=1, sticky=tk.W + tk.E, padx=10, pady=10) + + # Gamma slider + self.var_gamma = tk.DoubleVar(value=1) + gamma_label = ttk.Label(self.frame, text="Gamma:") + gamma_label.grid(row=1, column=0, sticky=tk.W, padx=10, pady=10) + gamma_slider = ttk.Scale( + self.frame, + from_=0.1, + to=2, + orient=tk.HORIZONTAL, + variable=self.var_gamma, + ) + gamma_slider.bind("", self.on_slide) + gamma_slider.grid( + row=1, column=1, sticky=tk.W + tk.E, padx=10, pady=10 + ) + + # Image preview + self.preview = ttk.Label(self, relief=tk.SUNKEN) + self.preview.pack(padx=10, pady=10) + + # Status bar + self.var_status = tk.StringVar() + status_bar = ttk.Label( + self, anchor=tk.W, relief=tk.SUNKEN, textvariable=self.var_status + ) + status_bar.pack(side=tk.BOTTOM, fill=tk.X) + + # Image pixels + self.pixels = np.array(image) + self.update() + self.show_preview(image) + + self.mainloop() + + def on_slide(self, *args, **kwargs) -> None: + # Get parameters + ev = 2.0 ** self.var_ev.get() + gamma = 1.0 / self.var_gamma.get() + + # Process pixels + t1 = time.perf_counter() + pixels = self.pixels.copy() + parallel.process(pixels, ev, gamma) + t2 = time.perf_counter() + + # Render preview + image = PIL.Image.fromarray(pixels) + self.show_preview(image) + t3 = time.perf_counter() + + # Update status + self.var_status.set( + f"Processed in {(t2 - t1) * 1000:.0f} ms " + f"(Rendered in {(t3 - t1) * 1000:.0f} ms)" + ) + + def show_preview(self, image: PIL.Image.Image) -> None: + scale = 0.75 + offset = 2.0 * self.frame.winfo_height() + image.thumbnail( + ( + int(self.winfo_screenwidth() * scale), + int(self.winfo_screenheight() * scale - offset), + ) + ) + image_tk = PIL.ImageTk.PhotoImage(image) + self.preview.configure(image=image_tk) + self.preview.image = image_tk + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("image_path", type=pathlib.Path) + return parser.parse_args() + + +def main(args: argparse.Namespace) -> None: + with PIL.Image.open(args.image_path) as image: + AppWindow(image) + + +if __name__ == "__main__": + main(parse_args()) diff --git a/python-parallel-processing/07_image_processing/image_processing_bonus.py b/python-parallel-processing/07_image_processing/image_processing_bonus.py new file mode 100644 index 0000000000..32b69b7013 --- /dev/null +++ b/python-parallel-processing/07_image_processing/image_processing_bonus.py @@ -0,0 +1,179 @@ +# image_processing.py + +import argparse +import enum +import pathlib +import time +import tkinter as tk +import tkinter.ttk as ttk + +import numpy as np +import PIL.Image +import PIL.ImageTk + +import parallel + + +class ProcessingMode(enum.StrEnum): + PYTHON = "Python" + NUMPY = "NumPy" + PARALLEL = "Parallel (GIL-Free)" + + +class AppWindow(tk.Tk): + def __init__(self, image: PIL.Image.Image) -> None: + super().__init__() + + # Main window + self.title("Exposure and Gamma Correction") + self.resizable(False, False) + + # Parameters frame + self.frame = ttk.LabelFrame(self, text="Parameters") + self.frame.pack(fill=tk.X, padx=10, pady=10) + self.frame.columnconfigure(0, weight=0) + self.frame.columnconfigure(1, weight=1) + + # Dropdown + self.var_mode = tk.StringVar(value=ProcessingMode.PYTHON) + mode_label = ttk.Label(self.frame, text="Mode:") + mode_label.grid(row=0, column=0, sticky=tk.W, padx=10, pady=10) + mode_dropdown = ttk.Combobox( + self.frame, + textvariable=self.var_mode, + values=list(ProcessingMode), + state="readonly", + ) + mode_dropdown.grid( + row=0, column=1, sticky=tk.W + tk.E, padx=10, pady=10 + ) + + # EV slider + self.var_ev = tk.DoubleVar(value=0) + ev_label = ttk.Label(self.frame, text="Exposure:") + ev_label.grid(row=1, column=0, sticky=tk.W, padx=10, pady=10) + ev_slider = ttk.Scale( + self.frame, + from_=-1, + to=1, + orient=tk.HORIZONTAL, + variable=self.var_ev, + ) + ev_slider.bind("", self.on_slide) + ev_slider.grid(row=1, column=1, sticky=tk.W + tk.E, padx=10, pady=10) + + # Gamma slider + self.var_gamma = tk.DoubleVar(value=1) + gamma_label = ttk.Label(self.frame, text="Gamma:") + gamma_label.grid(row=2, column=0, sticky=tk.W, padx=10, pady=10) + gamma_slider = ttk.Scale( + self.frame, + from_=0.1, + to=2, + orient=tk.HORIZONTAL, + variable=self.var_gamma, + ) + gamma_slider.bind("", self.on_slide) + gamma_slider.grid( + row=2, column=1, sticky=tk.W + tk.E, padx=10, pady=10 + ) + + # Image preview + self.preview = ttk.Label(self, relief=tk.SUNKEN) + self.preview.pack(padx=10, pady=10) + + # Status bar + self.var_status = tk.StringVar() + status_bar = ttk.Label( + self, + anchor=tk.W, + relief=tk.SUNKEN, + textvariable=self.var_status, + ) + status_bar.pack(side=tk.BOTTOM, fill=tk.X) + + # Image pixels + self.pixels = np.array(image) + self.update() + self.show_preview(image) + + self.mainloop() + + def on_slide(self, *args, **kwargs) -> None: + # Get parameters + ev = 2.0 ** self.var_ev.get() + gamma = 1.0 / self.var_gamma.get() + + # Process pixels + t1 = time.perf_counter() + pixels = self.process(ev, gamma) + t2 = time.perf_counter() + + # Render preview + image = PIL.Image.fromarray(pixels) + self.show_preview(image) + t3 = time.perf_counter() + + # Update status + self.var_status.set( + f"Processed in {(t2 - t1) * 1000:.0f} ms " + f"(Rendered in {(t3 - t1) * 1000:.0f} ms)" + ) + + def show_preview(self, image: PIL.Image.Image) -> None: + scale = 0.75 + offset = 2.0 * self.frame.winfo_height() + image.thumbnail( + ( + int(self.winfo_screenwidth() * scale), + int(self.winfo_screenheight() * scale - offset), + ) + ) + image_tk = PIL.ImageTk.PhotoImage(image) + self.preview.configure(image=image_tk) + self.preview.image = image_tk + + def process(self, ev: float, gamma: float) -> np.ndarray: + match mode := self.var_mode.get(): + case ProcessingMode.PYTHON: + return process_python(self.pixels, ev, gamma) + case ProcessingMode.NUMPY: + return process_numpy(self.pixels, ev, gamma) + case ProcessingMode.PARALLEL: + parallel.process(pixels := self.pixels.copy(), ev, gamma) + return pixels + case _: + raise ValueError(f"Invalid mode: {mode}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("image_path", type=pathlib.Path) + return parser.parse_args() + + +def main(args: argparse.Namespace) -> None: + with PIL.Image.open(args.image_path) as image: + AppWindow(image) + + +def process_python(pixels: np.ndarray, ev: float, gamma: float) -> np.ndarray: + lookup_table = [ + int(min(max(0, (((i / 255.0) * ev) ** gamma) * 255), 255)) + for i in range(256) + ] + values = [lookup_table[x] for x in pixels.flat] + return np.array(values).astype(np.uint8).reshape(pixels.shape) + + +def process_numpy(pixels: np.ndarray, ev: float, gamma: float) -> np.ndarray: + lookup_table = ( + ((((np.arange(256) / 255.0) * ev) ** gamma) * 255) + .clip(0, 255) + .astype(np.uint8) + ) + return lookup_table[pixels] + + +if __name__ == "__main__": + main(parse_args()) diff --git a/python-parallel-processing/07_image_processing/parallel/__init__.py b/python-parallel-processing/07_image_processing/parallel/__init__.py new file mode 100644 index 0000000000..1e1f52bac3 --- /dev/null +++ b/python-parallel-processing/07_image_processing/parallel/__init__.py @@ -0,0 +1,39 @@ +# parallel/__init__.py + +import ctypes +import pathlib +import threading + +import numpy as np + +# Load the compiled C library +library_path = str(pathlib.Path(__file__).parent / f"{__package__}.so") +library = ctypes.CDLL(library_path) + +# Define a C data type for "unsigned char*" +UnsignedByteArrayPointer = ctypes.POINTER(ctypes.c_ubyte) + +# Define a prototype for the C function +library.process.argtypes = [ + UnsignedByteArrayPointer, # unsigned char* pixels + ctypes.c_int, # int length + ctypes.c_int, # int offset + ctypes.c_float, # float ev + ctypes.c_float, # float gamma +] + + +# Define a wrapper function accessible from Python +def process(pixels: np.ndarray, ev: float, gamma: float) -> None: + pointer = pixels.ctypes.data_as(UnsignedByteArrayPointer) + threads = [ + threading.Thread( + target=library.process, + args=(pointer, pixels.size, offset, ev, gamma), + ) + for offset in range(3) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() diff --git a/python-parallel-processing/07_image_processing/parallel/compile.sh b/python-parallel-processing/07_image_processing/parallel/compile.sh new file mode 100755 index 0000000000..b89f6d31c3 --- /dev/null +++ b/python-parallel-processing/07_image_processing/parallel/compile.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +gcc -shared -fPIC -O3 -o parallel.so parallel.c diff --git a/python-parallel-processing/07_image_processing/parallel/parallel.c b/python-parallel-processing/07_image_processing/parallel/parallel.c new file mode 100644 index 0000000000..a94425a5c2 --- /dev/null +++ b/python-parallel-processing/07_image_processing/parallel/parallel.c @@ -0,0 +1,22 @@ +// parallel/parallel.c + +#include + +void process( + unsigned char* pixels, + int length, + int offset, + float ev, + float gamma) +{ + unsigned char lookup_table[256]; + + for (int i = 0; i < 256; i++) { + float value = powf((i / 255.0f * ev), gamma) * 255.0f; + lookup_table[i] = (unsigned char) fmin(fmax(0.0f, value), 255.0f); + } + + for (int i = offset; i < length; i += 3) { + pixels[i] = lookup_table[pixels[i]]; + } +} diff --git a/python-parallel-processing/07_image_processing/requirements.txt b/python-parallel-processing/07_image_processing/requirements.txt new file mode 100644 index 0000000000..0d33a9eee5 --- /dev/null +++ b/python-parallel-processing/07_image_processing/requirements.txt @@ -0,0 +1,2 @@ +numpy +pillow diff --git a/python-parallel-processing/07_image_processing/sample_image_big.jpg b/python-parallel-processing/07_image_processing/sample_image_big.jpg new file mode 100755 index 0000000000..0e1ff91a4e Binary files /dev/null and b/python-parallel-processing/07_image_processing/sample_image_big.jpg differ diff --git a/python-parallel-processing/07_image_processing/sample_image_small.jpg b/python-parallel-processing/07_image_processing/sample_image_small.jpg new file mode 100644 index 0000000000..689d129b8f Binary files /dev/null and b/python-parallel-processing/07_image_processing/sample_image_small.jpg differ diff --git a/python-parallel-processing/README.md b/python-parallel-processing/README.md new file mode 100644 index 0000000000..1b79be88cf --- /dev/null +++ b/python-parallel-processing/README.md @@ -0,0 +1,3 @@ +# Bypassing the GIL for Parallel Processing in Python + +This folder contains sample code for the Real Python tutorial [Bypassing the GIL for Parallel Processing in Python](https://realpython.com/python-parallel-processing/).