diff --git a/README.rst b/README.rst index 7be90b5..e9ae013 100644 --- a/README.rst +++ b/README.rst @@ -43,10 +43,12 @@ Run ``flycheck-select-checker``, pick ``python-coverage``. Coverage data ============= -This package reads the XML output produced by Python's ``coverage`` -package. Usually this file is named ``coverage.xml``. +This package can read both the SQLite ``.coverage`` file created by Python’s +``coverage`` package, and the XML report it produces. The latter is preferred if +present, because the ``coverage xml`` command does some additional analysis to +ignore docstrings etc. -With plain `coverage`: +With plain ``coverage``, generate the XML file like this: .. code-block:: shell @@ -61,7 +63,7 @@ With ``pytest-cov``, pass ``--cov-report=xml``, e.g. via ``pyproject.toml``: "--cov=your-package", "--cov=test", "--cov-report=xml", - ] + Customization ============= @@ -73,7 +75,8 @@ Command for manual coverage file selection: Customizable settings (see their description for details) in the ``python-coverage`` group, e.g. via ``M-x customize-group``: -- ``python-coverage-default-file-name`` +- ``python-coverage-default-xml-file-name`` +- ``python-coverage-default-sqlite-file-name`` - ``python-coverage-overlay-width`` Styling via custom faces, e.g. via ``M-x customize-face``: diff --git a/python-coverage.el b/python-coverage.el index 649e839..121b916 100644 --- a/python-coverage.el +++ b/python-coverage.el @@ -21,16 +21,23 @@ (require 'filenotify) (require 'python) (require 's) +(require 'sqlite3) (require 'xml) (require 'xml+) +(require 'cl-lib) (defgroup python-coverage nil "Python coverage" :group 'python :prefix "python-coverage") -(defcustom python-coverage-default-file-name "coverage.xml" - "Default file name to use when looking for coverage results." +(defcustom python-coverage-default-xml-file-name "coverage.xml" + "Default file name to use when looking for coverage XML report." + :group 'python-coverage + :type 'string) + +(defcustom python-coverage-default-sqlite-file-name ".coverage" + "Default file name to use when looking for coverage SQLite file." :group 'python-coverage :type 'string) @@ -143,28 +150,70 @@ This is only needed if autodetection does not work." "Obtain coverage info for the current buffer." (-when-let* ((coverage-file (python-coverage--find-coverage-file-current-buffer)) - (non-empty? (> (python-coverage--file-size coverage-file) 0)) - (tree (python-coverage--parse-coverage-xml-file coverage-file)) - (coverage-info (python-coverage--get-missing-file-coverage tree (buffer-file-name)))) - coverage-info)) + (non-empty? (> (python-coverage--file-size coverage-file) 0))) + ;; Prefer XML because coveragepy includes some analysis about blank lines, + ;; docstrings etc. + (if (python-coverage--is-xml coverage-file) + (-when-let + (tree (python-coverage--parse-coverage-xml-file coverage-file)) + (python-coverage--get-missing-file-coverage tree (buffer-file-name))) + (python-coverage--query-missing-file-coverage coverage-file (buffer-file-name))))) + +(defun python-coverage-rerun-pytest-current-region () + "Re-run the relevant pytest tests accoring to recorded 'context' information. + This requires that pytest was run with --cov-context=test previously. + Current region or current line in current buffer is used." + (interactive) + (-when-let* + ((coverage-file (python-coverage--find-coverage-file-current-buffer t)) + (python-file-name (buffer-file-name)) + (line-nums (if (region-active-p) + (apply 'number-sequence + (sort (list (line-number-at-pos (mark)) + (line-number-at-pos (point))) + '<)) + (list (line-number-at-pos (point))))) + (contexts (or (python-coverage--query-coverage-contexts coverage-file python-file-name line-nums) + (error "No related test contexts found for current region")))) + ;; In pytest, test contexts look like this: + ;; + ;; path/to/src/module.py::TestClassName::test_method_name|run + ;; + ;; The last part can also be `setup` or `teardown`. For now, we assume that + ;; any of these could be relevant. + ;; (print (mapcar (lambda (item) (car (s-split "\|" item))) contexts)))) + (-let [file-test-args (mapcar (lambda (item) (car (s-split "\|" item))) contexts)] + (print "Contexts:") + (print contexts) + (print "File test args:") + (print file-test-args) + (python-pytest--run + :args file-test-args + :edit current-prefix-arg)))) ;; Internal helpers for handling files -(defun python-coverage--find-coverage-file-current-buffer () +(defun python-coverage--find-coverage-file-current-buffer (&optional sqlite-only) "Find a coverage file for the current buffer." (-let [source-file-name (or (buffer-file-name) (error "Cannot detect source file name; buffer is not visiting a file"))] - (python-coverage--find-coverage-file source-file-name))) + (python-coverage--find-coverage-file source-file-name sqlite-only))) -(defun python-coverage--find-coverage-file (source-file-name) +(defun python-coverage--find-coverage-file (source-file-name &optional sqlite-only) "Find a coverage file for SOURCE-FILE-NAME." (or python-coverage--coverage-file-name - (-some-> - (python-coverage--locate-dominating-file source-file-name python-coverage-default-file-name) - (file-name-as-directory) - (s-concat python-coverage-default-file-name)) + (or + (when (not sqlite-only) + (-some-> + (python-coverage--locate-dominating-file source-file-name python-coverage-default-xml-file-name) + (file-name-as-directory) + (s-concat python-coverage-default-xml-file-name))) + (-some-> + (python-coverage--locate-dominating-file source-file-name python-coverage-default-sqlite-file-name) + (file-name-as-directory) + (s-concat python-coverage-default-sqlite-file-name))) (error "Could not find coverage file. (Hint: use ‘M-x python-coverage-select-coverage-file’ to choose manually.)"))) (declare-function projectile-locate-dominating-file "projectile" (file name)) @@ -196,8 +245,132 @@ FILE and NAME are handled like ‘locate-dominating-file’ does." (->> (file-attributes file-name) (nth 7))) +;; Internal helpers for handling the SQLite coverage format + +(defun python-coverage--query-missing-file-coverage (coverage-file python-file-name) + (-when-let* + ((dbh-version (python-coverage--open-coverage-db coverage-file)) + (dbh (car dbh-version)) + ;; Use hex() for the BLOB data because the sqlite3 bindings don't + ;; seem to have another way to pass binary data through. + (stmt (sqlite3-prepare dbh " + SELECT DISTINCT hex(numbits) + FROM line_bits + INNER JOIN file ON line_bits.file_id = file.id + WHERE path = ?"))) + (sqlite3-bind-text stmt 1 python-file-name) + (let* ((nums ())) + (while (= sqlite-row (sqlite3-step stmt)) + (-let + [(hex-numbits) (sqlite3-fetch stmt)] + (setq nums (cl-nunion nums (python-coverage--hex-numbits-to-nums hex-numbits))))) + (sqlite3-finalize stmt) + (sqlite3-close dbh) + + ;; Convert the 'hits=1' lines we've got into 'missing' lines. + ;; We reproduce a little bit of the analysis done by coveragepy: + ;; - empty lines shouldn't count as missing + (with-temp-buffer + (insert-file-contents python-file-name) + (let* ((line-count (count-lines (point-min) (point-max))) + (empty-source-lines (python-coverage--get-empty-source-lines)) + (missing-lines + (cl-set-difference (number-sequence 1 (+ 1 line-count)) + (cl-nunion nums empty-source-lines)))) + (python-coverage--merge-adjacent + (mapcar (lambda (num) + (list :line-beg num :line-end num :status 'missing)) + missing-lines))))))) + + +(defun python-coverage--query-coverage-contexts (coverage-file python-file-name line-nums) + "Use the SQLite coverage database COVERAGE-FILE to get the contexts that cover PYTHON-FILE-NAME lines LINE-NUMS" + (-when-let* + ((dbh-version (python-coverage--open-coverage-db coverage-file)) + (dbh (car dbh-version)) + (stmt (sqlite3-prepare dbh " + SELECT context, hex(numbits) + FROM context + INNER JOIN line_bits ON line_bits.context_id = context.id + INNER JOIN file on line_bits.file_id = file.id + WHERE path = ? AND context IS NOT NULL AND context != '';"))) + (sqlite3-bind-text stmt 1 python-file-name) + (let* ((contexts ())) + (while (= sqlite-row (sqlite3-step stmt)) + (-let* + (((context hex-numbits) (sqlite3-fetch stmt)) + (covered-line-nums (python-coverage--hex-numbits-to-nums hex-numbits))) + (when (cl-intersection line-nums covered-line-nums) + (setq contexts (cons context contexts))))) + (sqlite3-finalize stmt) + (sqlite3-close dbh) + (seq-uniq contexts)))) + +(defun python-coverage--open-coverage-db (coverage-file) + "Open coverage-file as a SQLite database, check the schema version, returning a 2-list (handle version)" + ;; We return the version number because in the future calling functions + ;; may need to issue different SQL statements based on this. + (-when-let* + ((dbh (sqlite3-open coverage-file sqlite-open-readonly)) + (stmt (sqlite3-prepare dbh "SELECT version FROM coverage_schema;"))) + (sqlite3-step stmt) + (-let + [(version) (sqlite3-fetch stmt)] + (when (/= version 7) + (message (format "python-coverage might not correctly support the schema version in your .coverage file - version %d. " version))) + (sqlite3-finalize stmt) + (list dbh version)))) + +(defun python-coverage--get-empty-source-lines () + (save-excursion + (let ((empty-lines ()) + (line-num 1)) + (goto-char (point-min)) + (while (not (eobp)) + (beginning-of-line) + (when (looking-at-p "[[:blank:]]*$") + (setq empty-lines (cons line-num empty-lines))) + (forward-line) + (setq line-num (1+ line-num))) + empty-lines))) + +(defun python-coverage--hex-to-bytes (hexstring) + "Convert string containing hex pairs (e.g. \"bc3509\") into vector of integers (bytes) for each pair" + (vconcat (mapcar (lambda (pair) + (string-to-number (concat pair) 16)) + (-partition 2 (string-to-list hexstring))))) + + +(defun python-coverage--numbits-to-nums (numbits) + "Convert vector of integers (bytes) into list of numbers, as per the coveragepy function" + ;; Follows the Python version fairly closely for simplicity + (let ((nums ())) + + (dotimes (byte-i (length numbits)) + (let ((byte (elt numbits byte-i))) + (dotimes (bit-i 8) + (when (> (logand byte (ash 1 bit-i)) 0) + (setq nums (cons (+ (* byte-i 8) bit-i) nums)))))) + + (reverse nums))) + +(defun python-coverage--hex-numbits-to-nums (hex-numbits) + (python-coverage--numbits-to-nums (python-coverage--hex-to-bytes hex-numbits))) + +(ert-deftest python-coverage--test-numbits-to-nums () + ;; Used Python version as oracle + (should (equal (python-coverage--hex-numbits-to-nums "DA5BCD70BE1024939024024000000008") + '(1 3 4 6 7 8 9 11 12 14 16 18 19 22 23 28 29 30 33 34 35 + 36 37 39 44 50 53 56 57 60 63 68 71 74 77 81 94 123)))) + ;; Internal helpers for handling the coverage XML format +(defun python-coverage--is-xml (file-name) + (with-temp-buffer + (insert-file-contents file-name nil 0 5) + (string= (buffer-substring-no-properties 1 6) "