Merge pull request #1936 from saulpw/v2.11.1rc1

V2.11.1rc1

CHANGELOG.md

@@ -1,5 +1,35 @@
 # VisiData version history
 
+# v2.11.1 (2023-07-XX)
+
+- [tests] fix tests for Python >=3.11
+- [path] update for Python 3.12 API (reported by @QuLogic #1934)
+
+## Improvements
+
+- [chooser] choose only exactly matching strings in chooser (PR by @daviewales #1902)
+- [columns] speed up `getMaxWidth()` for wide columns, and correct some edge cases (PR by @midichef #1747)
+- [freqtbl] Default `disp_histogram` to U+25A0 BLACK SQUARE (■)) (PR by @daviewales #1949)
+- [loaders fixed] do not truncate wide columns with fixed-width saver (PR by @daviewales #1890)
+
+## Bugfixes
+
+- add missing import `copy`
+- [graph] fix graph ranges for xmax, ymax < 1 (PR by @midichef #1752)
+- [graph] fix data on edges being drawn offscreen (PR by @midichef #1850)
+- [input] fix `Ctrl+T` swap on empty input (reported by @gfrmin #1684)
+- [inputsingle] loop until keystroke (do not timeout)
+- [fill] allow filling with values that are logically false (PR by @midichef #1794)
+- [macos] do not bind empty string to any keybinding
+- [paste] add new rows to sheet if insufficient rows
+- [path Dirsheet] set name to '.' for givenpath of '.' (reported by @geekscrapy #1768)
+- [path] fix progress for compressed files (reported by @bitwisecook #1255 #1175)
+- [replay] clearCaches before moving cursor (reported by @mokalan #1773)
+- [save] handle saving 0 sheets (reported by @reagle #1266 #1720)
+- [settings] clear cache correctly before set
+- [undo] fix so that undo is Sheet-specific on copied sheets (reported by @geekscrapy #1780)
+- [undo] undoing `zd` now removes `[M]` (modification mark) (reported by @Freed-Wu #1800)
+
 # v2.11 (2023-01-15)
 
 - [ci] drop support for Python 3.6 (related to https://github.com/actions/setup-python/issues/543)

README.md

@@ -1,5 +1,5 @@
 
-# VisiData v2.11
+# VisiData v2.11.1
 
 [![twitter @VisiData][1.1]][1]
 [![Tests](https://github.com/saulpw/visidata/workflows/visidata-ci-build/badge.svg)](https://github.com/saulpw/visidata/actions/workflows/main.yml)

dev/test.sh

@@ -3,6 +3,7 @@
 # Usage: test.sh [testname]
 
 set -e
+set -x
 shopt -s failglob
 
 trap "echo aborted; exit;" SIGINT SIGTERM
@@ -20,8 +21,22 @@ for i in $TESTS ; do
     outbase=${i##tests/}
     if [ "${i%-nosave.vd*}-nosave" == "${i%.vd*}" ];
     then
-        echo "$1"
+        TEST=false
+    elif [ "${i%-311.vd*}-311" == "${i%.vd*}" ];
+    then
+        if [ `python -c 'import sys; print(sys.version_info[:2] >= (3,11))'` == "True" ];
+        then
+            TEST=true
+        else
+            TEST=false
+        fi
+
     else
+        TEST=true
+    fi
+    if $TEST == true;
+    then
+        echo $TEST
         for goldfn in tests/golden/${outbase%.vd*}.*; do
             PYTHONPATH=. bin/vd --confirm-overwrite=False --play "$i" --batch --output "$goldfn" --config tests/.visidatarc --visidata-dir tests/.visidata
             echo "save: $goldfn"

docs/man.md

@@ -610,7 +610,7 @@ vd(1)
      <span style="font-weight:bold;">disp</span><span style="text-decoration:underline;">_</span><span style="font-weight:bold;">menu</span><span style="text-decoration:underline;">_</span><span style="font-weight:bold;">input</span>     …                   indicator if input required for command
      <span style="font-weight:bold;">disp</span><span style="text-decoration:underline;">_</span><span style="font-weight:bold;">menu</span><span style="text-decoration:underline;">_</span><span style="font-weight:bold;">fmt</span>       Ctrl+H for help menu
                                              right-side menu format string
-     <span style="font-weight:bold;">disp</span><span style="text-decoration:underline;">_</span><span style="font-weight:bold;">histogram</span>      *                   histogram element character
+     <span style="font-weight:bold;">disp</span><span style="text-decoration:underline;">_</span><span style="font-weight:bold;">histogram</span>      ■                   histogram element character
      <span style="font-weight:bold;">disp</span><span style="text-decoration:underline;">_</span><span style="font-weight:bold;">histolen</span>       50                  width of histogram column
      <span style="font-weight:bold;">disp</span><span style="text-decoration:underline;">_</span><span style="font-weight:bold;">canvas</span><span style="text-decoration:underline;">_</span><span style="font-weight:bold;">charset</span> ⠀⠁⠂⠃⠄⠅⠆⠇⠈⠉⠊⠋⠌⠍⠎⠏⠐⠑⠒⠓⠔⠕⠖⠗⠘⠙⠚⠛⠜⠝⠞⠟⠠⠡⠢⠣⠤⠥⠦⠧⠨⠩⠪⠫⠬⠭⠮⠯⠰⠱⠲⠳⠴⠵⠶⠷⠸⠹⠺⠻⠼⠽⠾⠿⡀⡁⡂⡃⡄⡅⡆⡇⡈⡉⡊⡋⡌⡍⡎⡏⡐⡑⡒⡓⡔⡕⡖⡗⡘⡙⡚⡛⡜⡝⡞⡟⡠⡡⡢⡣⡤⡥⡦⡧⡨⡩⡪⡫⡬⡭⡮⡯⡰⡱⡲⡳⡴⡵⡶⡷⡸⡹⡺⡻⡼⡽⡾⡿⢀⢁⢂⢃⢄⢅⢆⢇⢈⢉⢊⢋⢌⢍⢎⢏⢐⢑⢒⢓⢔⢕⢖⢗⢘⢙⢚⢛⢜⢝⢞⢟⢠⢡⢢⢣⢤⢥⢦⢧⢨⢩⢪⢫⢬⢭⢮⢯⢰⢱⢲⢳⢴⢵⢶⢷⢸⢹⢺⢻⢼⢽⢾⢿⣀⣁⣂⣃⣄⣅⣆⣇⣈⣉⣊⣋⣌⣍⣎⣏⣐⣑⣒⣓⣔⣕⣖⣗⣘⣙⣚⣛⣜⣝⣞⣟⣠⣡⣢⣣⣤⣥⣦⣧⣨⣩⣪⣫⣬⣭⣮⣯⣰⣱⣲⣳⣴⣵⣶⣷⣸⣹⣺⣻⣼⣽⣾⣿
                                              charset to render 2x4 blocks on canvas

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup
 # tox can't actually run python3 setup.py: https://github.com/tox-dev/tox/issues/96
 #from visidata import __version__
-__version__ = '2.11'
+__version__ = '2.11.1'
 
 setup(name='visidata',
       version=__version__,

tests/error-passthru.vd

@@ -4,3 +4,4 @@ benchmark	Quantity	1	select-equal-cell		,
 benchmark	Quantity		delete-cells		gzd	
 benchmark	Quantity		addcol-expr	Quantity.foo	=	
 benchmark	Quantity.foo		freq-col		F	
+benchmark_Quantity.foo_freq	histogram		hide-col		-	Hide current column

tests/extend.vd

@@ -1,20 +1,18 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	tests/data2.tsv	o	
 			open-file	tests/data1.tsv	o	
-data1			jump-sheet-2		^[2	
 data2	Key		key-col		!	
-data2			jump-sheet-1		^[1	
 data1	Key		key-col		!	
-			sheets-stack		S	
+data1			sheets-stack		S	
 sheets		キdata2	select-row		s	
 sheets		キdata1	select-row		s	
-sheets			join-sheets	extend	&	
+sheets			join-selected	extend	&	
 data1+data2	A		freq-col		F	
-data1+data2_A_freq			jump-sheet-1		^[1	
 data1			dup-rows		g"	
 data1_copy	A		key-col		!	
 data1_copy	Key		key-col		!	
 data1_copy			sheets-all		S	
 sheets_all		キdata1_copy	select-row		s	
 sheets_all		キdata1+data2_A_freq	select-row		s	
-sheets_all			join-sheets	extend	&	
+sheets_all			join-selected	extend	&	
+data1+data2_A_freq+data1_copy	histogram		hide-col		-	Hide current column

tests/freq-same-int.vd

@@ -7,6 +7,6 @@ benchmark	Quantity	キ2018-07-03	setcol-input	3	ge
 benchmark			unselect-rows		gu	
 benchmark	Quantity		type-int		#	
 benchmark	Quantity		freq-col		F	
-benchmark_Quantity_freq			quit-sheet		q	
 benchmark	Quantity	キ2018-07-03	edit-cell	2	e	
 benchmark	Quantity		freq-col		F	
+benchmark_Quantity_freq	histogram		hide-col		-	Hide current column

tests/golden/error-passthru.tsv

@@ -1,2 +1,2 @@
-Quantity.foo	count	percent	histogram
-#ERR	51	100.00	**************************************************
+Quantity.foo	count	percent
+#ERR	51	100.00

tests/golden/extend.tsv

@@ -1,4 +1,4 @@
-A	count	percent	histogram	data1_copy_Key	data1_copy_B
-a1	1	33.33	**************************************************	1	b1
-c1	1	33.33	**************************************************	2	d1
-e1	1	33.33	**************************************************	2	f1
+A	count	percent	data1_copy_Key	data1_copy_B
+a1	1	33.33	1	b1
+c1	1	33.33	2	d1
+e1	1	33.33	2	f1

tests/golden/freq-same-int.tsv

@@ -1,3 +1,3 @@
-Quantity	count	percent	histogram
-2	1	1.96	*
-3	50	98.04	**************************************************
+Quantity	count	percent
+2	1	1.96
+3	50	98.04

tests/golden/histogram.tsv

@@ -1,6 +1,6 @@
-Quantity	count	percent	histogram
-2.00 - 30.40	26	92.86	**************************************************
-30.40 - 58.80	1	3.57	*
-58.80 - 87.20	0	0.00	
-87.20 - 115.60	0	0.00	
-115.60 - 144.00	1	3.57	*
+Quantity	count	percent
+2.00 - 30.40	26	92.86
+30.40 - 58.80	1	3.57
+58.80 - 87.20	0	0.00
+87.20 - 115.60	0	0.00
+115.60 - 144.00	1	3.57

tests/golden/numeric_binning.tsv

@@ -1,8 +1,8 @@
-Quantity	count	percent	histogram
-1.00 - 21.43	46	90.20	**************************************************
-21.43 - 41.86	3	5.88	***
-41.86 - 62.29	1	1.96	*
-123.57 - 144.00	1	1.96	*
-62.29 - 82.71	0	0.00	
-82.71 - 103.14	0	0.00	
-103.14 - 123.57	0	0.00	
+Quantity	count	percent
+1.00 - 21.43	46	90.20
+21.43 - 41.86	3	5.88
+41.86 - 62.29	1	1.96
+123.57 - 144.00	1	1.96
+62.29 - 82.71	0	0.00
+82.71 - 103.14	0	0.00
+103.14 - 123.57	0	0.00

tests/histogram.vd

@@ -1,8 +1,9 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	sample_data/benchmark.csv	o	
-		numeric_binning	set-option	True		
+global		numeric_binning	set-option	True		
 benchmark	Quantity		type-int		#	
 benchmark	Quantity		addcol-expr	Quantity > 1	=	
 benchmark	Quantity > 1		select-col-regex	True	|	
 benchmark			dup-selected		"	
 benchmark_selectedref	Quantity		freq-col		F	
+benchmark_selectedref_Quantity_freq	histogram		hide-col		-	Hide current column

tests/numeric_binning.vd

@@ -1,6 +1,7 @@
 sheet	col	row	longname	input	keystrokes	comment
 			open-file	sample_data/benchmark.csv	o	
-	override	numeric_binning	set-option	True		
+global		numeric_binning	set-option	True		
 benchmark	Quantity		type-int		#	
 benchmark	Quantity		freq-col		F	
 benchmark_Quantity_freq	count		sort-desc		]	
+benchmark_Quantity_freq	histogram		hide-col		-	Hide current column

visidata/__init__.py

@@ -1,6 +1,6 @@
 'VisiData: a curses interface for exploring and arranging tabular data'
 
-__version__ = '2.11'
+__version__ = '2.11,1'
 __version_info__ = 'VisiData v' + __version__
 __author__ = 'Saul Pwanson <[email protected]>'
 __status__ = 'Production/Stable'

visidata/_input.py

@@ -222,7 +222,7 @@ def find_nonword(s, a, b, incr):
         elif ch == '^K':                           v = v[:i]  # ^Kill to end-of-line
         elif ch == '^O':                           v = vd.launchExternalEditor(v)
         elif ch == '^R':                           v = str(value)  # ^Reload initial value
-        elif ch == '^T':                           v = delchar(splice(v, i-2, v[i-1]), i)  # swap chars
+        elif ch == '^T':                           v = delchar(splice(v, i-2, v[i-1:i]), i)  # swap chars
         elif ch == '^U':                           v = v[i:]; i = 0  # clear to beginning
         elif ch == '^V':                           v = splice(v, i, until_get_wch(scr)); i += 1  # literal character
         elif ch == '^W':                           j = find_nonword(v, 0, i-1, -1); v = v[:j+1] + v[i:]; i = j+1  # erase word
@@ -300,7 +300,9 @@ def inputsingle(vd, prompt, record=True):
     rstatuslen = vd.drawRightStatus(sheet._scr, sheet)
     promptlen = clipdraw(sheet._scr, y, 0, prompt, 0, w=w-rstatuslen-1)
     sheet._scr.move(y, w-promptlen-rstatuslen-2)
-    v = vd.getkeystroke(sheet._scr)
+
+    while not v:
+        v = vd.getkeystroke(sheet._scr)
 
     if record and vd.cmdlog:
         vd.setLastArgs(v)

visidata/canvas.py

@@ -482,16 +482,26 @@ def resetBounds(self):
                     if ymin is None or y < ymin: ymin = y
                     if xmax is None or x > xmax: xmax = x
                     if ymax is None or y > ymax: ymax = y
-            self.canvasBox = BoundingBox(float(xmin or 0), float(ymin or 0), float(xmax or 0)+1, float(ymax or 0)+1)
-
+            xmin = xmin or 0
+            xmax = xmax or 0
+            ymin = ymin or 0
+            ymax = ymax or 0
+            if xmin == xmax:
+                xmax += 1
+            if ymin == ymax:
+                ymax += 1
+            self.canvasBox = BoundingBox(float(xmin), float(ymin), float(xmax), float(ymax))
+
+        w = self.calcVisibleBoxWidth()
+        h = self.calcVisibleBoxHeight()
         if not self.visibleBox:
             # initialize minx/miny, but w/h must be set first to center properly
-            self.visibleBox = Box(0, 0, self.plotviewBox.w/self.xScaler, self.plotviewBox.h/self.yScaler)
-            self.visibleBox.xmin = self.canvasBox.xcenter - self.visibleBox.w/2
-            self.visibleBox.ymin = self.canvasBox.ycenter - self.visibleBox.h/2
+            self.visibleBox = Box(0, 0, w, h)
+            self.visibleBox.xmin = self.canvasBox.xmin + (self.canvasBox.w / 2) * (1 - self.xzoomlevel)
+            self.visibleBox.ymin = self.canvasBox.ymin + (self.canvasBox.h / 2) * (1 - self.yzoomlevel)
         else:
-            self.visibleBox.w = self.plotviewBox.w/self.xScaler
-            self.visibleBox.h = self.plotviewBox.h/self.yScaler
+            self.visibleBox.w = w
+            self.visibleBox.h = h
 
         if not self.cursorBox:
             self.cursorBox = Box(self.visibleBox.xmin, self.visibleBox.ymin, self.canvasCharWidth, self.canvasCharHeight)
@@ -534,6 +544,33 @@ def yScaler(self):
         else:
             return yratio
 
+    def calcVisibleBoxWidth(self):
+        w = self.canvasBox.w * self.xzoomlevel
+        if self.aspectRatio:
+            h = self.canvasBox.h * self.yzoomlevel
+            xratio = self.plotviewBox.w / w
+            yratio = self.plotviewBox.h / h
+            if xratio <= yratio:
+                return w / self.aspectRatio
+            else:
+                return self.plotviewBox.w / (self.aspectRatio * yratio)
+        else:
+            return w
+
+    def calcVisibleBoxHeight(self):
+        h = self.canvasBox.h * self.yzoomlevel
+        if self.aspectRatio:
+            w = self.canvasBox.w * self.yzoomlevel
+            xratio = self.plotviewBox.w / w
+            yratio = self.plotviewBox.h / h
+            if xratio < yratio:
+                return self.plotviewBox.h / xratio
+            else:
+                return h
+        else:
+            return h
+
+    #could be called canvas_to_plotterX()
     def scaleX(self, x):
         'returns plotter x coordinate'
         return round(self.plotviewBox.xmin+(x-self.visibleBox.xmin)*self.xScaler)

visidata/choose.py

@@ -40,7 +40,12 @@ def chooseFancy(vd, choices):
 
 @VisiData.api
 def chooseMany(vd, choices):
-    'Return a list of 1 or more keys from *choices*, which is a list of dicts.  Each element dict must have a unique "key", which must be typed directly by the user in non-fancy mode (therefore no spaces).  All other items in the dicts are also shown in fancy chooser mode.  Use previous choices from the replay input if available.  Add chosen keys (space-separated) to the cmdlog as input for the current command.'''
+    '''Return a list of 1 or more keys from *choices*, which is a list of
+    dicts. Each element dict must have a unique "key", which must be typed
+    directly by the user in non-fancy mode (therefore no spaces).  All other
+    items in the dicts are also shown in fancy chooser mode.  Use previous
+    choices from the replay input if available.  Add chosen keys
+    (space-separated) to the cmdlog as input for the current command.'''
     if vd.cmdlog:
         v = vd.getLastArgs()
         if v is not None:
@@ -62,11 +67,10 @@ def throw_fancy(v, i):
                 return v, i
             chosenstr = vd.input(prompt+': ', completer=CompleteKey(choice_keys), bindings={'^X': throw_fancy})
             for c in chosenstr.split():
-                poss = [p for p in choice_keys if str(p).startswith(c)]
-                if not poss:
-                    vd.warning('invalid choice "%s"' % c)
+                if c in choice_keys:
+                    chosen.append(c)
                 else:
-                    chosen.extend(poss)
+                    vd.warning('invalid choice "%s"' % c)
         except ReturnValue as e:
             chosen = e.args[0]
 

visidata/clipboard.py

@@ -90,8 +90,18 @@ def pasteFromClipboard(vd, cols, rows):
     text = vd.getLastArgs() or vd.sysclip_value().strip() or vd.fail('system clipboard is empty')
 
     vd.addUndoSetValues(cols, rows)
+    lines = text.split('\n')
+    if not lines:
+        vd.warning('nothing to paste')
+        return
+
+    vs = cols[0].sheet
+    newrows = [vs.newRow() for i in range(len(lines)-len(rows))]
+    if newrows:
+        rows.extend(newrows)
+        vs.addRows(newrows)
 
-    for line, r in zip(text.split('\n'), rows):
+    for line, r in zip(lines, rows):
         for v, c in zip(line.split('\t'), cols):
             c.setValue(r, v)
 

visidata/cmdlog.py

@@ -287,11 +287,12 @@ def replay_cancel(vd):
 @VisiData.api
 def moveToReplayContext(vd, r, vs):
         'set the sheet/row/col to the values in the replay row'
+        vd.clearCaches()
         if r.row not in [None, '']:
-            vs.moveToRow(r.row) or vd.error('no "%s" row' % r.row)
+            vs.moveToRow(r.row) or vd.error(f'no {r.row} row on {vs}')
 
         if r.col not in [None, '']:
-            vs.moveToCol(r.col) or vd.error('no "%s" column' % r.col)
+            vs.moveToCol(r.col) or vd.error(f'no {r.col} column on {vs}')
 
 
 @VisiData.api