Merge pull request #78 from M1ha-Shvn/issue-76

Issue 76

.github/workflows/python-tests.yml

@@ -11,8 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.5", "3.6", "3.7", "3.8", "3.9"]
-        postgres-version: ["9.4", "9.5", "9.6", "10", "11", "12"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        postgres-version: ["9.4", "9.5", "9.6", "10", "11", "12", "13"]
         django-version: ["1.8", "1.9", "1.10", "1.11", "2.0", "2.1", "2.2", "3.0", "3.1", "3.2"]
         exclude:
         # Django 3.0+ doesn't support python 3.5
@@ -22,7 +22,7 @@ jobs:
           django-version: "3.1"
         - python-version: "3.5"
           django-version: "3.2"
-          
+
         # Django 3.0+ doesn't support PostgreSQL 9.4
         - django-version: "3.0"
           postgres-version: "9.4"
@@ -57,7 +57,21 @@ jobs:
           django-version: "1.9"
         - python-version: "3.9"
           django-version: "1.10"
-
+        - python-version: "3.10"
+          django-version: "1.8"
+        - python-version: "3.10"
+          django-version: "1.9"
+        - python-version: "3.10"
+          django-version: "1.10"
+
+        # Django before 2.1 is not compatible with python 3.10
+        #  as it uses collections.Iterator
+        - python-version: "3.10"
+          django-version: "2.0"
+        - python-version: "3.10"
+          django-version: "1.11"
+
+
     services:
       postgres:
         image: postgres:${{ matrix.postgres-version }}

README.md

@@ -4,7 +4,7 @@
 Django extension to update multiple table records with similar (but not equal) conditions in efficient way on PostgreSQL
 
 ## Requirements
-* Python 3.5+
+* Python 3.6+
   Previous versions may also work, but are not tested in CI  
 * django >= 1.8  
   Previous versions may also work, but are not tested in CI.  
@@ -561,7 +561,8 @@ You can find them in `tests` directory.
 ## Running tests
 ### Running in docker
 1. Install [docker and docker-compose](https://www.docker.com/)
-2. Run `docker-compose run run_tests` in project directory  
+2. Run `docker build . --tag django-pg-bulk-pupdate` in project directory
+3. Run `docker-compose run run_tests` in project directory  
 
 ### Running in virtual environment
 1. Install all requirements listed above  

docker-compose.yml

@@ -18,7 +18,7 @@ services:
         - PYTHON_VER=latest
     volumes:
       - ./.docker/wait-for-it.sh:/bin/wait-for-it.sh
-    command: ["/bin/wait-for-it.sh", "postgres_db:5432", "-s", "-t", "0", "--", "python3", "runtests.py"]
+    command: ["/bin/bash", "/bin/wait-for-it.sh", "postgres_db:5432", "-s", "-t", "0", "--", "python3", "runtests.py"]
     environment:
       - PGHOST=postgres_db
       - PGUSER=postgres

src/django_pg_bulk_update/compatibility.py

@@ -18,6 +18,13 @@
     # For django before 3.2
     from django.db import DefaultConnectionProxy as ConnectionProxy
 
+try:
+    # This approach applies to python 3.10+
+    from collections.abc import Iterable  # noqa F401
+except ImportError:
+    # This approach applies to python versions less than 3.10
+    from collections import Iterable  # noqa F401
+
 
 # six.string_types replacement in order to remove dependency
 string_types = (str,) if sys.version_info[0] == 3 else (str, unicode)  # noqa F821

src/django_pg_bulk_update/query.py

@@ -4,7 +4,7 @@
 
 import inspect
 import json
-from collections import Iterable
+
 from itertools import chain
 from logging import getLogger
 from typing import Any, Type, Iterable as TIterable, Union, Optional, List, Tuple
@@ -14,7 +14,7 @@
 from django.db.models.sql import UpdateQuery
 from django.db.models.sql.where import WhereNode
 
-from .compatibility import get_postgres_version, get_model_fields, returning_available, string_types
+from .compatibility import get_postgres_version, get_model_fields, returning_available, string_types, Iterable
 from .set_functions import AbstractSetFunction, NowSetFunction
 from .types import TOperators, TFieldNames, TUpdateValues, TSetFunctions, TOperatorsValid, TUpdateValuesValid, \
     TSetFunctionsValid, TDatabase, FieldDescriptor, AbstractFieldFormatter
@@ -220,12 +220,16 @@ def _validate_set_functions(model, fds, functions):
 
     for f in fds:
         field = f.get_field(model)
-        if getattr(field, 'auto_now', False):
-            f.set_function = NowSetFunction(if_null=False)
-        elif getattr(field, 'auto_now_add', False):
-            f.set_function = NowSetFunction(if_null=True)
-        else:
-            f.set_function = functions.get(f.name)
+
+        set_func = functions.get(f.name)
+        if set_func is None:
+            # Note that we should always respect any user provided set functions
+            if getattr(field, 'auto_now', False):
+                set_func = NowSetFunction(if_null=False)
+            elif getattr(field, 'auto_now_add', False):
+                set_func = NowSetFunction(if_null=True)
+
+        f.set_function = set_func
 
         if not f.set_function.field_is_supported(field):
             raise ValueError("'%s' doesn't support '%s' field" % (f.set_function.__class__.__name__, f.name))

tests/test_bulk_create.py

@@ -1,10 +1,10 @@
-from datetime import timedelta
+from datetime import date, datetime, timedelta
 from unittest import skipIf
 
 from django.test import TestCase
 from django.utils.timezone import now
 
-from django_pg_bulk_update.compatibility import jsonb_available, hstore_available, array_available
+from django_pg_bulk_update.compatibility import jsonb_available, hstore_available, array_available, tz_utc
 from django_pg_bulk_update.query import bulk_create
 from django_pg_bulk_update.set_functions import ConcatSetFunction
 from tests.compatibility import get_auto_now_date
@@ -410,6 +410,21 @@ def test_now(self):
         self.assertGreaterEqual(instance.checked, now() - timedelta(seconds=1))
         self.assertLessEqual(instance.checked, now() + timedelta(seconds=1))
 
+    def test_auto_now_respects_override(self):
+        # Now check to make sure we can explicitly set values
+        # (requires passing set functions)
+        bulk_create(AutoNowModel, [{
+            'id': 1,
+            'created': datetime(2011, 1, 2, 0, 0, 0, tzinfo=tz_utc),
+            'updated': date(2011, 1, 3),
+            'checked': datetime(2011, 1, 4, 0, 0, 0, tzinfo=tz_utc),
+        }], set_functions={"created": "eq", "updated": "eq", "checked": "eq"})
+
+        instance = AutoNowModel.objects.get()
+        self.assertEqual(datetime(2011, 1, 2, 0, 0, 0, tzinfo=tz_utc), instance.created)
+        self.assertEqual(date(2011, 1, 3), instance.updated)
+        self.assertEqual(datetime(2011, 1, 4, 0, 0, 0, tzinfo=tz_utc), instance.checked)
+
 
 class TestManager(TestCase):
     fixtures = ['test_model']

tests/test_bulk_update.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, date
 from unittest import skipIf
 
 from django.test import TestCase
@@ -397,6 +397,21 @@ def test_auto_now(self):
         self.assertEqual(datetime(2020, 1, 2, 0, 0, 0,  tzinfo=tz_utc), instance.checked)
         self.assertEqual(instance.updated, get_auto_now_date())
 
+    def test_auto_now_respects_override(self):
+        # Now check to make sure we can explicitly set values
+        # (requires passing set functions)
+        bulk_update(AutoNowModel, [{
+            'id': 1,
+            'created': datetime(2011, 1, 2, 0, 0, 0, tzinfo=tz_utc),
+            'updated': date(2011, 1, 3),
+            'checked': datetime(2011, 1, 4, 0, 0, 0, tzinfo=tz_utc),
+        }], set_functions={"created": "eq", "updated": "eq"})
+
+        instance = AutoNowModel.objects.get()
+        self.assertEqual(datetime(2011, 1, 2, 0, 0, 0, tzinfo=tz_utc), instance.created)
+        self.assertEqual(date(2011, 1, 3), instance.updated)
+        self.assertEqual(datetime(2011, 1, 4, 0, 0, 0, tzinfo=tz_utc), instance.checked)
+
     def test_quoted_table_name(self):
         # Test for https://github.com/M1ha-Shvn/django-pg-bulk-update/issues/63
         self.assertEqual(1, bulk_update(TestModelWithSchema, [{'id': 1, 'name': 'abc'}]))
@@ -664,6 +679,7 @@ def test_not_in(self):
             'id': list(range(7, 10)) + list(range(1, 4)),
             'name': '2'
         }], key_fields_ops=['!in'])
+
         self.assertEqual(6, res)
         for pk, name, int_field in TestModel.objects.all().order_by('id').values_list('id', 'name', 'int_field'):
             if pk in {1, 2, 3}:

tests/test_bulk_update_or_create.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import date, datetime, timedelta
 from unittest import skipIf
 
 from django.test import TestCase
@@ -493,6 +493,31 @@ def test_auto_now(self):
                 self.assertGreaterEqual(instance.created, now() - timedelta(seconds=1))
                 self.assertLessEqual(instance.created, now() + timedelta(seconds=1))
 
+    def test_auto_now_respects_override(self):
+        bulk_update_or_create(AutoNowModel, [{
+            'id': 1,
+            'created': datetime(2011, 1, 2, 0, 0, 0, tzinfo=tz_utc),
+            'updated': date(2011, 1, 3),
+            'checked': datetime(2011, 1, 4, 0, 0, 0, tzinfo=tz_utc),
+        }], set_functions={"created": "eq", "updated": "eq", "checked": "eq"})
+
+        instance = AutoNowModel.objects.get()
+        self.assertEqual(datetime(2011, 1, 2, 0, 0, 0, tzinfo=tz_utc), instance.created)
+        self.assertEqual(date(2011, 1, 3), instance.updated)
+        self.assertEqual(datetime(2011, 1, 4, 0, 0, 0, tzinfo=tz_utc), instance.checked)
+
+        bulk_update_or_create(AutoNowModel, [{
+            'id': 1,
+            'created': datetime(2012, 2, 5, 1, 2, 3, tzinfo=tz_utc),
+            'updated': date(2012, 2, 6),
+            'checked': datetime(2012, 2, 7, 3, 2, 1, tzinfo=tz_utc),
+        }], set_functions={"created": "eq", "updated": "eq", "checked": "eq"})
+
+        instance = AutoNowModel.objects.get()
+        self.assertEqual(datetime(2012, 2, 5, 1, 2, 3, tzinfo=tz_utc), instance.created)
+        self.assertEqual(date(2012, 2, 6), instance.updated)
+        self.assertEqual(datetime(2012, 2, 7, 3, 2, 1, tzinfo=tz_utc), instance.checked)
+
     def test_quoted_table_name(self):
         # Test for https://github.com/M1ha-Shvn/django-pg-bulk-update/issues/63
         self.assertEqual(2, bulk_update_or_create(