python 3.12 wip

Author: shomchak

Dockerfile

@@ -1,9 +1,9 @@
-FROM python:3.9-slim
+FROM python:3.12-slim
 
-MAINTAINER brian <[email protected]>
+LABEL org.opencontainers.image.authors="brian <[email protected]>"
 
-ENV LANG C.UTF-8
-ENV	LC_ALL C.UTF-8
+ENV LANG=C.UTF-8
+ENV	LC_ALL=C.UTF-8
 
 # TODO: keep requirements in one place
 RUN pip install \
@@ -29,20 +29,26 @@ RUN pip install \
     pyinotify>=0.9.4, \
     raven>=5.0.0 \
     'tox>4,<5' \
+    # v80 removes many setup.py develop otions
+    'setuptools<80.0.0' \
     'datalake<2'
 
+# RUN pip install -U pip
+
 RUN mkdir -p /opt/
 COPY . /opt/
 
+# RUN pip install -e /opt/client[test]
+
 # Take care to install clients such that the source code can be mounted into
 # the container and used for development. That is, the python paths and paths
 # to console scripts Just Work (TM)
 ENV PYTHONPATH=/opt/client:/opt/ingester:/opt/api
 RUN for d in ingester api; do \
     cd /opt/$d && \
-    python setup.py develop -s /usr/local/bin \
+    python setup.py develop --script-dir /usr/local/bin \
         --egg-path ../../../../../opt/$d/ \
-        -d /usr/local/lib/python3.9/site-packages/ \
+        -d /usr/local/lib/python3.12/site-packages/ \
         --no-deps; \
     done
 

api/setup.py

@@ -30,7 +30,7 @@ def get_version_from_pyver():
         if 'sdist' in sys.argv or 'bdist_wheel' in sys.argv:
             raise ImportError('You must install pyver to create a package')
         else:
-            return 'noversion'
+            return '0.dev'
     version, version_info = pyver.get_version(pkg="datalake_api",
                                               public=True)
     return version

client/datalake/scripts/cli.py

@@ -68,7 +68,7 @@ def wrapped(*args, **kwargs):
                     'environment variables, which can be overridden by '
                     'command-line arguments.'),
               envvar='DATALAKE_CONFIG')
-@click.option('-u', '--storage-url',
+@click.option('-U', '--storage-url',
               help=('The URL to the top-level storage resource where '
                     'datalake will archive all the files (e.g., '
                     's3://my-datalake). DATALAKE_STORAGE_URL is the '

client/datalake/tests/conftest.py

@@ -20,7 +20,7 @@
 
 
 try:
-    from moto import mock_s3
+    from moto import mock_aws
     import boto3
     from six.moves.urllib.parse import urlparse
     import json
@@ -142,7 +142,7 @@ def tear_down():
 
 @pytest.fixture
 def s3_connection(aws_connector):
-    with mock_s3():
+    with mock_aws():
         yield boto3.resource('s3')
 
 

client/got.txt

@@ -37,9 +37,10 @@ dynamic = ["version"]
 test = [
     'pytest<8.0.0',
     'pytest-cov>=2.5.1,<4',
-    'moto[s3]>4,<5',
+    'moto[s3]>5',
     'twine<4.0.0',
-    'pip>=20.0.0,<22.0.0',
+    # 'pip>=20.0.0,<22.0.0',
+    'pip>=20.0.0',
     'wheel<0.38.0',
     'flake8>=2.5.0,<4.1',
     'responses<0.22.0',

client/pyproject.toml

@@ -36,6 +36,19 @@ def test_push_file(archive, random_metadata, tmpfile, s3_object):
 def test_push_large_file(
         monkeypatch, archive, random_metadata, tmpfile, s3_object):
     monkeypatch.setenv('DATALAKE_CHUNK_SIZE_MB', '5')
+    from botocore.httpchecksum import StreamingChecksumBody
+
+    # When testing with Moto, newer versions of boto3/botocore cause a
+    # FlexibleChecksumError because Moto doesn't supply the checksum headers
+    # that boto3 now expects. The standard config objects can fail to apply
+    # to the underlying S3 transfer manager.
+    # The most reliable way to disable this for tests is to directly patch
+    # the validation function in botocore to do nothing.
+    monkeypatch.setattr(
+        StreamingChecksumBody,
+        "_validate_checksum",
+        lambda _: None)
+
     expected_content = ('big data' * 1024 * 1024).encode('utf-8')
     f = tmpfile(expected_content)
     url = archive.prepare_metadata_and_push(f, **random_metadata)

client/test/test_archive.py

@@ -101,3 +101,12 @@ def test_push_with_translation_expression(cli_tester, tmpdir):
     cmd += '--what=job --start=now --end=now --where=hostname '
     cmd += str(f)
     cli_tester(cmd)
+
+
+@pytest.mark.xfail(reason="We don't want the CLI to raise warnings.",
+                   strict=True)
+def test_no_duplicate_parameters(cli_tester):
+    # Unfortunately there is no clean way to test if a warning is NOT emitted.
+    # So we have to test that a warning IS emitted and XFAIL.
+    with pytest.warns(UserWarning, match='Remove its duplicate as parameters'):
+        cli_tester('--help')

client/test/test_cli.py

@@ -30,7 +30,7 @@ def get_version_from_pyver():
         if 'sdist' in sys.argv or 'bdist_wheel' in sys.argv:
             raise ImportError('You must install pyver to create a package')
         else:
-            return 'noversion'
+            return '0.dev'
     version, version_info = pyver.get_version(pkg="datalake_ingester",
                                               public=True)
     return version