From c54ebda2ff3216fa50f604c2f0cfde936b3252da Mon Sep 17 00:00:00 2001 From: Neng Lu Date: Fri, 10 Jan 2020 09:25:13 -0800 Subject: [PATCH 01/22] Initial Python 3 upgrade effort --- .travis.yml | 7 ++++--- README.md | 2 +- bazel_configure.py | 4 ++-- config/configure.ac | 4 ++-- examples/src/python/bolt/consume_bolt.py | 2 +- examples/src/python/bolt/count_bolt.py | 2 +- examples/src/python/bolt/half_ack_bolt.py | 2 +- examples/src/python/bolt/stateful_count_bolt.py | 2 +- examples/src/python/bolt/stream_aggregate_bolt.py | 2 +- examples/src/python/bolt/window_size_bolt.py | 2 +- examples/src/python/custom_grouping_topology.py | 2 +- examples/src/python/half_acking_topology.py | 2 +- examples/src/python/join_streamlet_topology.py | 2 +- examples/src/python/misc/test_task_hook.py | 2 +- examples/src/python/multi_stream_topology.py | 2 +- examples/src/python/pulsar_word_count_streamlet.py | 2 +- examples/src/python/spout/multi_stream_spout.py | 2 +- examples/src/python/spout/stateful_word_spout.py | 2 +- examples/src/python/spout/word_spout.py | 2 +- examples/src/python/stateful_word_count_topology.py | 2 +- examples/src/python/window_size_topology.py | 2 +- examples/src/python/word_count_streamlet.py | 2 +- examples/src/python/word_count_topology.py | 2 +- heron/common/src/python/pex_loader.py | 2 +- heron/common/src/python/utils/log.py | 2 +- heron/common/src/python/utils/proc.py | 2 +- heron/common/tests/python/pex_loader/constants.py | 2 +- .../common/tests/python/pex_loader/pex_loader_unittest.py | 2 +- .../common/tests/python/pex_loader/testdata/src/sample.py | 2 +- heron/executor/src/python/heron_executor.py | 2 +- heron/executor/tests/python/heron_executor_unittest.py | 2 +- heron/instance/src/python/basics/base_instance.py | 2 +- heron/instance/src/python/basics/bolt_instance.py | 2 +- heron/instance/src/python/basics/spout_instance.py | 2 +- heron/instance/src/python/instance/st_heron_instance.py | 2 +- heron/instance/src/python/network/event_looper.py | 2 +- heron/instance/src/python/network/gateway_looper.py | 2 +- heron/instance/src/python/network/heron_client.py | 2 +- heron/instance/src/python/network/metricsmgr_client.py | 2 +- heron/instance/src/python/network/protocol.py | 2 +- heron/instance/src/python/network/socket_options.py | 2 +- heron/instance/src/python/network/st_stmgr_client.py | 2 +- heron/instance/src/python/utils/metrics/metrics_helper.py | 2 +- heron/instance/src/python/utils/metrics/py_metrics.py | 2 +- heron/instance/src/python/utils/misc/communicator.py | 2 +- .../src/python/utils/misc/custom_grouping_helper.py | 2 +- .../src/python/utils/misc/outgoing_tuple_helper.py | 2 +- heron/instance/src/python/utils/misc/pplan_helper.py | 2 +- heron/instance/src/python/utils/misc/serializer_helper.py | 2 +- heron/instance/src/python/utils/system_config.py | 2 +- heron/instance/src/python/utils/system_constants.py | 2 +- .../src/python/utils/topology/topology_context_impl.py | 2 +- heron/instance/src/python/utils/tuple.py | 2 +- heron/instance/tests/python/mock_protobuf.py | 2 +- .../instance/tests/python/network/event_looper_unittest.py | 2 +- .../tests/python/network/gateway_looper_unittest.py | 2 +- .../instance/tests/python/network/heron_client_unittest.py | 2 +- .../tests/python/network/metricsmgr_client_unittest.py | 2 +- heron/instance/tests/python/network/mock_generator.py | 2 +- .../instance/tests/python/network/mock_generator_client.py | 2 +- heron/instance/tests/python/network/protocol_unittest.py | 2 +- .../tests/python/network/st_stmgr_client_unittest.py | 2 +- heron/instance/tests/python/utils/communicator_unittest.py | 2 +- .../tests/python/utils/custom_grouping_unittest.py | 2 +- .../instance/tests/python/utils/global_metrics_unittest.py | 2 +- heron/instance/tests/python/utils/log_unittest.py | 2 +- .../instance/tests/python/utils/metrics_helper_unittest.py | 2 +- heron/instance/tests/python/utils/mock_generator.py | 2 +- .../tests/python/utils/outgoing_tuple_helper_unittest.py | 2 +- heron/instance/tests/python/utils/pplan_helper_unittest.py | 2 +- heron/instance/tests/python/utils/py_metrics_unittest.py | 2 +- .../tests/python/utils/topology_context_impl_unittest.py | 2 +- heron/instance/tests/python/utils/tuple_helper_unittest.py | 2 +- heron/shell/src/python/handlers/browsehandler.py | 2 +- heron/shell/src/python/handlers/downloadhandler.py | 2 +- heron/shell/src/python/handlers/filedatahandler.py | 2 +- heron/shell/src/python/handlers/filehandler.py | 2 +- heron/shell/src/python/handlers/filestatshandler.py | 2 +- heron/shell/src/python/handlers/healthhandler.py | 2 +- heron/shell/src/python/handlers/jmaphandler.py | 2 +- heron/shell/src/python/handlers/jstackhandler.py | 2 +- heron/shell/src/python/handlers/killexecutorhandler.py | 2 +- heron/shell/src/python/handlers/memoryhistogramhandler.py | 2 +- heron/shell/src/python/handlers/pidhandler.py | 2 +- heron/shell/src/python/handlers/pmaphandler.py | 2 +- heron/shell/src/python/main.py | 2 +- heron/shell/src/python/utils.py | 2 +- heron/statemgrs/src/python/config.py | 2 +- heron/statemgrs/src/python/configloader.py | 2 +- heron/statemgrs/src/python/filestatemanager.py | 2 +- heron/statemgrs/src/python/log.py | 2 +- heron/statemgrs/src/python/stateexceptions.py | 2 +- heron/statemgrs/src/python/statemanager.py | 2 +- heron/statemgrs/src/python/statemanagerfactory.py | 2 +- heron/statemgrs/src/python/zkstatemanager.py | 2 +- heron/statemgrs/tests/python/configloader_unittest.py | 2 +- .../statemgrs/tests/python/statemanagerfactory_unittest.py | 2 +- heron/statemgrs/tests/python/zkstatemanager_unittest.py | 2 +- heron/tools/admin/src/python/main.py | 2 +- heron/tools/cli/src/python/activate.py | 2 +- heron/tools/cli/src/python/args.py | 2 +- heron/tools/cli/src/python/cdefs.py | 2 +- heron/tools/cli/src/python/cli_helper.py | 2 +- heron/tools/cli/src/python/config.py | 2 +- heron/tools/cli/src/python/deactivate.py | 2 +- heron/tools/cli/src/python/execute.py | 2 +- heron/tools/cli/src/python/help.py | 2 +- heron/tools/cli/src/python/jars.py | 2 +- heron/tools/cli/src/python/kill.py | 2 +- heron/tools/cli/src/python/main.py | 2 +- heron/tools/cli/src/python/opts.py | 2 +- heron/tools/cli/src/python/rest.py | 2 +- heron/tools/cli/src/python/restart.py | 2 +- heron/tools/cli/src/python/result.py | 2 +- heron/tools/cli/src/python/submit.py | 2 +- heron/tools/cli/src/python/update.py | 2 +- heron/tools/cli/src/python/version.py | 2 +- heron/tools/cli/tests/python/client_command_unittest.py | 2 +- heron/tools/cli/tests/python/opts_unittest.py | 2 +- heron/tools/common/src/python/access/fetch.py | 2 +- heron/tools/common/src/python/access/heron_api.py | 2 +- heron/tools/common/src/python/access/query.py | 2 +- heron/tools/common/src/python/access/tracker_access.py | 2 +- heron/tools/common/src/python/utils/classpath.py | 2 +- heron/tools/common/src/python/utils/config.py | 2 +- heron/tools/common/src/python/utils/heronparser.py | 2 +- heron/tools/explorer/src/python/args.py | 2 +- heron/tools/explorer/src/python/clusters.py | 2 +- heron/tools/explorer/src/python/help.py | 2 +- heron/tools/explorer/src/python/logicalplan.py | 2 +- heron/tools/explorer/src/python/main.py | 2 +- heron/tools/explorer/src/python/opts.py | 2 +- heron/tools/explorer/src/python/physicalplan.py | 2 +- heron/tools/explorer/src/python/topologies.py | 2 +- heron/tools/explorer/src/python/version.py | 2 +- heron/tools/explorer/tests/python/explorer_unittest.py | 2 +- heron/tools/tracker/src/python/config.py | 2 +- heron/tools/tracker/src/python/constants.py | 2 +- heron/tools/tracker/src/python/graph.py | 2 +- heron/tools/tracker/src/python/handlers/basehandler.py | 2 +- heron/tools/tracker/src/python/handlers/clustershandler.py | 2 +- .../tracker/src/python/handlers/containerfilehandler.py | 2 +- heron/tools/tracker/src/python/handlers/defaulthandler.py | 2 +- .../tools/tracker/src/python/handlers/exceptionhandler.py | 2 +- .../tracker/src/python/handlers/exceptionsummaryhandler.py | 2 +- .../tracker/src/python/handlers/executionstatehandler.py | 2 +- heron/tools/tracker/src/python/handlers/jmaphandler.py | 2 +- heron/tools/tracker/src/python/handlers/jstackhandler.py | 2 +- .../tracker/src/python/handlers/logicalplanhandler.py | 2 +- heron/tools/tracker/src/python/handlers/machineshandler.py | 2 +- heron/tools/tracker/src/python/handlers/mainhandler.py | 2 +- .../tracker/src/python/handlers/memoryhistogramhandler.py | 2 +- heron/tools/tracker/src/python/handlers/metadatahandler.py | 2 +- heron/tools/tracker/src/python/handlers/metricshandler.py | 2 +- .../tracker/src/python/handlers/metricsqueryhandler.py | 2 +- .../tracker/src/python/handlers/metricstimelinehandler.py | 2 +- .../tracker/src/python/handlers/packingplanhandler.py | 2 +- .../tracker/src/python/handlers/physicalplanhandler.py | 2 +- heron/tools/tracker/src/python/handlers/pidhandler.py | 2 +- .../tracker/src/python/handlers/runtimestatehandler.py | 2 +- .../src/python/handlers/schedulerlocationhandler.py | 2 +- heron/tools/tracker/src/python/handlers/stateshandler.py | 2 +- .../tools/tracker/src/python/handlers/topologieshandler.py | 2 +- .../tracker/src/python/handlers/topologyconfighandler.py | 2 +- heron/tools/tracker/src/python/handlers/topologyhandler.py | 2 +- heron/tools/tracker/src/python/javaobj.py | 2 +- heron/tools/tracker/src/python/main.py | 2 +- heron/tools/tracker/src/python/metricstimeline.py | 2 +- heron/tools/tracker/src/python/pyutils.py | 2 +- heron/tools/tracker/src/python/query.py | 2 +- heron/tools/tracker/src/python/query_operators.py | 2 +- heron/tools/tracker/src/python/topology.py | 2 +- heron/tools/tracker/src/python/tracker.py | 2 +- heron/tools/tracker/src/python/utils.py | 2 +- heron/tools/ui/src/python/args.py | 2 +- heron/tools/ui/src/python/consts.py | 2 +- heron/tools/ui/src/python/handlers/api/metrics.py | 2 +- heron/tools/ui/src/python/handlers/api/topology.py | 2 +- heron/tools/ui/src/python/handlers/base.py | 2 +- heron/tools/ui/src/python/handlers/common/consts.py | 2 +- heron/tools/ui/src/python/handlers/common/utils.py | 2 +- heron/tools/ui/src/python/handlers/mainhandler.py | 2 +- heron/tools/ui/src/python/handlers/notfound.py | 2 +- heron/tools/ui/src/python/handlers/ranges.py | 2 +- heron/tools/ui/src/python/handlers/topology.py | 2 +- heron/tools/ui/src/python/main.py | 2 +- heronpy/api/api_constants.py | 2 +- heronpy/api/bolt/window_bolt.py | 2 +- heronpy/api/component/component_spec.py | 2 +- heronpy/api/custom_grouping.py | 2 +- heronpy/api/global_metrics.py | 2 +- heronpy/api/metrics.py | 2 +- heronpy/api/serializer.py | 2 +- heronpy/api/spout/spout.py | 2 +- heronpy/api/state/state.py | 2 +- heronpy/api/state/stateful_component.py | 2 +- heronpy/api/task_hook.py | 2 +- heronpy/api/tests/python/component_unittest.py | 2 +- heronpy/api/tests/python/metrics_unittest.py | 2 +- heronpy/api/tests/python/serializer_unittest.py | 2 +- heronpy/api/tests/python/stream_unittest.py | 2 +- heronpy/api/tests/python/topology_unittest.py | 2 +- heronpy/api/topology.py | 2 +- heronpy/api/topology_context.py | 2 +- heronpy/api/tuple.py | 2 +- heronpy/connectors/mock/arraylooper.py | 2 +- heronpy/connectors/pulsar/pulsarspout.py | 2 +- heronpy/connectors/pulsar/pulsarstreamlet.py | 2 +- heronpy/connectors/textfiles/textfilesgenerator.py | 2 +- heronpy/streamlet/builder.py | 2 +- heronpy/streamlet/config.py | 2 +- heronpy/streamlet/context.py | 2 +- heronpy/streamlet/impl/consumebolt.py | 2 +- heronpy/streamlet/impl/contextimpl.py | 2 +- heronpy/streamlet/impl/filterbolt.py | 2 +- heronpy/streamlet/impl/flatmapbolt.py | 2 +- heronpy/streamlet/impl/generatorspout.py | 2 +- heronpy/streamlet/impl/joinbolt.py | 2 +- heronpy/streamlet/impl/logbolt.py | 2 +- heronpy/streamlet/impl/mapbolt.py | 2 +- heronpy/streamlet/impl/reducebykeyandwindowbolt.py | 2 +- heronpy/streamlet/impl/reducebywindowbolt.py | 2 +- heronpy/streamlet/impl/repartitionbolt.py | 2 +- heronpy/streamlet/impl/streamletboltbase.py | 2 +- heronpy/streamlet/impl/streamletspoutbase.py | 2 +- heronpy/streamlet/impl/supplierspout.py | 2 +- heronpy/streamlet/impl/transformbolt.py | 2 +- heronpy/streamlet/impl/unionbolt.py | 2 +- heronpy/streamlet/keyedwindow.py | 2 +- heronpy/streamlet/resources.py | 2 +- heronpy/streamlet/runner.py | 2 +- heronpy/streamlet/streamlet.py | 2 +- heronpy/streamlet/window.py | 2 +- heronpy/streamlet/windowconfig.py | 2 +- integration_test/src/python/common/status.py | 2 +- .../integration_test/common/bolt/count_aggregator_bolt.py | 2 +- .../integration_test/common/bolt/double_tuples_bolt.py | 2 +- .../python/integration_test/common/bolt/identity_bolt.py | 2 +- .../python/integration_test/common/bolt/word_count_bolt.py | 2 +- .../src/python/integration_test/common/spout/ab_spout.py | 2 +- .../src/python/integration_test/core/aggregator_bolt.py | 2 +- .../src/python/integration_test/core/batch_bolt.py | 2 +- .../src/python/integration_test/core/constants.py | 2 +- .../python/integration_test/core/integration_test_bolt.py | 2 +- .../python/integration_test/core/integration_test_spout.py | 2 +- .../src/python/integration_test/core/terminal_bolt.py | 2 +- .../src/python/integration_test/core/test_runner.py | 2 +- .../python/integration_test/core/test_topology_builder.py | 2 +- .../integration_test/topology/all_grouping/all_grouping.py | 2 +- .../topology/basic_one_task/basic_one_task.py | 2 +- .../bolt_double_emit_tuples/bolt_double_emit_tuples.py | 2 +- .../topology/fields_grouping/fields_grouping.py | 2 +- .../topology/global_grouping/global_grouping.py | 2 +- .../multi_spouts_multi_tasks/multi_spouts_multi_tasks.py | 2 +- .../topology/none_grouping/none_grouping.py | 2 +- .../topology/one_bolt_multi_tasks/one_bolt_multi_tasks.py | 2 +- .../one_spout_bolt_multi_tasks.py | 2 +- .../one_spout_multi_tasks/one_spout_multi_tasks.py | 2 +- .../topology/one_spout_two_bolts/one_spout_two_bolts.py | 2 +- .../topology/shuffle_grouping/shuffle_grouping.py | 2 +- .../topology/streamlet/word_count_streamlet.py | 2 +- .../python/integration_test/topology/test_topology_main.py | 2 +- integration_test/src/python/local_test_runner/main.py | 2 +- .../src/python/local_test_runner/test_kill_bolt.py | 2 +- .../src/python/local_test_runner/test_kill_metricsmgr.py | 2 +- .../src/python/local_test_runner/test_kill_stmgr.py | 2 +- .../python/local_test_runner/test_kill_stmgr_metricsmgr.py | 2 +- .../src/python/local_test_runner/test_kill_tmaster.py | 2 +- .../src/python/local_test_runner/test_scale_up.py | 2 +- .../src/python/local_test_runner/test_template.py | 2 +- scripts/shutils/save-logs.py | 2 +- third_party/python/cpplint/cpplint.py | 2 +- tools/bazel.rc | 2 +- tools/rules/pex/wrapper/pex_wrapper.py | 2 +- 274 files changed, 279 insertions(+), 278 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9819a779573..945e6380159 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ addons: packages: - libtool-bin - libcppunit-dev + - python3 - pkg-config - python-dev - python-wheel @@ -41,6 +42,6 @@ script: - g++ --version - which python - python -V - - which python2.7 - - python2.7 -V - - scripts/travis/ci.sh \ No newline at end of file + - which python3 + - python3 -V + - scripts/travis/ci.sh diff --git a/README.md b/README.md index 77ab0dec95a..84a2f251706 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Confluence: https://cwiki.apache.org/confluence/display/HERON #### Heron Requirements: * Java 11 - * Python 2.7 + * Python 3 * Bazel 3.0.0 ## Contact diff --git a/bazel_configure.py b/bazel_configure.py index 02e67f14e6b..63d12480913 100755 --- a/bazel_configure.py +++ b/bazel_configure.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -413,7 +413,7 @@ def main(): env_map['AUTOMAKE'] = discover_tool('automake', 'Automake', 'AUTOMAKE', '1.9.6') env_map['AUTOCONF'] = discover_tool('autoconf', 'Autoconf', 'AUTOCONF', '2.6.3') env_map['MAKE'] = discover_tool('make', 'Make', 'MAKE', '3.81') - env_map['PYTHON'] = discover_tool('python', 'Python', 'PYTHON', '2.7') + env_map['PYTHON3'] = discover_tool('python3', 'Python3', 'PYTHON3', '3.4') if platform == 'Darwin': env_map['LIBTOOL'] = discover_tool('glibtool', 'Libtool', 'LIBTOOL', '2.4.2') diff --git a/config/configure.ac b/config/configure.ac index b7ab8905f7f..caba2fa18de 100644 --- a/config/configure.ac +++ b/config/configure.ac @@ -100,8 +100,8 @@ AM_CONDITIONAL([SYSLINUX], [test x$sys_os = xlinux]) ACX_PTHREAD # Check the python version required -AM_PATH_PYTHON([2.4.3]) -AC_PATH_PROG([PYTHON], [python], [],[]) +AM_PATH_PYTHON([3.4]) +AC_PATH_PROG([PYTHON3], [python3], [],[]) abs_top_builddir=`pwd` AC_SUBST(abs_top_builddir) diff --git a/examples/src/python/bolt/consume_bolt.py b/examples/src/python/bolt/consume_bolt.py index 4a8f70beeed..55df3441d59 100644 --- a/examples/src/python/bolt/consume_bolt.py +++ b/examples/src/python/bolt/consume_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/bolt/count_bolt.py b/examples/src/python/bolt/count_bolt.py index 34ac1e8d57d..0a9a33abf40 100644 --- a/examples/src/python/bolt/count_bolt.py +++ b/examples/src/python/bolt/count_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/bolt/half_ack_bolt.py b/examples/src/python/bolt/half_ack_bolt.py index f7346cbff5d..5b6dab4852e 100644 --- a/examples/src/python/bolt/half_ack_bolt.py +++ b/examples/src/python/bolt/half_ack_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/bolt/stateful_count_bolt.py b/examples/src/python/bolt/stateful_count_bolt.py index 260c0bf8263..26fd0002b42 100644 --- a/examples/src/python/bolt/stateful_count_bolt.py +++ b/examples/src/python/bolt/stateful_count_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/bolt/stream_aggregate_bolt.py b/examples/src/python/bolt/stream_aggregate_bolt.py index e94c3ca2f52..c73af5d6017 100644 --- a/examples/src/python/bolt/stream_aggregate_bolt.py +++ b/examples/src/python/bolt/stream_aggregate_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/bolt/window_size_bolt.py b/examples/src/python/bolt/window_size_bolt.py index 083b793f9ce..56040386b4e 100644 --- a/examples/src/python/bolt/window_size_bolt.py +++ b/examples/src/python/bolt/window_size_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/custom_grouping_topology.py b/examples/src/python/custom_grouping_topology.py index ec6e0e2503b..2d0f199712b 100644 --- a/examples/src/python/custom_grouping_topology.py +++ b/examples/src/python/custom_grouping_topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/half_acking_topology.py b/examples/src/python/half_acking_topology.py index 7c89aebfd5d..e4fd3cc4d84 100644 --- a/examples/src/python/half_acking_topology.py +++ b/examples/src/python/half_acking_topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/join_streamlet_topology.py b/examples/src/python/join_streamlet_topology.py index 866731500bf..82c45dd4ca9 100644 --- a/examples/src/python/join_streamlet_topology.py +++ b/examples/src/python/join_streamlet_topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/misc/test_task_hook.py b/examples/src/python/misc/test_task_hook.py index 8ad2eecd226..f3eb767aedf 100644 --- a/examples/src/python/misc/test_task_hook.py +++ b/examples/src/python/misc/test_task_hook.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/multi_stream_topology.py b/examples/src/python/multi_stream_topology.py index 7507e559fe1..24aad643d46 100644 --- a/examples/src/python/multi_stream_topology.py +++ b/examples/src/python/multi_stream_topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/pulsar_word_count_streamlet.py b/examples/src/python/pulsar_word_count_streamlet.py index 6d6d7d883c5..3e1a4095444 100644 --- a/examples/src/python/pulsar_word_count_streamlet.py +++ b/examples/src/python/pulsar_word_count_streamlet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/spout/multi_stream_spout.py b/examples/src/python/spout/multi_stream_spout.py index b9e60a49589..d2ded5d0808 100644 --- a/examples/src/python/spout/multi_stream_spout.py +++ b/examples/src/python/spout/multi_stream_spout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/spout/stateful_word_spout.py b/examples/src/python/spout/stateful_word_spout.py index 3a09777bf96..c976ab84f5a 100644 --- a/examples/src/python/spout/stateful_word_spout.py +++ b/examples/src/python/spout/stateful_word_spout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/spout/word_spout.py b/examples/src/python/spout/word_spout.py index d8a44c96a01..25ab969500e 100644 --- a/examples/src/python/spout/word_spout.py +++ b/examples/src/python/spout/word_spout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/stateful_word_count_topology.py b/examples/src/python/stateful_word_count_topology.py index 7068aa0e468..b7bd3f82e42 100644 --- a/examples/src/python/stateful_word_count_topology.py +++ b/examples/src/python/stateful_word_count_topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/window_size_topology.py b/examples/src/python/window_size_topology.py index 3ef94097adc..9a2b5130c75 100644 --- a/examples/src/python/window_size_topology.py +++ b/examples/src/python/window_size_topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/word_count_streamlet.py b/examples/src/python/word_count_streamlet.py index 112749ff830..4f2ecd6d850 100644 --- a/examples/src/python/word_count_streamlet.py +++ b/examples/src/python/word_count_streamlet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/examples/src/python/word_count_topology.py b/examples/src/python/word_count_topology.py index c849699771f..ed01cde011c 100644 --- a/examples/src/python/word_count_topology.py +++ b/examples/src/python/word_count_topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/common/src/python/pex_loader.py b/heron/common/src/python/pex_loader.py index 622e357acb8..9603390f9d4 100644 --- a/heron/common/src/python/pex_loader.py +++ b/heron/common/src/python/pex_loader.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/common/src/python/utils/log.py b/heron/common/src/python/utils/log.py index f3e4b015028..f0de9737dfd 100644 --- a/heron/common/src/python/utils/log.py +++ b/heron/common/src/python/utils/log.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/common/src/python/utils/proc.py b/heron/common/src/python/utils/proc.py index 2adcb29c006..969ab7f0c60 100644 --- a/heron/common/src/python/utils/proc.py +++ b/heron/common/src/python/utils/proc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/common/tests/python/pex_loader/constants.py b/heron/common/tests/python/pex_loader/constants.py index 63bdd27255b..ff99926c7b5 100644 --- a/heron/common/tests/python/pex_loader/constants.py +++ b/heron/common/tests/python/pex_loader/constants.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/common/tests/python/pex_loader/pex_loader_unittest.py b/heron/common/tests/python/pex_loader/pex_loader_unittest.py index 06b4b646822..cf92ee14a90 100644 --- a/heron/common/tests/python/pex_loader/pex_loader_unittest.py +++ b/heron/common/tests/python/pex_loader/pex_loader_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/common/tests/python/pex_loader/testdata/src/sample.py b/heron/common/tests/python/pex_loader/testdata/src/sample.py index 382bd6b24ba..d9136f66c4f 100644 --- a/heron/common/tests/python/pex_loader/testdata/src/sample.py +++ b/heron/common/tests/python/pex_loader/testdata/src/sample.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py index 674c3fcb38c..ed20e414ebb 100755 --- a/heron/executor/src/python/heron_executor.py +++ b/heron/executor/src/python/heron_executor.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/executor/tests/python/heron_executor_unittest.py b/heron/executor/tests/python/heron_executor_unittest.py index 255b9581802..061e0541118 100644 --- a/heron/executor/tests/python/heron_executor_unittest.py +++ b/heron/executor/tests/python/heron_executor_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/basics/base_instance.py b/heron/instance/src/python/basics/base_instance.py index 0a175206f35..c37e32a7fc7 100644 --- a/heron/instance/src/python/basics/base_instance.py +++ b/heron/instance/src/python/basics/base_instance.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/basics/bolt_instance.py b/heron/instance/src/python/basics/bolt_instance.py index 33915130a96..38164af2182 100644 --- a/heron/instance/src/python/basics/bolt_instance.py +++ b/heron/instance/src/python/basics/bolt_instance.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/basics/spout_instance.py b/heron/instance/src/python/basics/spout_instance.py index 3b4dccf1629..620de48305a 100644 --- a/heron/instance/src/python/basics/spout_instance.py +++ b/heron/instance/src/python/basics/spout_instance.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/instance/st_heron_instance.py b/heron/instance/src/python/instance/st_heron_instance.py index eeed39edcfc..24d3978f27f 100644 --- a/heron/instance/src/python/instance/st_heron_instance.py +++ b/heron/instance/src/python/instance/st_heron_instance.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/network/event_looper.py b/heron/instance/src/python/network/event_looper.py index 607de7eecd2..e2af1a49cac 100644 --- a/heron/instance/src/python/network/event_looper.py +++ b/heron/instance/src/python/network/event_looper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/network/gateway_looper.py b/heron/instance/src/python/network/gateway_looper.py index 62c8e8ccaa8..a6b9a041938 100644 --- a/heron/instance/src/python/network/gateway_looper.py +++ b/heron/instance/src/python/network/gateway_looper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/network/heron_client.py b/heron/instance/src/python/network/heron_client.py index 61eb99e0a79..49d6a7e0856 100644 --- a/heron/instance/src/python/network/heron_client.py +++ b/heron/instance/src/python/network/heron_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/network/metricsmgr_client.py b/heron/instance/src/python/network/metricsmgr_client.py index 1f63e265ec9..22c38d1e5d3 100644 --- a/heron/instance/src/python/network/metricsmgr_client.py +++ b/heron/instance/src/python/network/metricsmgr_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/network/protocol.py b/heron/instance/src/python/network/protocol.py index e15a85f601d..524d1bc0408 100644 --- a/heron/instance/src/python/network/protocol.py +++ b/heron/instance/src/python/network/protocol.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/network/socket_options.py b/heron/instance/src/python/network/socket_options.py index 7d9321907e1..d23520a5cf8 100644 --- a/heron/instance/src/python/network/socket_options.py +++ b/heron/instance/src/python/network/socket_options.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/network/st_stmgr_client.py b/heron/instance/src/python/network/st_stmgr_client.py index e2317e4fe8d..b993314ebe5 100644 --- a/heron/instance/src/python/network/st_stmgr_client.py +++ b/heron/instance/src/python/network/st_stmgr_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/metrics/metrics_helper.py b/heron/instance/src/python/utils/metrics/metrics_helper.py index 39555bab9d4..e9d49ae8479 100644 --- a/heron/instance/src/python/utils/metrics/metrics_helper.py +++ b/heron/instance/src/python/utils/metrics/metrics_helper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/metrics/py_metrics.py b/heron/instance/src/python/utils/metrics/py_metrics.py index 96b095fb861..26ad2b6b0d4 100644 --- a/heron/instance/src/python/utils/metrics/py_metrics.py +++ b/heron/instance/src/python/utils/metrics/py_metrics.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/misc/communicator.py b/heron/instance/src/python/utils/misc/communicator.py index f43d4d83b84..bf1b4802861 100644 --- a/heron/instance/src/python/utils/misc/communicator.py +++ b/heron/instance/src/python/utils/misc/communicator.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/misc/custom_grouping_helper.py b/heron/instance/src/python/utils/misc/custom_grouping_helper.py index 18819d4dc6e..335c3c37ba3 100644 --- a/heron/instance/src/python/utils/misc/custom_grouping_helper.py +++ b/heron/instance/src/python/utils/misc/custom_grouping_helper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/misc/outgoing_tuple_helper.py b/heron/instance/src/python/utils/misc/outgoing_tuple_helper.py index 71a7f812e9d..77bfc379d5e 100644 --- a/heron/instance/src/python/utils/misc/outgoing_tuple_helper.py +++ b/heron/instance/src/python/utils/misc/outgoing_tuple_helper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/misc/pplan_helper.py b/heron/instance/src/python/utils/misc/pplan_helper.py index 477aaa30f73..f7729f325c2 100644 --- a/heron/instance/src/python/utils/misc/pplan_helper.py +++ b/heron/instance/src/python/utils/misc/pplan_helper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/misc/serializer_helper.py b/heron/instance/src/python/utils/misc/serializer_helper.py index f81d1f00c38..1a9af24524d 100644 --- a/heron/instance/src/python/utils/misc/serializer_helper.py +++ b/heron/instance/src/python/utils/misc/serializer_helper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/system_config.py b/heron/instance/src/python/utils/system_config.py index 63275781fcf..6801fc98a6f 100644 --- a/heron/instance/src/python/utils/system_config.py +++ b/heron/instance/src/python/utils/system_config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/system_constants.py b/heron/instance/src/python/utils/system_constants.py index aa080b66410..fd75423b8bd 100644 --- a/heron/instance/src/python/utils/system_constants.py +++ b/heron/instance/src/python/utils/system_constants.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/topology/topology_context_impl.py b/heron/instance/src/python/utils/topology/topology_context_impl.py index 579aea966db..4372295c3bb 100644 --- a/heron/instance/src/python/utils/topology/topology_context_impl.py +++ b/heron/instance/src/python/utils/topology/topology_context_impl.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/src/python/utils/tuple.py b/heron/instance/src/python/utils/tuple.py index 129c16dd63a..f8946d9c7e6 100644 --- a/heron/instance/src/python/utils/tuple.py +++ b/heron/instance/src/python/utils/tuple.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/mock_protobuf.py b/heron/instance/tests/python/mock_protobuf.py index 13ed8e323e6..fb5a147d664 100644 --- a/heron/instance/tests/python/mock_protobuf.py +++ b/heron/instance/tests/python/mock_protobuf.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/network/event_looper_unittest.py b/heron/instance/tests/python/network/event_looper_unittest.py index 69f4249d74b..0fe3411eedc 100644 --- a/heron/instance/tests/python/network/event_looper_unittest.py +++ b/heron/instance/tests/python/network/event_looper_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/network/gateway_looper_unittest.py b/heron/instance/tests/python/network/gateway_looper_unittest.py index 8f8b3489c77..de8ecc72331 100644 --- a/heron/instance/tests/python/network/gateway_looper_unittest.py +++ b/heron/instance/tests/python/network/gateway_looper_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/network/heron_client_unittest.py b/heron/instance/tests/python/network/heron_client_unittest.py index ead3a366976..68e4711a836 100644 --- a/heron/instance/tests/python/network/heron_client_unittest.py +++ b/heron/instance/tests/python/network/heron_client_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/network/metricsmgr_client_unittest.py b/heron/instance/tests/python/network/metricsmgr_client_unittest.py index 605d8365b6f..39cbe39f640 100644 --- a/heron/instance/tests/python/network/metricsmgr_client_unittest.py +++ b/heron/instance/tests/python/network/metricsmgr_client_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/network/mock_generator.py b/heron/instance/tests/python/network/mock_generator.py index 708471b480b..e5893285f9e 100644 --- a/heron/instance/tests/python/network/mock_generator.py +++ b/heron/instance/tests/python/network/mock_generator.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/network/mock_generator_client.py b/heron/instance/tests/python/network/mock_generator_client.py index 7e61cb01e93..0f88d6db5aa 100644 --- a/heron/instance/tests/python/network/mock_generator_client.py +++ b/heron/instance/tests/python/network/mock_generator_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/network/protocol_unittest.py b/heron/instance/tests/python/network/protocol_unittest.py index e76b0c3aeab..f408795ba1e 100644 --- a/heron/instance/tests/python/network/protocol_unittest.py +++ b/heron/instance/tests/python/network/protocol_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/network/st_stmgr_client_unittest.py b/heron/instance/tests/python/network/st_stmgr_client_unittest.py index 1bc63b7a6e8..9bc9855d37d 100644 --- a/heron/instance/tests/python/network/st_stmgr_client_unittest.py +++ b/heron/instance/tests/python/network/st_stmgr_client_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/communicator_unittest.py b/heron/instance/tests/python/utils/communicator_unittest.py index 31f3a486034..caf1c217361 100644 --- a/heron/instance/tests/python/utils/communicator_unittest.py +++ b/heron/instance/tests/python/utils/communicator_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/custom_grouping_unittest.py b/heron/instance/tests/python/utils/custom_grouping_unittest.py index 8ba69d18327..00cc8abddc4 100644 --- a/heron/instance/tests/python/utils/custom_grouping_unittest.py +++ b/heron/instance/tests/python/utils/custom_grouping_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/global_metrics_unittest.py b/heron/instance/tests/python/utils/global_metrics_unittest.py index 2216d76ae28..90df97424a2 100644 --- a/heron/instance/tests/python/utils/global_metrics_unittest.py +++ b/heron/instance/tests/python/utils/global_metrics_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/log_unittest.py b/heron/instance/tests/python/utils/log_unittest.py index daf6ccf9421..2fde7dd7033 100644 --- a/heron/instance/tests/python/utils/log_unittest.py +++ b/heron/instance/tests/python/utils/log_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/metrics_helper_unittest.py b/heron/instance/tests/python/utils/metrics_helper_unittest.py index ad220241746..6e23eb2a485 100644 --- a/heron/instance/tests/python/utils/metrics_helper_unittest.py +++ b/heron/instance/tests/python/utils/metrics_helper_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/mock_generator.py b/heron/instance/tests/python/utils/mock_generator.py index 0a4060bd407..79c9a5edb56 100644 --- a/heron/instance/tests/python/utils/mock_generator.py +++ b/heron/instance/tests/python/utils/mock_generator.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/outgoing_tuple_helper_unittest.py b/heron/instance/tests/python/utils/outgoing_tuple_helper_unittest.py index 62ff330dc42..c81bedcfd16 100644 --- a/heron/instance/tests/python/utils/outgoing_tuple_helper_unittest.py +++ b/heron/instance/tests/python/utils/outgoing_tuple_helper_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/pplan_helper_unittest.py b/heron/instance/tests/python/utils/pplan_helper_unittest.py index 7de571f7d15..427f28ca54b 100644 --- a/heron/instance/tests/python/utils/pplan_helper_unittest.py +++ b/heron/instance/tests/python/utils/pplan_helper_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/py_metrics_unittest.py b/heron/instance/tests/python/utils/py_metrics_unittest.py index 33ab25cfa07..11bc4ae170e 100644 --- a/heron/instance/tests/python/utils/py_metrics_unittest.py +++ b/heron/instance/tests/python/utils/py_metrics_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/topology_context_impl_unittest.py b/heron/instance/tests/python/utils/topology_context_impl_unittest.py index ed02412ec55..409dbb424d8 100644 --- a/heron/instance/tests/python/utils/topology_context_impl_unittest.py +++ b/heron/instance/tests/python/utils/topology_context_impl_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/instance/tests/python/utils/tuple_helper_unittest.py b/heron/instance/tests/python/utils/tuple_helper_unittest.py index a169907d3de..04c2864ed96 100644 --- a/heron/instance/tests/python/utils/tuple_helper_unittest.py +++ b/heron/instance/tests/python/utils/tuple_helper_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/browsehandler.py b/heron/shell/src/python/handlers/browsehandler.py index c5a35abfbe2..760764cb548 100644 --- a/heron/shell/src/python/handlers/browsehandler.py +++ b/heron/shell/src/python/handlers/browsehandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/downloadhandler.py b/heron/shell/src/python/handlers/downloadhandler.py index 9629e3fe643..20e6adfa517 100644 --- a/heron/shell/src/python/handlers/downloadhandler.py +++ b/heron/shell/src/python/handlers/downloadhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/filedatahandler.py b/heron/shell/src/python/handlers/filedatahandler.py index 75185600e88..c0baa7b7fa5 100644 --- a/heron/shell/src/python/handlers/filedatahandler.py +++ b/heron/shell/src/python/handlers/filedatahandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/filehandler.py b/heron/shell/src/python/handlers/filehandler.py index 55fbf2c5e99..ab26247a780 100644 --- a/heron/shell/src/python/handlers/filehandler.py +++ b/heron/shell/src/python/handlers/filehandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/filestatshandler.py b/heron/shell/src/python/handlers/filestatshandler.py index a3774cab39e..0c32af91d9f 100644 --- a/heron/shell/src/python/handlers/filestatshandler.py +++ b/heron/shell/src/python/handlers/filestatshandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/healthhandler.py b/heron/shell/src/python/handlers/healthhandler.py index 2455ad5a86f..19c8585e85c 100644 --- a/heron/shell/src/python/handlers/healthhandler.py +++ b/heron/shell/src/python/handlers/healthhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/jmaphandler.py b/heron/shell/src/python/handlers/jmaphandler.py index d147e5c23f4..0352addd3b1 100644 --- a/heron/shell/src/python/handlers/jmaphandler.py +++ b/heron/shell/src/python/handlers/jmaphandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/jstackhandler.py b/heron/shell/src/python/handlers/jstackhandler.py index 3e5181c707b..dd1c6ab68be 100644 --- a/heron/shell/src/python/handlers/jstackhandler.py +++ b/heron/shell/src/python/handlers/jstackhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/killexecutorhandler.py b/heron/shell/src/python/handlers/killexecutorhandler.py index 6a0416122af..65eb1e3790a 100644 --- a/heron/shell/src/python/handlers/killexecutorhandler.py +++ b/heron/shell/src/python/handlers/killexecutorhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/memoryhistogramhandler.py b/heron/shell/src/python/handlers/memoryhistogramhandler.py index 7b355cfed3c..5ea48c761cc 100644 --- a/heron/shell/src/python/handlers/memoryhistogramhandler.py +++ b/heron/shell/src/python/handlers/memoryhistogramhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/pidhandler.py b/heron/shell/src/python/handlers/pidhandler.py index da1c5c3e169..b63e5aed444 100644 --- a/heron/shell/src/python/handlers/pidhandler.py +++ b/heron/shell/src/python/handlers/pidhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/handlers/pmaphandler.py b/heron/shell/src/python/handlers/pmaphandler.py index acef828824c..fb507ce6fb6 100644 --- a/heron/shell/src/python/handlers/pmaphandler.py +++ b/heron/shell/src/python/handlers/pmaphandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/main.py b/heron/shell/src/python/main.py index 0f375c8888f..c476eca56e7 100644 --- a/heron/shell/src/python/main.py +++ b/heron/shell/src/python/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/shell/src/python/utils.py b/heron/shell/src/python/utils.py index ffbe2ce19b6..bf9c1bdd5f2 100644 --- a/heron/shell/src/python/utils.py +++ b/heron/shell/src/python/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/src/python/config.py b/heron/statemgrs/src/python/config.py index 14975b3d638..d32c2a1597d 100644 --- a/heron/statemgrs/src/python/config.py +++ b/heron/statemgrs/src/python/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/src/python/configloader.py b/heron/statemgrs/src/python/configloader.py index 384a5c71cc6..9fe98f2c5d0 100644 --- a/heron/statemgrs/src/python/configloader.py +++ b/heron/statemgrs/src/python/configloader.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/src/python/filestatemanager.py b/heron/statemgrs/src/python/filestatemanager.py index ca56e5634df..fca9cc4666a 100644 --- a/heron/statemgrs/src/python/filestatemanager.py +++ b/heron/statemgrs/src/python/filestatemanager.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/src/python/log.py b/heron/statemgrs/src/python/log.py index cf23b294e81..3c3f8009ae3 100644 --- a/heron/statemgrs/src/python/log.py +++ b/heron/statemgrs/src/python/log.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/src/python/stateexceptions.py b/heron/statemgrs/src/python/stateexceptions.py index 69380d98b36..639da76135b 100644 --- a/heron/statemgrs/src/python/stateexceptions.py +++ b/heron/statemgrs/src/python/stateexceptions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/src/python/statemanager.py b/heron/statemgrs/src/python/statemanager.py index a4affff5285..ad591fa8797 100644 --- a/heron/statemgrs/src/python/statemanager.py +++ b/heron/statemgrs/src/python/statemanager.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/src/python/statemanagerfactory.py b/heron/statemgrs/src/python/statemanagerfactory.py index 86b51f7271f..a0d03b5f793 100644 --- a/heron/statemgrs/src/python/statemanagerfactory.py +++ b/heron/statemgrs/src/python/statemanagerfactory.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/src/python/zkstatemanager.py b/heron/statemgrs/src/python/zkstatemanager.py index 438a0199a7a..31cd056eb50 100644 --- a/heron/statemgrs/src/python/zkstatemanager.py +++ b/heron/statemgrs/src/python/zkstatemanager.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/tests/python/configloader_unittest.py b/heron/statemgrs/tests/python/configloader_unittest.py index 3549e6632bf..7018cada012 100644 --- a/heron/statemgrs/tests/python/configloader_unittest.py +++ b/heron/statemgrs/tests/python/configloader_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/tests/python/statemanagerfactory_unittest.py b/heron/statemgrs/tests/python/statemanagerfactory_unittest.py index dea8cfea56b..0dcfcda19df 100644 --- a/heron/statemgrs/tests/python/statemanagerfactory_unittest.py +++ b/heron/statemgrs/tests/python/statemanagerfactory_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/statemgrs/tests/python/zkstatemanager_unittest.py b/heron/statemgrs/tests/python/zkstatemanager_unittest.py index 8decf633eb2..ee6da0e3448 100644 --- a/heron/statemgrs/tests/python/zkstatemanager_unittest.py +++ b/heron/statemgrs/tests/python/zkstatemanager_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/admin/src/python/main.py b/heron/tools/admin/src/python/main.py index 4986d3045e6..3283eaacb07 100644 --- a/heron/tools/admin/src/python/main.py +++ b/heron/tools/admin/src/python/main.py @@ -16,7 +16,7 @@ # under the License. -# !/usr/bin/env python2.7 +# !/usr/bin/env python3 ''' main.py ''' import argparse import os diff --git a/heron/tools/cli/src/python/activate.py b/heron/tools/cli/src/python/activate.py index daabcb5fdf5..35674847028 100644 --- a/heron/tools/cli/src/python/activate.py +++ b/heron/tools/cli/src/python/activate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/args.py b/heron/tools/cli/src/python/args.py index 3d280ce3e92..60879d98075 100644 --- a/heron/tools/cli/src/python/args.py +++ b/heron/tools/cli/src/python/args.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/cdefs.py b/heron/tools/cli/src/python/cdefs.py index a8ca744e843..d89cb25ef3d 100644 --- a/heron/tools/cli/src/python/cdefs.py +++ b/heron/tools/cli/src/python/cdefs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/cli_helper.py b/heron/tools/cli/src/python/cli_helper.py index 4b73661c645..51195fa53d6 100644 --- a/heron/tools/cli/src/python/cli_helper.py +++ b/heron/tools/cli/src/python/cli_helper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/config.py b/heron/tools/cli/src/python/config.py index a48a4f09e97..870ef037d9f 100644 --- a/heron/tools/cli/src/python/config.py +++ b/heron/tools/cli/src/python/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/deactivate.py b/heron/tools/cli/src/python/deactivate.py index 9534de8164e..7cef5aba4e9 100644 --- a/heron/tools/cli/src/python/deactivate.py +++ b/heron/tools/cli/src/python/deactivate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/execute.py b/heron/tools/cli/src/python/execute.py index 9534d1c4a9e..99c4e28645d 100644 --- a/heron/tools/cli/src/python/execute.py +++ b/heron/tools/cli/src/python/execute.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/help.py b/heron/tools/cli/src/python/help.py index 6edd8a0aefd..a535de83ec7 100644 --- a/heron/tools/cli/src/python/help.py +++ b/heron/tools/cli/src/python/help.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/jars.py b/heron/tools/cli/src/python/jars.py index 611de05c7c2..32152944c2e 100644 --- a/heron/tools/cli/src/python/jars.py +++ b/heron/tools/cli/src/python/jars.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/kill.py b/heron/tools/cli/src/python/kill.py index d2902695a0c..2b85fcde50b 100644 --- a/heron/tools/cli/src/python/kill.py +++ b/heron/tools/cli/src/python/kill.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/main.py b/heron/tools/cli/src/python/main.py index c37ce09914b..91f67c2823f 100644 --- a/heron/tools/cli/src/python/main.py +++ b/heron/tools/cli/src/python/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/opts.py b/heron/tools/cli/src/python/opts.py index be29bf22fd7..160be97796e 100644 --- a/heron/tools/cli/src/python/opts.py +++ b/heron/tools/cli/src/python/opts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/rest.py b/heron/tools/cli/src/python/rest.py index cc95a4f0dda..ec6b55403d8 100644 --- a/heron/tools/cli/src/python/rest.py +++ b/heron/tools/cli/src/python/rest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/restart.py b/heron/tools/cli/src/python/restart.py index 9b731fa0cc0..ab8802ef007 100644 --- a/heron/tools/cli/src/python/restart.py +++ b/heron/tools/cli/src/python/restart.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/result.py b/heron/tools/cli/src/python/result.py index e8b556dbdc1..c8e47ff02c9 100644 --- a/heron/tools/cli/src/python/result.py +++ b/heron/tools/cli/src/python/result.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/submit.py b/heron/tools/cli/src/python/submit.py index 2a9c2298bcd..83e9e7f1c56 100644 --- a/heron/tools/cli/src/python/submit.py +++ b/heron/tools/cli/src/python/submit.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/update.py b/heron/tools/cli/src/python/update.py index bc39c7be55a..cc3e05815ae 100644 --- a/heron/tools/cli/src/python/update.py +++ b/heron/tools/cli/src/python/update.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/src/python/version.py b/heron/tools/cli/src/python/version.py index 581a153b64f..b101f94c323 100644 --- a/heron/tools/cli/src/python/version.py +++ b/heron/tools/cli/src/python/version.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/tests/python/client_command_unittest.py b/heron/tools/cli/tests/python/client_command_unittest.py index 00f35a41f3b..891f31a69d3 100644 --- a/heron/tools/cli/tests/python/client_command_unittest.py +++ b/heron/tools/cli/tests/python/client_command_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/cli/tests/python/opts_unittest.py b/heron/tools/cli/tests/python/opts_unittest.py index 2f31ff63628..49e128b2e0c 100644 --- a/heron/tools/cli/tests/python/opts_unittest.py +++ b/heron/tools/cli/tests/python/opts_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/common/src/python/access/fetch.py b/heron/tools/common/src/python/access/fetch.py index 19adf2be391..bc8b8a3da39 100644 --- a/heron/tools/common/src/python/access/fetch.py +++ b/heron/tools/common/src/python/access/fetch.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/common/src/python/access/heron_api.py b/heron/tools/common/src/python/access/heron_api.py index c5a52038616..6e77e51126c 100644 --- a/heron/tools/common/src/python/access/heron_api.py +++ b/heron/tools/common/src/python/access/heron_api.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/common/src/python/access/query.py b/heron/tools/common/src/python/access/query.py index b65dff63ddb..11aa2e88bae 100644 --- a/heron/tools/common/src/python/access/query.py +++ b/heron/tools/common/src/python/access/query.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/common/src/python/access/tracker_access.py b/heron/tools/common/src/python/access/tracker_access.py index 8b49b97e16b..c0700f50ac0 100644 --- a/heron/tools/common/src/python/access/tracker_access.py +++ b/heron/tools/common/src/python/access/tracker_access.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/common/src/python/utils/classpath.py b/heron/tools/common/src/python/utils/classpath.py index a0293afeb59..2fc734d2916 100644 --- a/heron/tools/common/src/python/utils/classpath.py +++ b/heron/tools/common/src/python/utils/classpath.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/common/src/python/utils/config.py b/heron/tools/common/src/python/utils/config.py index 83afb248ead..1d5198a3d53 100644 --- a/heron/tools/common/src/python/utils/config.py +++ b/heron/tools/common/src/python/utils/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/common/src/python/utils/heronparser.py b/heron/tools/common/src/python/utils/heronparser.py index 7a7acae6188..29cb7a8018a 100755 --- a/heron/tools/common/src/python/utils/heronparser.py +++ b/heron/tools/common/src/python/utils/heronparser.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/src/python/args.py b/heron/tools/explorer/src/python/args.py index 2d5aca4285e..c312c35f48f 100644 --- a/heron/tools/explorer/src/python/args.py +++ b/heron/tools/explorer/src/python/args.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/src/python/clusters.py b/heron/tools/explorer/src/python/clusters.py index 4efb25ab316..bdd208f4ef0 100644 --- a/heron/tools/explorer/src/python/clusters.py +++ b/heron/tools/explorer/src/python/clusters.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/src/python/help.py b/heron/tools/explorer/src/python/help.py index 37edee9ea74..6eaf116415a 100644 --- a/heron/tools/explorer/src/python/help.py +++ b/heron/tools/explorer/src/python/help.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/src/python/logicalplan.py b/heron/tools/explorer/src/python/logicalplan.py index f6eb2453934..3b6fa48d97a 100644 --- a/heron/tools/explorer/src/python/logicalplan.py +++ b/heron/tools/explorer/src/python/logicalplan.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/src/python/main.py b/heron/tools/explorer/src/python/main.py index 4147e044387..49006fde069 100644 --- a/heron/tools/explorer/src/python/main.py +++ b/heron/tools/explorer/src/python/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/src/python/opts.py b/heron/tools/explorer/src/python/opts.py index 8b3e0698e04..47c91f0f682 100644 --- a/heron/tools/explorer/src/python/opts.py +++ b/heron/tools/explorer/src/python/opts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/src/python/physicalplan.py b/heron/tools/explorer/src/python/physicalplan.py index 50ad7c3ee31..a577ee2f319 100644 --- a/heron/tools/explorer/src/python/physicalplan.py +++ b/heron/tools/explorer/src/python/physicalplan.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/src/python/topologies.py b/heron/tools/explorer/src/python/topologies.py index dab24a790ab..137f29086d9 100644 --- a/heron/tools/explorer/src/python/topologies.py +++ b/heron/tools/explorer/src/python/topologies.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/src/python/version.py b/heron/tools/explorer/src/python/version.py index 00f7e512f20..84773f59ab7 100644 --- a/heron/tools/explorer/src/python/version.py +++ b/heron/tools/explorer/src/python/version.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/explorer/tests/python/explorer_unittest.py b/heron/tools/explorer/tests/python/explorer_unittest.py index 40703374374..ab942fa6838 100644 --- a/heron/tools/explorer/tests/python/explorer_unittest.py +++ b/heron/tools/explorer/tests/python/explorer_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/config.py b/heron/tools/tracker/src/python/config.py index 089b03e91f6..f19182bf975 100644 --- a/heron/tools/tracker/src/python/config.py +++ b/heron/tools/tracker/src/python/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/constants.py b/heron/tools/tracker/src/python/constants.py index 695a60834cc..2ae59bad5fa 100644 --- a/heron/tools/tracker/src/python/constants.py +++ b/heron/tools/tracker/src/python/constants.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/graph.py b/heron/tools/tracker/src/python/graph.py index 747fd5d69f8..be94778ecca 100644 --- a/heron/tools/tracker/src/python/graph.py +++ b/heron/tools/tracker/src/python/graph.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/basehandler.py b/heron/tools/tracker/src/python/handlers/basehandler.py index 300fbf37ef1..49c2892dded 100644 --- a/heron/tools/tracker/src/python/handlers/basehandler.py +++ b/heron/tools/tracker/src/python/handlers/basehandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/clustershandler.py b/heron/tools/tracker/src/python/handlers/clustershandler.py index d8aeede5b07..73af25539d1 100644 --- a/heron/tools/tracker/src/python/handlers/clustershandler.py +++ b/heron/tools/tracker/src/python/handlers/clustershandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/containerfilehandler.py b/heron/tools/tracker/src/python/handlers/containerfilehandler.py index 234ae2096ce..3132567640d 100644 --- a/heron/tools/tracker/src/python/handlers/containerfilehandler.py +++ b/heron/tools/tracker/src/python/handlers/containerfilehandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/defaulthandler.py b/heron/tools/tracker/src/python/handlers/defaulthandler.py index ff8093ee144..a30443be48f 100644 --- a/heron/tools/tracker/src/python/handlers/defaulthandler.py +++ b/heron/tools/tracker/src/python/handlers/defaulthandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/exceptionhandler.py b/heron/tools/tracker/src/python/handlers/exceptionhandler.py index 2e67e7980ee..c43f28555df 100644 --- a/heron/tools/tracker/src/python/handlers/exceptionhandler.py +++ b/heron/tools/tracker/src/python/handlers/exceptionhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/exceptionsummaryhandler.py b/heron/tools/tracker/src/python/handlers/exceptionsummaryhandler.py index b943e5f7c9b..b613807a397 100644 --- a/heron/tools/tracker/src/python/handlers/exceptionsummaryhandler.py +++ b/heron/tools/tracker/src/python/handlers/exceptionsummaryhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/executionstatehandler.py b/heron/tools/tracker/src/python/handlers/executionstatehandler.py index 4d1793ba0bd..b97c217809f 100644 --- a/heron/tools/tracker/src/python/handlers/executionstatehandler.py +++ b/heron/tools/tracker/src/python/handlers/executionstatehandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/jmaphandler.py b/heron/tools/tracker/src/python/handlers/jmaphandler.py index 2e102cd4a03..6d41c4ca00b 100644 --- a/heron/tools/tracker/src/python/handlers/jmaphandler.py +++ b/heron/tools/tracker/src/python/handlers/jmaphandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/jstackhandler.py b/heron/tools/tracker/src/python/handlers/jstackhandler.py index 48c583f4603..d5f4a64843e 100644 --- a/heron/tools/tracker/src/python/handlers/jstackhandler.py +++ b/heron/tools/tracker/src/python/handlers/jstackhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py b/heron/tools/tracker/src/python/handlers/logicalplanhandler.py index 2da1cd22bbd..25c2acf8b01 100644 --- a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py +++ b/heron/tools/tracker/src/python/handlers/logicalplanhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/machineshandler.py b/heron/tools/tracker/src/python/handlers/machineshandler.py index bef23d97e9b..6dfa8385162 100644 --- a/heron/tools/tracker/src/python/handlers/machineshandler.py +++ b/heron/tools/tracker/src/python/handlers/machineshandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/mainhandler.py b/heron/tools/tracker/src/python/handlers/mainhandler.py index 90e93472ce6..1b09e08c3f3 100644 --- a/heron/tools/tracker/src/python/handlers/mainhandler.py +++ b/heron/tools/tracker/src/python/handlers/mainhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/memoryhistogramhandler.py b/heron/tools/tracker/src/python/handlers/memoryhistogramhandler.py index 10c32b9e488..3ac3293cd70 100644 --- a/heron/tools/tracker/src/python/handlers/memoryhistogramhandler.py +++ b/heron/tools/tracker/src/python/handlers/memoryhistogramhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/metadatahandler.py b/heron/tools/tracker/src/python/handlers/metadatahandler.py index f143258522f..c581ff8c4d8 100644 --- a/heron/tools/tracker/src/python/handlers/metadatahandler.py +++ b/heron/tools/tracker/src/python/handlers/metadatahandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/metricshandler.py b/heron/tools/tracker/src/python/handlers/metricshandler.py index 50471165f10..5f79f6206a6 100644 --- a/heron/tools/tracker/src/python/handlers/metricshandler.py +++ b/heron/tools/tracker/src/python/handlers/metricshandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/metricsqueryhandler.py b/heron/tools/tracker/src/python/handlers/metricsqueryhandler.py index 2ea91835441..a0e367a063c 100644 --- a/heron/tools/tracker/src/python/handlers/metricsqueryhandler.py +++ b/heron/tools/tracker/src/python/handlers/metricsqueryhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/metricstimelinehandler.py b/heron/tools/tracker/src/python/handlers/metricstimelinehandler.py index fb4138555bf..267734e48da 100644 --- a/heron/tools/tracker/src/python/handlers/metricstimelinehandler.py +++ b/heron/tools/tracker/src/python/handlers/metricstimelinehandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/packingplanhandler.py b/heron/tools/tracker/src/python/handlers/packingplanhandler.py index ced6eb18638..387b4889bdf 100644 --- a/heron/tools/tracker/src/python/handlers/packingplanhandler.py +++ b/heron/tools/tracker/src/python/handlers/packingplanhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/physicalplanhandler.py b/heron/tools/tracker/src/python/handlers/physicalplanhandler.py index ff98e5fd61e..2fd6223c121 100644 --- a/heron/tools/tracker/src/python/handlers/physicalplanhandler.py +++ b/heron/tools/tracker/src/python/handlers/physicalplanhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/pidhandler.py b/heron/tools/tracker/src/python/handlers/pidhandler.py index 39d3e9b4400..0eb549110e4 100644 --- a/heron/tools/tracker/src/python/handlers/pidhandler.py +++ b/heron/tools/tracker/src/python/handlers/pidhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/runtimestatehandler.py b/heron/tools/tracker/src/python/handlers/runtimestatehandler.py index b776517e500..44e2fc1b055 100644 --- a/heron/tools/tracker/src/python/handlers/runtimestatehandler.py +++ b/heron/tools/tracker/src/python/handlers/runtimestatehandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/schedulerlocationhandler.py b/heron/tools/tracker/src/python/handlers/schedulerlocationhandler.py index 502e9227c4b..670e1a644aa 100644 --- a/heron/tools/tracker/src/python/handlers/schedulerlocationhandler.py +++ b/heron/tools/tracker/src/python/handlers/schedulerlocationhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/stateshandler.py b/heron/tools/tracker/src/python/handlers/stateshandler.py index c2f398fdf11..2cf80cbff4d 100644 --- a/heron/tools/tracker/src/python/handlers/stateshandler.py +++ b/heron/tools/tracker/src/python/handlers/stateshandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/topologieshandler.py b/heron/tools/tracker/src/python/handlers/topologieshandler.py index 1beb56736ff..cbb9e1b38c3 100644 --- a/heron/tools/tracker/src/python/handlers/topologieshandler.py +++ b/heron/tools/tracker/src/python/handlers/topologieshandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/topologyconfighandler.py b/heron/tools/tracker/src/python/handlers/topologyconfighandler.py index c16a969e104..14cbf42912e 100644 --- a/heron/tools/tracker/src/python/handlers/topologyconfighandler.py +++ b/heron/tools/tracker/src/python/handlers/topologyconfighandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/handlers/topologyhandler.py b/heron/tools/tracker/src/python/handlers/topologyhandler.py index 242cfc1cdf0..0a1b4173209 100644 --- a/heron/tools/tracker/src/python/handlers/topologyhandler.py +++ b/heron/tools/tracker/src/python/handlers/topologyhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/javaobj.py b/heron/tools/tracker/src/python/javaobj.py index d5978baf476..ed962a01b7e 100644 --- a/heron/tools/tracker/src/python/javaobj.py +++ b/heron/tools/tracker/src/python/javaobj.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/main.py b/heron/tools/tracker/src/python/main.py index 581894dbe25..689d0328e15 100644 --- a/heron/tools/tracker/src/python/main.py +++ b/heron/tools/tracker/src/python/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/metricstimeline.py b/heron/tools/tracker/src/python/metricstimeline.py index 5cc0cf11ae0..29fcc2a3cba 100644 --- a/heron/tools/tracker/src/python/metricstimeline.py +++ b/heron/tools/tracker/src/python/metricstimeline.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/pyutils.py b/heron/tools/tracker/src/python/pyutils.py index 8bdff1ade20..71af4a90c3e 100644 --- a/heron/tools/tracker/src/python/pyutils.py +++ b/heron/tools/tracker/src/python/pyutils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/query.py b/heron/tools/tracker/src/python/query.py index 67599a626a5..ceaa23ad5b1 100644 --- a/heron/tools/tracker/src/python/query.py +++ b/heron/tools/tracker/src/python/query.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/query_operators.py b/heron/tools/tracker/src/python/query_operators.py index dd348a7e041..216f1f5a1fa 100644 --- a/heron/tools/tracker/src/python/query_operators.py +++ b/heron/tools/tracker/src/python/query_operators.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/topology.py b/heron/tools/tracker/src/python/topology.py index 0c55856bad7..798b43d5003 100644 --- a/heron/tools/tracker/src/python/topology.py +++ b/heron/tools/tracker/src/python/topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/tracker.py b/heron/tools/tracker/src/python/tracker.py index e2d16fc637d..5cdbea40818 100644 --- a/heron/tools/tracker/src/python/tracker.py +++ b/heron/tools/tracker/src/python/tracker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/tracker/src/python/utils.py b/heron/tools/tracker/src/python/utils.py index 6e7bed0f9fc..1b61232b671 100644 --- a/heron/tools/tracker/src/python/utils.py +++ b/heron/tools/tracker/src/python/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/args.py b/heron/tools/ui/src/python/args.py index 14f996d1577..b5c040492f8 100644 --- a/heron/tools/ui/src/python/args.py +++ b/heron/tools/ui/src/python/args.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/consts.py b/heron/tools/ui/src/python/consts.py index 7c643a7c986..63e7adff9c7 100644 --- a/heron/tools/ui/src/python/consts.py +++ b/heron/tools/ui/src/python/consts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/handlers/api/metrics.py b/heron/tools/ui/src/python/handlers/api/metrics.py index 770d2708f50..d70b1fd1b74 100644 --- a/heron/tools/ui/src/python/handlers/api/metrics.py +++ b/heron/tools/ui/src/python/handlers/api/metrics.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/handlers/api/topology.py b/heron/tools/ui/src/python/handlers/api/topology.py index f7c910076d5..6dd20dbcf53 100644 --- a/heron/tools/ui/src/python/handlers/api/topology.py +++ b/heron/tools/ui/src/python/handlers/api/topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/handlers/base.py b/heron/tools/ui/src/python/handlers/base.py index 1d0ee7d8595..059f3f6cc4b 100644 --- a/heron/tools/ui/src/python/handlers/base.py +++ b/heron/tools/ui/src/python/handlers/base.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/handlers/common/consts.py b/heron/tools/ui/src/python/handlers/common/consts.py index b9a34218409..d03315359d6 100644 --- a/heron/tools/ui/src/python/handlers/common/consts.py +++ b/heron/tools/ui/src/python/handlers/common/consts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/handlers/common/utils.py b/heron/tools/ui/src/python/handlers/common/utils.py index 5b49236137b..ccb7b8553a0 100644 --- a/heron/tools/ui/src/python/handlers/common/utils.py +++ b/heron/tools/ui/src/python/handlers/common/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/handlers/mainhandler.py b/heron/tools/ui/src/python/handlers/mainhandler.py index a3d82f287b8..59b5b5dee3b 100644 --- a/heron/tools/ui/src/python/handlers/mainhandler.py +++ b/heron/tools/ui/src/python/handlers/mainhandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/handlers/notfound.py b/heron/tools/ui/src/python/handlers/notfound.py index 57547dca969..57c62a031ea 100644 --- a/heron/tools/ui/src/python/handlers/notfound.py +++ b/heron/tools/ui/src/python/handlers/notfound.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/handlers/ranges.py b/heron/tools/ui/src/python/handlers/ranges.py index be1f5ec379c..eca3d4286bb 100644 --- a/heron/tools/ui/src/python/handlers/ranges.py +++ b/heron/tools/ui/src/python/handlers/ranges.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/handlers/topology.py b/heron/tools/ui/src/python/handlers/topology.py index 441a1745afe..98f502e13ea 100644 --- a/heron/tools/ui/src/python/handlers/topology.py +++ b/heron/tools/ui/src/python/handlers/topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heron/tools/ui/src/python/main.py b/heron/tools/ui/src/python/main.py index ee96365ed4e..3ca49476e2a 100644 --- a/heron/tools/ui/src/python/main.py +++ b/heron/tools/ui/src/python/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/api_constants.py b/heronpy/api/api_constants.py index b9e16e1a077..d4cbf653f67 100644 --- a/heronpy/api/api_constants.py +++ b/heronpy/api/api_constants.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/bolt/window_bolt.py b/heronpy/api/bolt/window_bolt.py index 901e08e88e9..892994c2f19 100644 --- a/heronpy/api/bolt/window_bolt.py +++ b/heronpy/api/bolt/window_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/component/component_spec.py b/heronpy/api/component/component_spec.py index bf134d7b48a..dc00aadadc3 100644 --- a/heronpy/api/component/component_spec.py +++ b/heronpy/api/component/component_spec.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/custom_grouping.py b/heronpy/api/custom_grouping.py index 6c72a30fac2..4654c90d209 100644 --- a/heronpy/api/custom_grouping.py +++ b/heronpy/api/custom_grouping.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/global_metrics.py b/heronpy/api/global_metrics.py index cf052b14a36..a9bf599d1b6 100644 --- a/heronpy/api/global_metrics.py +++ b/heronpy/api/global_metrics.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/metrics.py b/heronpy/api/metrics.py index b944d58d1bf..848d972b120 100644 --- a/heronpy/api/metrics.py +++ b/heronpy/api/metrics.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/serializer.py b/heronpy/api/serializer.py index 6a922f63b8a..4441c43cd03 100644 --- a/heronpy/api/serializer.py +++ b/heronpy/api/serializer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/spout/spout.py b/heronpy/api/spout/spout.py index 81ff5d73697..504f0820fb8 100644 --- a/heronpy/api/spout/spout.py +++ b/heronpy/api/spout/spout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/state/state.py b/heronpy/api/state/state.py index 7b15481e7d7..43a9d289b2c 100644 --- a/heronpy/api/state/state.py +++ b/heronpy/api/state/state.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/state/stateful_component.py b/heronpy/api/state/stateful_component.py index 33f32af45de..7df55655cf7 100644 --- a/heronpy/api/state/stateful_component.py +++ b/heronpy/api/state/stateful_component.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/task_hook.py b/heronpy/api/task_hook.py index c95a1d8376b..91a863c4eb4 100644 --- a/heronpy/api/task_hook.py +++ b/heronpy/api/task_hook.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/tests/python/component_unittest.py b/heronpy/api/tests/python/component_unittest.py index 9da4a8d95c0..f8b5f46f38f 100644 --- a/heronpy/api/tests/python/component_unittest.py +++ b/heronpy/api/tests/python/component_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/tests/python/metrics_unittest.py b/heronpy/api/tests/python/metrics_unittest.py index 063651a5280..23f35a9af52 100644 --- a/heronpy/api/tests/python/metrics_unittest.py +++ b/heronpy/api/tests/python/metrics_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/tests/python/serializer_unittest.py b/heronpy/api/tests/python/serializer_unittest.py index 65db24a3a94..8b8bf8bc4da 100644 --- a/heronpy/api/tests/python/serializer_unittest.py +++ b/heronpy/api/tests/python/serializer_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/tests/python/stream_unittest.py b/heronpy/api/tests/python/stream_unittest.py index 3869bcb002c..013c1f05eb8 100644 --- a/heronpy/api/tests/python/stream_unittest.py +++ b/heronpy/api/tests/python/stream_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/tests/python/topology_unittest.py b/heronpy/api/tests/python/topology_unittest.py index 80e27aa98d1..bdd172bf124 100644 --- a/heronpy/api/tests/python/topology_unittest.py +++ b/heronpy/api/tests/python/topology_unittest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/topology.py b/heronpy/api/topology.py index c62a6bd4bf3..5de7057a887 100644 --- a/heronpy/api/topology.py +++ b/heronpy/api/topology.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/topology_context.py b/heronpy/api/topology_context.py index e3bb263c904..1a054594c9c 100644 --- a/heronpy/api/topology_context.py +++ b/heronpy/api/topology_context.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/api/tuple.py b/heronpy/api/tuple.py index e21ff5dbb13..e1d07d22fc5 100644 --- a/heronpy/api/tuple.py +++ b/heronpy/api/tuple.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/connectors/mock/arraylooper.py b/heronpy/connectors/mock/arraylooper.py index 3824a2e4db0..b22569674fb 100644 --- a/heronpy/connectors/mock/arraylooper.py +++ b/heronpy/connectors/mock/arraylooper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/connectors/pulsar/pulsarspout.py b/heronpy/connectors/pulsar/pulsarspout.py index 83bc0554293..e81bfb66d1c 100644 --- a/heronpy/connectors/pulsar/pulsarspout.py +++ b/heronpy/connectors/pulsar/pulsarspout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/connectors/pulsar/pulsarstreamlet.py b/heronpy/connectors/pulsar/pulsarstreamlet.py index 319692f888c..2b7665d7484 100644 --- a/heronpy/connectors/pulsar/pulsarstreamlet.py +++ b/heronpy/connectors/pulsar/pulsarstreamlet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/connectors/textfiles/textfilesgenerator.py b/heronpy/connectors/textfiles/textfilesgenerator.py index 832ada65b44..129e04684a1 100644 --- a/heronpy/connectors/textfiles/textfilesgenerator.py +++ b/heronpy/connectors/textfiles/textfilesgenerator.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/builder.py b/heronpy/streamlet/builder.py index aa2bd22530d..06a27e3f052 100644 --- a/heronpy/streamlet/builder.py +++ b/heronpy/streamlet/builder.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/config.py b/heronpy/streamlet/config.py index d141ccc1fac..bbf10fd6da9 100644 --- a/heronpy/streamlet/config.py +++ b/heronpy/streamlet/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/context.py b/heronpy/streamlet/context.py index f1fe62eeee7..bcfbae2d5aa 100644 --- a/heronpy/streamlet/context.py +++ b/heronpy/streamlet/context.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/consumebolt.py b/heronpy/streamlet/impl/consumebolt.py index ae63e563865..3e84d234265 100644 --- a/heronpy/streamlet/impl/consumebolt.py +++ b/heronpy/streamlet/impl/consumebolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/contextimpl.py b/heronpy/streamlet/impl/contextimpl.py index d944ec71019..d74542dea73 100644 --- a/heronpy/streamlet/impl/contextimpl.py +++ b/heronpy/streamlet/impl/contextimpl.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/filterbolt.py b/heronpy/streamlet/impl/filterbolt.py index 820a56ed0c1..30ffd8a8235 100644 --- a/heronpy/streamlet/impl/filterbolt.py +++ b/heronpy/streamlet/impl/filterbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/flatmapbolt.py b/heronpy/streamlet/impl/flatmapbolt.py index 19cd07bffc8..3af381d5005 100644 --- a/heronpy/streamlet/impl/flatmapbolt.py +++ b/heronpy/streamlet/impl/flatmapbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/generatorspout.py b/heronpy/streamlet/impl/generatorspout.py index 4b250ce011b..265b71f0044 100644 --- a/heronpy/streamlet/impl/generatorspout.py +++ b/heronpy/streamlet/impl/generatorspout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/joinbolt.py b/heronpy/streamlet/impl/joinbolt.py index 199e33b87c3..ba708567a64 100644 --- a/heronpy/streamlet/impl/joinbolt.py +++ b/heronpy/streamlet/impl/joinbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/logbolt.py b/heronpy/streamlet/impl/logbolt.py index 1740b01b0d6..c18ada4ecba 100644 --- a/heronpy/streamlet/impl/logbolt.py +++ b/heronpy/streamlet/impl/logbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/mapbolt.py b/heronpy/streamlet/impl/mapbolt.py index 34d82bf5f86..444585242d6 100644 --- a/heronpy/streamlet/impl/mapbolt.py +++ b/heronpy/streamlet/impl/mapbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/reducebykeyandwindowbolt.py b/heronpy/streamlet/impl/reducebykeyandwindowbolt.py index 7cd4a93c857..977ece1789b 100644 --- a/heronpy/streamlet/impl/reducebykeyandwindowbolt.py +++ b/heronpy/streamlet/impl/reducebykeyandwindowbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/reducebywindowbolt.py b/heronpy/streamlet/impl/reducebywindowbolt.py index 6b09ef714e2..4a76cd2a725 100644 --- a/heronpy/streamlet/impl/reducebywindowbolt.py +++ b/heronpy/streamlet/impl/reducebywindowbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/repartitionbolt.py b/heronpy/streamlet/impl/repartitionbolt.py index c4a44a1e6f7..78fc49e223f 100644 --- a/heronpy/streamlet/impl/repartitionbolt.py +++ b/heronpy/streamlet/impl/repartitionbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/streamletboltbase.py b/heronpy/streamlet/impl/streamletboltbase.py index 0d6af23ca50..b4fbd7e9168 100644 --- a/heronpy/streamlet/impl/streamletboltbase.py +++ b/heronpy/streamlet/impl/streamletboltbase.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/streamletspoutbase.py b/heronpy/streamlet/impl/streamletspoutbase.py index 46b6944c7d5..260957c20b7 100644 --- a/heronpy/streamlet/impl/streamletspoutbase.py +++ b/heronpy/streamlet/impl/streamletspoutbase.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/supplierspout.py b/heronpy/streamlet/impl/supplierspout.py index af656c32d8f..46e18339eee 100644 --- a/heronpy/streamlet/impl/supplierspout.py +++ b/heronpy/streamlet/impl/supplierspout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/transformbolt.py b/heronpy/streamlet/impl/transformbolt.py index 4bf9fee6242..a0906a91178 100644 --- a/heronpy/streamlet/impl/transformbolt.py +++ b/heronpy/streamlet/impl/transformbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/impl/unionbolt.py b/heronpy/streamlet/impl/unionbolt.py index 76ed95792bc..fb9b3c5126f 100644 --- a/heronpy/streamlet/impl/unionbolt.py +++ b/heronpy/streamlet/impl/unionbolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/keyedwindow.py b/heronpy/streamlet/keyedwindow.py index b5255dcdf77..51c84f1ccb9 100644 --- a/heronpy/streamlet/keyedwindow.py +++ b/heronpy/streamlet/keyedwindow.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/resources.py b/heronpy/streamlet/resources.py index 75782c1c0a1..e4be526b0d9 100644 --- a/heronpy/streamlet/resources.py +++ b/heronpy/streamlet/resources.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/runner.py b/heronpy/streamlet/runner.py index 2d3d73d586c..a955b42ecda 100644 --- a/heronpy/streamlet/runner.py +++ b/heronpy/streamlet/runner.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/streamlet.py b/heronpy/streamlet/streamlet.py index cad88dce596..4fd83f6f005 100644 --- a/heronpy/streamlet/streamlet.py +++ b/heronpy/streamlet/streamlet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/window.py b/heronpy/streamlet/window.py index 04b895db44f..4113bc4c070 100644 --- a/heronpy/streamlet/window.py +++ b/heronpy/streamlet/window.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/heronpy/streamlet/windowconfig.py b/heronpy/streamlet/windowconfig.py index 1fa0fb54bb7..f0ce7d8e493 100644 --- a/heronpy/streamlet/windowconfig.py +++ b/heronpy/streamlet/windowconfig.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/common/status.py b/integration_test/src/python/common/status.py index d2f69e4bc12..876b3e84faa 100644 --- a/integration_test/src/python/common/status.py +++ b/integration_test/src/python/common/status.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/common/bolt/count_aggregator_bolt.py b/integration_test/src/python/integration_test/common/bolt/count_aggregator_bolt.py index d18ec245840..6bfe7c368a2 100644 --- a/integration_test/src/python/integration_test/common/bolt/count_aggregator_bolt.py +++ b/integration_test/src/python/integration_test/common/bolt/count_aggregator_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/common/bolt/double_tuples_bolt.py b/integration_test/src/python/integration_test/common/bolt/double_tuples_bolt.py index 561db78ce13..68341b475ec 100644 --- a/integration_test/src/python/integration_test/common/bolt/double_tuples_bolt.py +++ b/integration_test/src/python/integration_test/common/bolt/double_tuples_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/common/bolt/identity_bolt.py b/integration_test/src/python/integration_test/common/bolt/identity_bolt.py index 4740f38d99d..1680ce327cc 100644 --- a/integration_test/src/python/integration_test/common/bolt/identity_bolt.py +++ b/integration_test/src/python/integration_test/common/bolt/identity_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/common/bolt/word_count_bolt.py b/integration_test/src/python/integration_test/common/bolt/word_count_bolt.py index 6824309bee0..823aac31ecf 100644 --- a/integration_test/src/python/integration_test/common/bolt/word_count_bolt.py +++ b/integration_test/src/python/integration_test/common/bolt/word_count_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/common/spout/ab_spout.py b/integration_test/src/python/integration_test/common/spout/ab_spout.py index 828420c69a1..58d1c592f54 100644 --- a/integration_test/src/python/integration_test/common/spout/ab_spout.py +++ b/integration_test/src/python/integration_test/common/spout/ab_spout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/core/aggregator_bolt.py b/integration_test/src/python/integration_test/core/aggregator_bolt.py index 4fd6cf12e2a..d2047c50b0c 100644 --- a/integration_test/src/python/integration_test/core/aggregator_bolt.py +++ b/integration_test/src/python/integration_test/core/aggregator_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/core/batch_bolt.py b/integration_test/src/python/integration_test/core/batch_bolt.py index d35f09e386b..ecaa4c64415 100644 --- a/integration_test/src/python/integration_test/core/batch_bolt.py +++ b/integration_test/src/python/integration_test/core/batch_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/core/constants.py b/integration_test/src/python/integration_test/core/constants.py index 5b72cd3d758..0530928e647 100644 --- a/integration_test/src/python/integration_test/core/constants.py +++ b/integration_test/src/python/integration_test/core/constants.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/core/integration_test_bolt.py b/integration_test/src/python/integration_test/core/integration_test_bolt.py index cfb4d7d339d..ea51c4cb7a5 100644 --- a/integration_test/src/python/integration_test/core/integration_test_bolt.py +++ b/integration_test/src/python/integration_test/core/integration_test_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/core/integration_test_spout.py b/integration_test/src/python/integration_test/core/integration_test_spout.py index 7c7ddccae75..4edd807703a 100644 --- a/integration_test/src/python/integration_test/core/integration_test_spout.py +++ b/integration_test/src/python/integration_test/core/integration_test_spout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/core/terminal_bolt.py b/integration_test/src/python/integration_test/core/terminal_bolt.py index c4f6afa44b6..6a868bbfa4a 100644 --- a/integration_test/src/python/integration_test/core/terminal_bolt.py +++ b/integration_test/src/python/integration_test/core/terminal_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/core/test_runner.py b/integration_test/src/python/integration_test/core/test_runner.py index a4b6a10dbaa..eb5285fc46c 100644 --- a/integration_test/src/python/integration_test/core/test_runner.py +++ b/integration_test/src/python/integration_test/core/test_runner.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/core/test_topology_builder.py b/integration_test/src/python/integration_test/core/test_topology_builder.py index d64a87401e6..6881644734d 100644 --- a/integration_test/src/python/integration_test/core/test_topology_builder.py +++ b/integration_test/src/python/integration_test/core/test_topology_builder.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/all_grouping/all_grouping.py b/integration_test/src/python/integration_test/topology/all_grouping/all_grouping.py index ba5e67141b0..40980459910 100644 --- a/integration_test/src/python/integration_test/topology/all_grouping/all_grouping.py +++ b/integration_test/src/python/integration_test/topology/all_grouping/all_grouping.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/basic_one_task/basic_one_task.py b/integration_test/src/python/integration_test/topology/basic_one_task/basic_one_task.py index da1edcb1f51..08e232c9090 100644 --- a/integration_test/src/python/integration_test/topology/basic_one_task/basic_one_task.py +++ b/integration_test/src/python/integration_test/topology/basic_one_task/basic_one_task.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/bolt_double_emit_tuples/bolt_double_emit_tuples.py b/integration_test/src/python/integration_test/topology/bolt_double_emit_tuples/bolt_double_emit_tuples.py index c4ddee1e941..fa817e7cdf0 100644 --- a/integration_test/src/python/integration_test/topology/bolt_double_emit_tuples/bolt_double_emit_tuples.py +++ b/integration_test/src/python/integration_test/topology/bolt_double_emit_tuples/bolt_double_emit_tuples.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/fields_grouping/fields_grouping.py b/integration_test/src/python/integration_test/topology/fields_grouping/fields_grouping.py index 8de71679884..5aa00f4d4d7 100644 --- a/integration_test/src/python/integration_test/topology/fields_grouping/fields_grouping.py +++ b/integration_test/src/python/integration_test/topology/fields_grouping/fields_grouping.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/global_grouping/global_grouping.py b/integration_test/src/python/integration_test/topology/global_grouping/global_grouping.py index 898beb6b6c2..7a624244c57 100644 --- a/integration_test/src/python/integration_test/topology/global_grouping/global_grouping.py +++ b/integration_test/src/python/integration_test/topology/global_grouping/global_grouping.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/multi_spouts_multi_tasks/multi_spouts_multi_tasks.py b/integration_test/src/python/integration_test/topology/multi_spouts_multi_tasks/multi_spouts_multi_tasks.py index 52af55b9fae..2d56adefacd 100644 --- a/integration_test/src/python/integration_test/topology/multi_spouts_multi_tasks/multi_spouts_multi_tasks.py +++ b/integration_test/src/python/integration_test/topology/multi_spouts_multi_tasks/multi_spouts_multi_tasks.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/none_grouping/none_grouping.py b/integration_test/src/python/integration_test/topology/none_grouping/none_grouping.py index c69609475c2..66dcd3613aa 100644 --- a/integration_test/src/python/integration_test/topology/none_grouping/none_grouping.py +++ b/integration_test/src/python/integration_test/topology/none_grouping/none_grouping.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/one_bolt_multi_tasks/one_bolt_multi_tasks.py b/integration_test/src/python/integration_test/topology/one_bolt_multi_tasks/one_bolt_multi_tasks.py index b2d808c0f4c..c94b6220fd8 100644 --- a/integration_test/src/python/integration_test/topology/one_bolt_multi_tasks/one_bolt_multi_tasks.py +++ b/integration_test/src/python/integration_test/topology/one_bolt_multi_tasks/one_bolt_multi_tasks.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/one_spout_bolt_multi_tasks/one_spout_bolt_multi_tasks.py b/integration_test/src/python/integration_test/topology/one_spout_bolt_multi_tasks/one_spout_bolt_multi_tasks.py index f0716cb8244..6b0ce646489 100644 --- a/integration_test/src/python/integration_test/topology/one_spout_bolt_multi_tasks/one_spout_bolt_multi_tasks.py +++ b/integration_test/src/python/integration_test/topology/one_spout_bolt_multi_tasks/one_spout_bolt_multi_tasks.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/one_spout_multi_tasks/one_spout_multi_tasks.py b/integration_test/src/python/integration_test/topology/one_spout_multi_tasks/one_spout_multi_tasks.py index 310f15e72e9..0a5559757ef 100644 --- a/integration_test/src/python/integration_test/topology/one_spout_multi_tasks/one_spout_multi_tasks.py +++ b/integration_test/src/python/integration_test/topology/one_spout_multi_tasks/one_spout_multi_tasks.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/one_spout_two_bolts/one_spout_two_bolts.py b/integration_test/src/python/integration_test/topology/one_spout_two_bolts/one_spout_two_bolts.py index adf9b5f8daf..83ba5ac611b 100644 --- a/integration_test/src/python/integration_test/topology/one_spout_two_bolts/one_spout_two_bolts.py +++ b/integration_test/src/python/integration_test/topology/one_spout_two_bolts/one_spout_two_bolts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/shuffle_grouping/shuffle_grouping.py b/integration_test/src/python/integration_test/topology/shuffle_grouping/shuffle_grouping.py index 5bd2bbdc2ea..dafec6be6b3 100644 --- a/integration_test/src/python/integration_test/topology/shuffle_grouping/shuffle_grouping.py +++ b/integration_test/src/python/integration_test/topology/shuffle_grouping/shuffle_grouping.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/streamlet/word_count_streamlet.py b/integration_test/src/python/integration_test/topology/streamlet/word_count_streamlet.py index 7cdc51fba6e..704bac3a7ab 100644 --- a/integration_test/src/python/integration_test/topology/streamlet/word_count_streamlet.py +++ b/integration_test/src/python/integration_test/topology/streamlet/word_count_streamlet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/integration_test/topology/test_topology_main.py b/integration_test/src/python/integration_test/topology/test_topology_main.py index e9b41c19d9d..a83ca597fd4 100644 --- a/integration_test/src/python/integration_test/topology/test_topology_main.py +++ b/integration_test/src/python/integration_test/topology/test_topology_main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/local_test_runner/main.py b/integration_test/src/python/local_test_runner/main.py index 37e59f4fe56..cf45651b0d5 100644 --- a/integration_test/src/python/local_test_runner/main.py +++ b/integration_test/src/python/local_test_runner/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/local_test_runner/test_kill_bolt.py b/integration_test/src/python/local_test_runner/test_kill_bolt.py index 01945e26e5f..92489611512 100644 --- a/integration_test/src/python/local_test_runner/test_kill_bolt.py +++ b/integration_test/src/python/local_test_runner/test_kill_bolt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/local_test_runner/test_kill_metricsmgr.py b/integration_test/src/python/local_test_runner/test_kill_metricsmgr.py index dbff9c1aac4..e8cc9292172 100644 --- a/integration_test/src/python/local_test_runner/test_kill_metricsmgr.py +++ b/integration_test/src/python/local_test_runner/test_kill_metricsmgr.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/local_test_runner/test_kill_stmgr.py b/integration_test/src/python/local_test_runner/test_kill_stmgr.py index bdb28f4c320..0b34e63d14c 100644 --- a/integration_test/src/python/local_test_runner/test_kill_stmgr.py +++ b/integration_test/src/python/local_test_runner/test_kill_stmgr.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/local_test_runner/test_kill_stmgr_metricsmgr.py b/integration_test/src/python/local_test_runner/test_kill_stmgr_metricsmgr.py index 5ae9f6aa9b9..ac54d3511a2 100644 --- a/integration_test/src/python/local_test_runner/test_kill_stmgr_metricsmgr.py +++ b/integration_test/src/python/local_test_runner/test_kill_stmgr_metricsmgr.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/local_test_runner/test_kill_tmaster.py b/integration_test/src/python/local_test_runner/test_kill_tmaster.py index ea7f6c65696..573519b6238 100644 --- a/integration_test/src/python/local_test_runner/test_kill_tmaster.py +++ b/integration_test/src/python/local_test_runner/test_kill_tmaster.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/local_test_runner/test_scale_up.py b/integration_test/src/python/local_test_runner/test_scale_up.py index 93d3faeb4bc..330f62e9836 100644 --- a/integration_test/src/python/local_test_runner/test_scale_up.py +++ b/integration_test/src/python/local_test_runner/test_scale_up.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/integration_test/src/python/local_test_runner/test_template.py b/integration_test/src/python/local_test_runner/test_template.py index c779f01d4d9..9f0d3ba9548 100644 --- a/integration_test/src/python/local_test_runner/test_template.py +++ b/integration_test/src/python/local_test_runner/test_template.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one diff --git a/scripts/shutils/save-logs.py b/scripts/shutils/save-logs.py index c378936b411..5ac5b670f29 100755 --- a/scripts/shutils/save-logs.py +++ b/scripts/shutils/save-logs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/third_party/python/cpplint/cpplint.py b/third_party/python/cpplint/cpplint.py index c30a3bc91d2..4e196e8ea3f 100755 --- a/third_party/python/cpplint/cpplint.py +++ b/third_party/python/cpplint/cpplint.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (c) 2009 Google Inc. All rights reserved. # diff --git a/tools/bazel.rc b/tools/bazel.rc index 0e350971f90..6c625477283 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -17,7 +17,7 @@ build --genrule_strategy=standalone -build --host_force_python=PY2 +build --host_force_python=PY3 build --ignore_unsupported_sandboxing build --spawn_strategy=standalone build --workspace_status_command scripts/release/status.sh diff --git a/tools/rules/pex/wrapper/pex_wrapper.py b/tools/rules/pex/wrapper/pex_wrapper.py index edd9ca83caf..5df48072c15 100644 --- a/tools/rules/pex/wrapper/pex_wrapper.py +++ b/tools/rules/pex/wrapper/pex_wrapper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); From 5eecc703b924f99d01ac946b1d0c4b1b2147463d Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 2 May 2020 15:50:31 +0100 Subject: [PATCH 02/22] Fixes towards python3 support * update heronpy release scripts for python3 * update dist Dockerfiles to only use python3 * remove python2 from docker images * upgrade pylint for python3 support * upgrade PEX so transative dependencies are captured Additionally: * fix Ubuntu 16.04 images * fix linting issues found by newer pylint There is an issue with encapsulation in the builds where the global python3 environment is used while PEX installs a nested transitive dependency of pylint: `pylint>astroid>wrapt`. This seems to be because of logic in its setup.py which can be disabled with `WRAPT_INSTALL_EXTENSIONS=false` --- .travis.yml | 2 +- WORKSPACE | 8 +++---- docker/base/Dockerfile.base.debian9 | 2 +- docker/compile/Dockerfile.centos7 | 3 +-- docker/compile/Dockerfile.debian10 | 6 +++--- docker/compile/Dockerfile.debian9 | 3 +-- docker/compile/Dockerfile.ubuntu14.04 | 2 +- docker/compile/Dockerfile.ubuntu16.04 | 17 ++++++++------- docker/compile/Dockerfile.ubuntu18.04 | 2 +- docker/compile/Dockerfile.ubuntu20.04 | 2 +- docker/dist/Dockerfile.dist.centos7 | 10 +++++---- docker/dist/Dockerfile.dist.debian10 | 4 ++-- docker/dist/Dockerfile.dist.debian9 | 4 ++-- docker/dist/Dockerfile.dist.ubuntu14.04 | 1 - docker/dist/Dockerfile.dist.ubuntu16.04 | 8 ++++--- docker/dist/Dockerfile.dist.ubuntu18.04 | 2 +- docker/dist/Dockerfile.dist.ubuntu20.04 | 2 +- docker/test/Dockerfile.centos7 | 2 +- docker/test/Dockerfile.ubuntu18.04 | 1 - heron/statemgrs/src/python/config.py | 2 +- .../statemgrs/src/python/filestatemanager.py | 12 +++++------ heron/statemgrs/src/python/statemanager.py | 3 --- heron/statemgrs/src/python/zkstatemanager.py | 18 ---------------- scripts/packages/BUILD | 8 +++---- scripts/packages/heronpy/setup.py.template | 8 ++++--- scripts/setup-intellij.sh | 4 ++-- scripts/travis/build.sh | 6 ++---- third_party/python/pylint/BUILD | 5 ++--- third_party/python/pylint/main.py | 21 ------------------- tools/rules/pex/BUILD | 12 +++++++---- tools/rules/pex/wrapper/pex_wrapper.py | 20 +++++------------- tools/rules/pex/wrapper/setup.py | 1 + website2/docs/schedulers-nomad.md | 2 +- .../compiling-docker.md | 2 +- 34 files changed, 80 insertions(+), 125 deletions(-) delete mode 100644 third_party/python/pylint/main.py diff --git a/.travis.yml b/.travis.yml index 945e6380159..0d8ebe8e550 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ addons: - python3 - pkg-config - python-dev - - python-wheel + - python3-wheel - wget - zip - zlib1g-dev diff --git a/WORKSPACE b/WORKSPACE index a00f7f8c23e..e204e5ca1d4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -172,7 +172,7 @@ py_repositories() # pip_repositories() # for pex repos -PEX_SRC = "https://pypi.python.org/packages/3a/1d/cd41cd3765b78a4353bbf27d18b099f7afbcd13e7f2dc9520f304ec8981c/pex-1.2.15.tar.gz" +PEX_WHEEL = "https://pypi.python.org/packages/18/92/99270775cfc5ddb60c19588de1c475f9ff2837a6e0bbd5eaa5286a6a472b/pex-2.1.9-py2.py3-none-any.whl" PY_WHEEL = "https://pypi.python.org/packages/53/67/9620edf7803ab867b175e4fd23c7b8bd8eba11cb761514dcd2e726ef07da/py-1.4.34-py2.py3-none-any.whl" @@ -211,9 +211,9 @@ http_file( http_file( name = "pex_src", - downloaded_file_path = "pex-1.2.15.tar.gz", - sha256 = "0147d19123340677b9793b00ec86fe65b6697db3ec99afb796da2300ae5fec14", - urls = [PEX_SRC], + downloaded_file_path = "pex-2.1.9-py2.py3-none-any.whl", + sha256 = "5cad8d960c187541f71682fc938a843ef9092aab46f27b33ace7e570325e2626", + urls = [PEX_WHEEL], ) http_file( diff --git a/docker/base/Dockerfile.base.debian9 b/docker/base/Dockerfile.base.debian9 index 3a4ba7fe967..652f6e799fe 100644 --- a/docker/base/Dockerfile.base.debian9 +++ b/docker/base/Dockerfile.base.debian9 @@ -19,7 +19,7 @@ FROM openjdk:11-jdk-slim-stretch RUN apt-get -y update && apt-get -y install \ netcat-openbsd \ - python \ + python3 \ unzip \ curl \ supervisor && \ diff --git a/docker/compile/Dockerfile.centos7 b/docker/compile/Dockerfile.centos7 index c117ee95f28..6d1bb0cad40 100644 --- a/docker/compile/Dockerfile.centos7 +++ b/docker/compile/Dockerfile.centos7 @@ -35,8 +35,7 @@ RUN yum -y install \ libtool \ make \ patch \ - python-devel \ - cppunit-devel \ + python3-devel \ zip \ unzip \ wget \ diff --git a/docker/compile/Dockerfile.debian10 b/docker/compile/Dockerfile.debian10 index 2731995f536..fab07045a6c 100644 --- a/docker/compile/Dockerfile.debian10 +++ b/docker/compile/Dockerfile.debian10 @@ -32,10 +32,10 @@ RUN apt-get update && apt-get -y install \ libtool-bin \ libcppunit-dev \ pkg-config \ - python \ - python-dev \ + python3 \ + python3-dev \ software-properties-common \ - python-setuptools \ + python3-setuptools \ tree \ zip \ unzip \ diff --git a/docker/compile/Dockerfile.debian9 b/docker/compile/Dockerfile.debian9 index f5e7b110e6b..e37da48fe24 100644 --- a/docker/compile/Dockerfile.debian9 +++ b/docker/compile/Dockerfile.debian9 @@ -32,9 +32,8 @@ RUN apt-get update && apt-get -y install \ libtool-bin \ libcppunit-dev \ pkg-config \ - python-dev \ - python3-dev \ software-properties-common \ + python3-dev \ python3-setuptools \ tree \ zip \ diff --git a/docker/compile/Dockerfile.ubuntu14.04 b/docker/compile/Dockerfile.ubuntu14.04 index 922cd899fec..1794be75168 100644 --- a/docker/compile/Dockerfile.ubuntu14.04 +++ b/docker/compile/Dockerfile.ubuntu14.04 @@ -33,7 +33,7 @@ RUN apt-get update && apt-get -y install \ libssl-dev \ git \ libtool \ - python-dev \ + python3-dev \ libcppunit-dev \ zip \ unzip \ diff --git a/docker/compile/Dockerfile.ubuntu16.04 b/docker/compile/Dockerfile.ubuntu16.04 index a3a2265f509..fbbcf4383aa 100644 --- a/docker/compile/Dockerfile.ubuntu16.04 +++ b/docker/compile/Dockerfile.ubuntu16.04 @@ -21,23 +21,26 @@ FROM ubuntu:16.04 ENV TARGET_PLATFORM ubuntu ENV bazelVersion 3.0.0 -RUN apt-get update && apt-get -y install \ + +RUN apt-get -y update && apt-get -y install \ automake \ build-essential \ cmake \ curl \ - libssl-dev \ git \ + libssl-dev \ libtool-bin \ - libunwind8 \ libunwind-setjmp0-dev \ - python-dev \ + python3-dev \ libcppunit-dev \ + software-properties-common \ tree \ - zip \ unzip \ - wget \ - openjdk-11-jdk-headless + zip \ + wget + +RUN add-apt-repository ppa:openjdk-r/ppa && apt-get -y update && \ + apt-get -y install openjdk-11-jdk-headless ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04 index 2f115cf25ea..e276c7b461a 100644 --- a/docker/compile/Dockerfile.ubuntu18.04 +++ b/docker/compile/Dockerfile.ubuntu18.04 @@ -29,7 +29,7 @@ RUN apt-get update && apt-get -y install \ libunwind8 \ libcppunit-dev \ patch \ - python-dev \ + python3-dev \ pkg-config \ wget \ zip \ diff --git a/docker/compile/Dockerfile.ubuntu20.04 b/docker/compile/Dockerfile.ubuntu20.04 index c963910a5d0..eab62993d24 100644 --- a/docker/compile/Dockerfile.ubuntu20.04 +++ b/docker/compile/Dockerfile.ubuntu20.04 @@ -31,7 +31,7 @@ RUN apt-get update && apt-get -y install \ libunwind8 \ libcppunit-dev \ patch \ - python-dev \ + python3-dev \ pkg-config \ wget \ zip \ diff --git a/docker/dist/Dockerfile.dist.centos7 b/docker/dist/Dockerfile.dist.centos7 index d548ff5c6e4..6a047ea1b4f 100644 --- a/docker/dist/Dockerfile.dist.centos7 +++ b/docker/dist/Dockerfile.dist.centos7 @@ -18,14 +18,16 @@ FROM centos:centos7 RUN yum -y upgrade -RUN yum -y install python; yum clean all RUN yum -y install unzip; yum clean all RUN yum -y install which; yum clean all RUN yum -y install curl; yum clean all RUN yum -y install nmap-ncat; yum clean all -RUN yum -y install python python3-setuptools; yum clean all -RUN easy_install supervisor -RUN echo_supervisord_conf > /etc/supervisord.conf +RUN yum -y install python3 python3-setuptools; yum clean all +RUN yum -y install epel-release \ + && yum -y update \ + && yum -y install supervisor \ + && yum clean all \ + && echo_supervisord_conf > /etc/supervisord.conf RUN yum -y install java-11-openjdk java-11-openjdk-devel; yum clean all diff --git a/docker/dist/Dockerfile.dist.debian10 b/docker/dist/Dockerfile.dist.debian10 index a78d6f94625..0a12bc25017 100644 --- a/docker/dist/Dockerfile.dist.debian10 +++ b/docker/dist/Dockerfile.dist.debian10 @@ -20,8 +20,8 @@ FROM openjdk:11.0.6-slim-buster RUN apt-get update && apt-get -y install \ netcat-openbsd \ curl \ - python \ - python2.7-dev \ + python3 \ + python3-dev \ supervisor \ unzip diff --git a/docker/dist/Dockerfile.dist.debian9 b/docker/dist/Dockerfile.dist.debian9 index 5d56c1ec2af..7d532f48af7 100644 --- a/docker/dist/Dockerfile.dist.debian9 +++ b/docker/dist/Dockerfile.dist.debian9 @@ -19,8 +19,8 @@ FROM openjdk:11-jdk-slim-stretch RUN apt-get -y update && apt-get -y install \ netcat-openbsd \ - python \ - python-dev \ + python3 \ + python3-dev \ unzip \ curl \ vim \ diff --git a/docker/dist/Dockerfile.dist.ubuntu14.04 b/docker/dist/Dockerfile.dist.ubuntu14.04 index df807aa4e9b..fb1194ecc7f 100644 --- a/docker/dist/Dockerfile.dist.ubuntu14.04 +++ b/docker/dist/Dockerfile.dist.ubuntu14.04 @@ -18,7 +18,6 @@ FROM ubuntu:14.04 RUN apt-get -y update && apt-get -y install \ - python \ python3 \ unzip \ software-properties-common \ diff --git a/docker/dist/Dockerfile.dist.ubuntu16.04 b/docker/dist/Dockerfile.dist.ubuntu16.04 index 0fe59f955a0..dd4c48a00fc 100644 --- a/docker/dist/Dockerfile.dist.ubuntu16.04 +++ b/docker/dist/Dockerfile.dist.ubuntu16.04 @@ -20,13 +20,15 @@ FROM ubuntu:16.04 RUN apt-get update RUN apt-get -y install \ - python \ + python3 \ python3 \ unzip \ software-properties-common \ curl \ - supervisor \ - openjdk-11-jdk-headless + supervisor + +RUN add-apt-repository ppa:openjdk-r/ppa && apt-get -y update && \ + apt-get -y install openjdk-11-jdk-headless ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 RUN update-ca-certificates -f diff --git a/docker/dist/Dockerfile.dist.ubuntu18.04 b/docker/dist/Dockerfile.dist.ubuntu18.04 index 89d4ad7bbbb..893c171afc7 100644 --- a/docker/dist/Dockerfile.dist.ubuntu18.04 +++ b/docker/dist/Dockerfile.dist.ubuntu18.04 @@ -19,7 +19,7 @@ FROM ubuntu:18.04 RUN apt-get update RUN apt-get -y install \ - unzip software-properties-common curl python python3 supervisor openjdk-11-jdk-headless + unzip software-properties-common curl python3 supervisor openjdk-11-jdk-headless ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 RUN update-ca-certificates -f diff --git a/docker/dist/Dockerfile.dist.ubuntu20.04 b/docker/dist/Dockerfile.dist.ubuntu20.04 index 6e2284584b7..204276c3090 100644 --- a/docker/dist/Dockerfile.dist.ubuntu20.04 +++ b/docker/dist/Dockerfile.dist.ubuntu20.04 @@ -21,7 +21,7 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update RUN apt-get -y install \ - unzip software-properties-common curl python python3 supervisor openjdk-11-jdk-headless + unzip software-properties-common curl python3 supervisor openjdk-11-jdk-headless ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 RUN update-ca-certificates -f diff --git a/docker/test/Dockerfile.centos7 b/docker/test/Dockerfile.centos7 index 3a36273dcad..d2ad420fd92 100644 --- a/docker/test/Dockerfile.centos7 +++ b/docker/test/Dockerfile.centos7 @@ -36,7 +36,7 @@ RUN yum -y install \ cppunit-devel \ make \ patch \ - python-devel \ + python3-devel \ python3-devel \ python3-setuptools \ zip \ diff --git a/docker/test/Dockerfile.ubuntu18.04 b/docker/test/Dockerfile.ubuntu18.04 index 2586947ab85..a62a77ab09a 100644 --- a/docker/test/Dockerfile.ubuntu18.04 +++ b/docker/test/Dockerfile.ubuntu18.04 @@ -29,7 +29,6 @@ RUN apt-get update && apt-get -y install \ libunwind8 \ libcppunit-dev \ patch \ - python-dev \ python3-dev \ wget \ zip \ diff --git a/heron/statemgrs/src/python/config.py b/heron/statemgrs/src/python/config.py index d32c2a1597d..d6a282bfd66 100644 --- a/heron/statemgrs/src/python/config.py +++ b/heron/statemgrs/src/python/config.py @@ -20,7 +20,7 @@ ''' config.py ''' -class Config(object): +class Config: """ Responsible for reading the yaml config files and exposing state locations through python methods. diff --git a/heron/statemgrs/src/python/filestatemanager.py b/heron/statemgrs/src/python/filestatemanager.py index fca9cc4666a..8ef1f82d51f 100644 --- a/heron/statemgrs/src/python/filestatemanager.py +++ b/heron/statemgrs/src/python/filestatemanager.py @@ -167,6 +167,7 @@ def get_topologies(self, callback=None): topologies_path = self.get_topologies_path() return [f for f in os.listdir(topologies_path) if os.path.isfile(os.path.join(topologies_path, f))] + return None def get_topology(self, topologyName, callback=None): """get topology""" @@ -179,18 +180,17 @@ def get_topology(self, topologyName, callback=None): topology = Topology() topology.ParseFromString(data) return topology + return None def create_topology(self, topologyName, topology): """ Create path is currently not supported in file based state manager. """ - pass def delete_topology(self, topologyName): """ Delete path is currently not supported in file based state manager. """ - pass def get_packing_plan(self, topologyName, callback=None): """ get packing plan """ @@ -216,18 +216,17 @@ def get_pplan(self, topologyName, callback=None): pplan = PhysicalPlan() pplan.ParseFromString(data) return pplan + return None def create_pplan(self, topologyName, pplan): """ Create path is currently not supported in file based state manager. """ - pass def delete_pplan(self, topologyName): """ Delete path is currently not supported in file based state manager. """ - pass def get_execution_state(self, topologyName, callback=None): """ @@ -242,18 +241,17 @@ def get_execution_state(self, topologyName, callback=None): executionState = ExecutionState() executionState.ParseFromString(data) return executionState + return None def create_execution_state(self, topologyName, executionState): """ Create path is currently not supported in file based state manager. """ - pass def delete_execution_state(self, topologyName): """ Delete path is currently not supported in file based state manager. """ - pass def get_tmaster(self, topologyName, callback=None): """ @@ -268,6 +266,7 @@ def get_tmaster(self, topologyName, callback=None): tmaster = TMasterLocation() tmaster.ParseFromString(data) return tmaster + return None def get_scheduler_location(self, topologyName, callback=None): """ @@ -282,3 +281,4 @@ def get_scheduler_location(self, topologyName, callback=None): scheduler_location = SchedulerLocation() scheduler_location.ParseFromString(data) return scheduler_location + return None diff --git a/heron/statemgrs/src/python/statemanager.py b/heron/statemgrs/src/python/statemanager.py index ad591fa8797..655cf25c2ee 100644 --- a/heron/statemgrs/src/python/statemanager.py +++ b/heron/statemgrs/src/python/statemanager.py @@ -127,12 +127,10 @@ def terminate_ssh_tunnel(self): @abc.abstractmethod def start(self): """ If the state manager needs to connect to a remote host. """ - pass @abc.abstractmethod def stop(self): """ If the state manager had connected to a remote server, it would need to stop as well. """ - pass def get_topologies_path(self): return HERON_TOPOLOGIES_KEY.format(self.rootpath) @@ -179,7 +177,6 @@ def get_packing_plan(self, topologyName, callback=None): sets watch on the path and calls the callback with the new packing_plan. """ - pass @abc.abstractmethod def get_pplan(self, topologyName, callback=None): diff --git a/heron/statemgrs/src/python/zkstatemanager.py b/heron/statemgrs/src/python/zkstatemanager.py index 31cd056eb50..9f2939083c6 100644 --- a/heron/statemgrs/src/python/zkstatemanager.py +++ b/heron/statemgrs/src/python/zkstatemanager.py @@ -206,9 +206,6 @@ def create_topology(self, topologyName, topology): except ZookeeperError: raise_(StateException("Zookeeper while creating topology", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) - except Exception: - # Just re raise the exception. - raise def delete_topology(self, topologyName): """ delete topology """ @@ -227,9 +224,6 @@ def delete_topology(self, topologyName): except ZookeeperError: raise_(StateException("Zookeeper while deleting topology", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) - except Exception: - # Just re raise the exception. - raise def get_packing_plan(self, topologyName, callback=None): """ get packing plan """ @@ -349,9 +343,6 @@ def create_pplan(self, topologyName, pplan): except ZookeeperError: raise_(StateException("Zookeeper while creating pplan", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) - except Exception: - # Just re raise the exception. - raise def delete_pplan(self, topologyName): """ delete physical plan info """ @@ -370,9 +361,6 @@ def delete_pplan(self, topologyName): except ZookeeperError: raise_(StateException("Zookeeper while deleting pplan", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) - except Exception: - # Just re raise the exception. - raise def get_execution_state(self, topologyName, callback=None): """ get execution state """ @@ -445,9 +433,6 @@ def create_execution_state(self, topologyName, executionState): except ZookeeperError: raise_(StateException("Zookeeper while creating execution state", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) - except Exception: - # Just re raise the exception. - raise def delete_execution_state(self, topologyName): """ delete execution state """ @@ -466,9 +451,6 @@ def delete_execution_state(self, topologyName): except ZookeeperError: raise_(StateException("Zookeeper while deleting execution state", StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2]) - except Exception: - # Just re raise the exception. - raise def get_tmaster(self, topologyName, callback=None): """ get tmaster """ diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD index 4f0f7140f96..1cdf4cca229 100644 --- a/scripts/packages/BUILD +++ b/scripts/packages/BUILD @@ -644,12 +644,12 @@ genrule( 'find heronpy -type f -name "*.bak" -delete', "rm setup.py.template", "tree $$HERONPY_DIR", - "/usr/bin/env python2.7 setup.py sdist", - "/usr/bin/env python2.7 setup.py bdist_wheel --universal", + "/usr/bin/env python3 setup.py sdist", + "/usr/bin/env python3 setup.py bdist_wheel", "cd -", "ls -l $$HERONPY_DIR/dist", - "cp $$HERONPY_DIR/dist/heronpy-*-py2.py3-*.whl $$OUTPUT_DIR", - 'cp $$HERONPY_DIR/dist/heronpy-*-py2.py3-*.whl "$@"', + "cp $$HERONPY_DIR/dist/heronpy-*-py3-*.whl $$OUTPUT_DIR", + 'cp $$HERONPY_DIR/dist/heronpy-*-py3-*.whl "$@"', "cp $$HERONPY_DIR/dist/heronpy-*.tar.gz $$OUTPUT_DIR", "touch $$OUTPUT_DIR/heronpy.whl", "rm -rf $$TMP_DIR", diff --git a/scripts/packages/heronpy/setup.py.template b/scripts/packages/heronpy/setup.py.template index 6c7f5219acd..93581407040 100644 --- a/scripts/packages/heronpy/setup.py.template +++ b/scripts/packages/heronpy/setup.py.template @@ -29,7 +29,7 @@ with open(requirement_file, 'r') as f: raw_requirements = f.read().strip() requirements = raw_requirements.split('\n') -print "Requirements: %s" % requirements +print("Requirements: %s" % requirements) long_description = "The heronpy package enables you to write Heron topologies in Python. " \ "Topologies can be run using a variety of schedulers, including Mesos/Aurora," \ @@ -51,14 +51,16 @@ setup( 'Intended Audience :: Developers', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ], keywords='heron topology python', packages=find_packages(), - install_requires=requirements + install_requires=requirements, + python_requires='~=3.4', ) diff --git a/scripts/setup-intellij.sh b/scripts/setup-intellij.sh index 32f4d6de03a..80cfd22db6b 100755 --- a/scripts/setup-intellij.sh +++ b/scripts/setup-intellij.sh @@ -35,7 +35,7 @@ cat > $iml_file < - + @@ -159,7 +159,7 @@ done #write_jar_entry "bazel-bin/heron/metricsmgr/src/thrift" cat >> $iml_file <<'EOF' - + EOF diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh index a02eddf93d3..b0cc412ee66 100755 --- a/scripts/travis/build.sh +++ b/scripts/travis/build.sh @@ -36,9 +36,8 @@ if [ "$JARS" ]; then fi # verify that eggs have not been added to the repo -# ./third_party/pex/wheel-0.23.0-py2.7.egg should be the only one set +e -EGGS=`find . -name "*.egg" | grep -v "third_party/pex/wheel"` +EGGS=`find . -name "*.egg"` set -e if [ "$EGGS" ]; then echo 'ERROR: The following eggs were found in the repo, '\ @@ -49,9 +48,8 @@ if [ "$EGGS" ]; then fi # verify that wheels have not been added to the repo -# ./third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl should be the only one set +e -WHEELS=`find . -name "*.whl" | grep -v "third_party/pex/setuptools"` +WHEELS=`find . -name "*.whl"` set -e if [ "$WHEELS" ]; then echo 'ERROR: The following wheels were found in the repo, '\ diff --git a/third_party/python/pylint/BUILD b/third_party/python/pylint/BUILD index 40a8b181d88..040baf6c066 100644 --- a/third_party/python/pylint/BUILD +++ b/third_party/python/pylint/BUILD @@ -4,7 +4,6 @@ package(default_visibility = ["//visibility:public"]) pex_binary( name = "pylint", - srcs = ["main.py"], - main = "main.py", - reqs = ["pylint==1.5.5"], + entrypoint = "pylint", + reqs = ["pylint==2.5.0"], ) diff --git a/third_party/python/pylint/main.py b/third_party/python/pylint/main.py deleted file mode 100644 index 7ead47190b9..00000000000 --- a/third_party/python/pylint/main.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import pylint - -if __name__ == '__main__': - pylint.run_pylint() diff --git a/tools/rules/pex/BUILD b/tools/rules/pex/BUILD index e9d84631066..510481c45d3 100644 --- a/tools/rules/pex/BUILD +++ b/tools/rules/pex/BUILD @@ -26,9 +26,11 @@ POST_EXECUTE = [ 'ln -sf "$$OUTDIR" "$$TMPF"', 'VENV="$${TMPF}/venv"', '$(location @virtualenv//:virtualenv) --no-download --quiet --clear "$$VENV"', - 'PYTHON="$$VENV/bin/python"', + '# this is just to keep the activate script happy', + 'PS1=', + 'source "$$VENV/bin/activate"', - '$$VENV/bin/pip install pex \ + 'pip install pex \ --quiet --no-cache-dir --no-index --build $(@D)/pexbuild \ --find-links $$(dirname $(location @pex_src//file)) \ --find-links $$(dirname $(location @wheel_src//file)) \ @@ -39,8 +41,10 @@ POST_EXECUTE = [ 'cp -r $$(dirname $(location wrapper/setup.py)) $(@D)/.pex_wrapper', '# Use the bootstrapped pex to build pex_wrapper.pex', - '$$VENV/bin/pex $(@D)/.pex_wrapper \ - --disable-cache --no-index -m pex_wrapper -o $@ \ + 'pex $(@D)/.pex_wrapper \ + --disable-cache --no-index \ + --entry-point=pex_wrapper \ + --output-file=$@ \ --find-links $$(dirname $(location @pex_src//file)) \ --find-links $$(dirname $(location @setuptools_src//file)) \ --find-links $$(dirname $(location @requests_src//file)) \ diff --git a/tools/rules/pex/wrapper/pex_wrapper.py b/tools/rules/pex/wrapper/pex_wrapper.py index 5df48072c15..6454f753c2d 100644 --- a/tools/rules/pex/wrapper/pex_wrapper.py +++ b/tools/rules/pex/wrapper/pex_wrapper.py @@ -52,29 +52,19 @@ def parse_manifest(manifest_text): def main(): - pparser, resolver_options_builder = pexbin.configure_clp() + pparser = pexbin.configure_clp() poptions, args = pparser.parse_args(sys.argv) manifest_file = args[1] manifest_text = open(manifest_file, 'r').read() manifest = parse_manifest(manifest_text) - if poptions.pex_root: - ENV.set('PEX_ROOT', poptions.pex_root) - else: - poptions.pex_root = ENV.PEX_ROOT - - if poptions.cache_dir: - poptions.cache_dir = pexbin.make_relative_to_root(poptions.cache_dir) - poptions.interpreter_cache_dir = pexbin.make_relative_to_root( - poptions.interpreter_cache_dir) - reqs = manifest.get('requirements', []) - with ENV.patch(PEX_VERBOSE=str(poptions.verbosity)): + with ENV.patch(PEX_VERBOSE=str(poptions.verbosity), + PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT): with TRACER.timed('Building pex'): - pex_builder = pexbin.build_pex(reqs, poptions, - resolver_options_builder) + pex_builder = pexbin.build_pex(reqs, poptions) # Add source files from the manifest for modmap in manifest.get('modules', []): @@ -112,7 +102,7 @@ def main(): # TODO(mikekap): Do something about manifest['nativeLibraries']. pexbin.log('Saving PEX file to %s' % poptions.pex_name, - v=poptions.verbosity) + V=poptions.verbosity) tmp_name = poptions.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) diff --git a/tools/rules/pex/wrapper/setup.py b/tools/rules/pex/wrapper/setup.py index 6324befc92b..b8bd36a15dd 100644 --- a/tools/rules/pex/wrapper/setup.py +++ b/tools/rules/pex/wrapper/setup.py @@ -27,6 +27,7 @@ version="0.1", install_requires=[ "pex", + "setuptools", "wheel", # Not strictly required, but requests makes SSL more likely to work "requests", diff --git a/website2/docs/schedulers-nomad.md b/website2/docs/schedulers-nomad.md index 3c411c692da..94025a494c7 100644 --- a/website2/docs/schedulers-nomad.md +++ b/website2/docs/schedulers-nomad.md @@ -39,7 +39,7 @@ The advantages of this mode is that it is incredibly lightweight and likely do n When setting up your Nomad cluster, the following are required: * The [Heron CLI tool](user-manuals-heron-cli) must be installed on each machine used to deploy Heron topologies -* Python 2.7, Java 7 or 8, and [curl](https://curl.haxx.se/) must be installed on every machine in the cluster +* Python 3, Java 7 or 8, and [curl](https://curl.haxx.se/) must be installed on every machine in the cluster * A [ZooKeeper cluster](https://zookeeper.apache.org) ## Configuring Heron settings diff --git a/website2/website/versioned_docs/version-0.20.0-incubating/compiling-docker.md b/website2/website/versioned_docs/version-0.20.0-incubating/compiling-docker.md index dc714487627..a3eb0b6a96d 100644 --- a/website2/website/versioned_docs/version-0.20.0-incubating/compiling-docker.md +++ b/website2/website/versioned_docs/version-0.20.0-incubating/compiling-docker.md @@ -205,7 +205,7 @@ RUN apt-get update && apt-get -y install \ libunwind8 \ libunwind-setjmp0-dev \ python \ - python2.7-dev \ + python3-dev \ python-software-properties \ software-properties-common \ python-setuptools \ From 33d29d25c96a9c08d0f4ba86d5073693ec0d651a Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Mon, 4 May 2020 12:53:25 +0100 Subject: [PATCH 03/22] Fix new pylint issues --- heron/common/src/python/pex_loader.py | 6 +- heron/common/src/python/utils/log.py | 5 +- heron/common/src/python/utils/proc.py | 4 +- .../python/pex_loader/testdata/src/sample.py | 2 +- heron/executor/src/python/heron_executor.py | 6 +- .../tests/python/heron_executor_unittest.py | 2 +- .../src/python/basics/base_instance.py | 4 +- .../src/python/basics/bolt_instance.py | 7 +- .../src/python/basics/spout_instance.py | 10 +-- .../src/python/instance/st_heron_instance.py | 5 +- .../src/python/network/event_looper.py | 12 ++- .../src/python/network/gateway_looper.py | 3 +- .../src/python/network/heron_client.py | 5 +- heron/instance/src/python/network/protocol.py | 13 ++- .../python/utils/metrics/metrics_helper.py | 6 +- .../src/python/utils/misc/communicator.py | 2 +- .../utils/misc/custom_grouping_helper.py | 19 ++--- .../utils/misc/outgoing_tuple_helper.py | 2 +- .../src/python/utils/misc/pplan_helper.py | 16 ++-- .../python/utils/misc/serializer_helper.py | 21 +++-- .../utils/topology/topology_context_impl.py | 3 +- heron/instance/src/python/utils/tuple.py | 2 +- .../python/network/mock_generator_client.py | 2 +- .../src/python/handlers/filedatahandler.py | 3 +- .../shell/src/python/handlers/filehandler.py | 1 + .../python/handlers/killexecutorhandler.py | 5 +- heron/shell/src/python/utils.py | 5 +- heron/tools/admin/src/python/main.py | 5 +- heron/tools/admin/src/python/standalone.py | 19 ++--- heron/tools/cli/src/python/cli_helper.py | 3 +- heron/tools/cli/src/python/config.py | 5 +- heron/tools/cli/src/python/execute.py | 38 +++++---- heron/tools/cli/src/python/help.py | 5 +- heron/tools/cli/src/python/main.py | 5 +- heron/tools/cli/src/python/restart.py | 5 +- heron/tools/cli/src/python/result.py | 18 ++-- heron/tools/cli/src/python/submit.py | 9 +- heron/tools/cli/src/python/update.py | 7 +- heron/tools/common/src/python/access/query.py | 13 ++- .../common/src/python/utils/classpath.py | 9 +- heron/tools/common/src/python/utils/config.py | 14 +-- .../common/src/python/utils/heronparser.py | 5 +- heron/tools/explorer/src/python/help.py | 5 +- heron/tools/explorer/src/python/main.py | 16 ++-- heron/tools/explorer/src/python/topologies.py | 12 ++- heron/tools/tracker/src/python/config.py | 2 +- heron/tools/tracker/src/python/graph.py | 2 +- .../python/handlers/containerfilehandler.py | 1 + heron/tools/tracker/src/python/javaobj.py | 20 ++--- heron/tools/tracker/src/python/pyutils.py | 3 +- heron/tools/tracker/src/python/query.py | 5 +- .../tracker/src/python/query_operators.py | 85 +++++++++---------- heron/tools/tracker/src/python/topology.py | 9 +- heron/tools/tracker/src/python/tracker.py | 24 ++---- heron/tools/tracker/src/python/utils.py | 3 +- .../tools/tracker/tests/python/mock_proto.py | 2 +- .../ui/src/python/handlers/api/topology.py | 8 +- heronpy/api/api_constants.py | 2 +- heronpy/api/bolt/bolt.py | 2 - heronpy/api/bolt/window_bolt.py | 10 +-- heronpy/api/cloudpickle.py | 4 +- heronpy/api/component/base_component.py | 2 +- heronpy/api/component/component_spec.py | 18 ++-- heronpy/api/custom_grouping.py | 4 +- heronpy/api/metrics.py | 11 +-- heronpy/api/serializer.py | 5 +- heronpy/api/spout/spout.py | 6 -- heronpy/api/state/state.py | 17 ++-- heronpy/api/state/stateful_component.py | 4 +- heronpy/api/stream.py | 24 +++--- heronpy/api/task_hook.py | 10 +-- .../api/tests/python/component_unittest.py | 1 - heronpy/api/topology.py | 7 +- heronpy/api/topology_context.py | 10 +-- heronpy/api/tuple.py | 2 +- heronpy/streamlet/builder.py | 2 +- heronpy/streamlet/config.py | 2 +- heronpy/streamlet/context.py | 9 +- heronpy/streamlet/generator.py | 2 +- heronpy/streamlet/impl/joinbolt.py | 1 + heronpy/streamlet/impl/repartitionbolt.py | 2 + heronpy/streamlet/impl/streamletboltbase.py | 2 +- heronpy/streamlet/impl/streamletspoutbase.py | 2 +- heronpy/streamlet/keyedwindow.py | 2 +- heronpy/streamlet/resources.py | 2 +- heronpy/streamlet/runner.py | 3 +- heronpy/streamlet/streamlet.py | 18 +++- heronpy/streamlet/transformoperator.py | 2 +- heronpy/streamlet/window.py | 2 +- heronpy/streamlet/windowconfig.py | 2 +- integration_test/src/python/common/status.py | 2 +- .../python/local_test_runner/test_template.py | 2 +- .../src/python/test_runner/main.py | 8 +- .../src/python/topology_test_runner/main.py | 8 +- third_party/python/cpplint/cpplint.py | 16 ++-- 95 files changed, 335 insertions(+), 421 deletions(-) diff --git a/heron/common/src/python/pex_loader.py b/heron/common/src/python/pex_loader.py index 9603390f9d4..bfcc0da2f72 100644 --- a/heron/common/src/python/pex_loader.py +++ b/heron/common/src/python/pex_loader.py @@ -36,8 +36,8 @@ def _get_deps_list(abs_path_to_pex): Note that dependencies are located under `.deps` directory """ pex = zipfile.ZipFile(abs_path_to_pex, mode='r') - deps = list(set([re.match(egg_regex, i).group(1) for i in pex.namelist() - if re.match(egg_regex, i) is not None])) + deps = list({re.match(egg_regex, i).group(1) for i in pex.namelist() + if re.match(egg_regex, i) is not None}) return deps def load_pex(path_to_pex, include_deps=True): @@ -81,6 +81,7 @@ def resolve_heron_suffix_issue(abs_pex_path, class_path): tests have to have a pex with its top level package name of ``heron``. """ # import top-level package named `heron` of a given pex file + # pylint: disable=no-member importer = zipimport.zipimporter(abs_pex_path) importer.load_module("heron") @@ -89,6 +90,7 @@ def resolve_heron_suffix_issue(abs_pex_path, class_path): loaded = ['heron'] loaded_mod = None for to_load in to_load_lst: + # pylint: disable=no-member sub_importer = zipimport.zipimporter(os.path.join(abs_pex_path, '/'.join(loaded))) loaded_mod = sub_importer.load_module(to_load) loaded.append(to_load) diff --git a/heron/common/src/python/utils/log.py b/heron/common/src/python/utils/log.py index f0de9737dfd..b7fb209f3db 100644 --- a/heron/common/src/python/utils/log.py +++ b/heron/common/src/python/utils/log.py @@ -61,7 +61,6 @@ def configure(level=logging.INFO, logfile=None): # otherwise, use StreamHandler to output to stream (stdout, stderr...) else: log_format = "[%(asctime)s] %(log_color)s[%(levelname)s]%(reset)s: %(message)s" - # pylint: disable=redefined-variable-type formatter = colorlog.ColoredFormatter(fmt=log_format, datefmt=date_format) stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) @@ -85,9 +84,9 @@ def init_rotating_logger(level, logfile, max_files, max_bytes): root_logger.addHandler(handler) for handler in root_logger.handlers: - root_logger.debug("Associated handlers - " + str(handler)) + root_logger.debug("Associated handlers - %s", str(handler)) if isinstance(handler, logging.StreamHandler): - root_logger.debug("Removing StreamHandler: " + str(handler)) + root_logger.debug("Removing StreamHandler: %s", str(handler)) root_logger.handlers.remove(handler) def set_logging_level(cl_args): diff --git a/heron/common/src/python/utils/proc.py b/heron/common/src/python/utils/proc.py index 969ab7f0c60..706c4bfe01f 100644 --- a/heron/common/src/python/utils/proc.py +++ b/heron/common/src/python/utils/proc.py @@ -81,7 +81,7 @@ def async_stream_process_stderr(process, handler): """ return _async_stream_process_output(process, stream_process_stderr, handler) -class StringBuilder(object): +class StringBuilder: def __init__(self): self.end = False self.strs = [] @@ -96,8 +96,6 @@ def result(self): while True: if self.end: return ''.join(self.strs) - else: - continue def async_stdout_builder(proc): """ Save stdout into string builder diff --git a/heron/common/tests/python/pex_loader/testdata/src/sample.py b/heron/common/tests/python/pex_loader/testdata/src/sample.py index d9136f66c4f..fcc4780a4ff 100644 --- a/heron/common/tests/python/pex_loader/testdata/src/sample.py +++ b/heron/common/tests/python/pex_loader/testdata/src/sample.py @@ -20,7 +20,7 @@ '''sample.py: sample module as testdata for pex_loader unittest''' -class SampleClass(object): +class SampleClass: """Sample class""" name = "sample class" age = 100 diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py index ed20e414ebb..a79bf1ca256 100755 --- a/heron/executor/src/python/heron_executor.py +++ b/heron/executor/src/python/heron_executor.py @@ -148,7 +148,7 @@ def stdout_log_fn(cmd): # Log the messages to stdout and strip off the newline because Log.info adds one automatically return lambda line: Log.info("%s stdout: %s", cmd, line.rstrip('\n')) -class Command(object): +class Command: """ Command to run as a separate process using subprocess.POpen :param cmd: command to run (as a list) @@ -179,7 +179,7 @@ def __str__(self): def __eq__(self, other): return self.cmd == other.cmd -class ProcessInfo(object): +class ProcessInfo: def __init__(self, process, name, command, attempts=1): """ Container for info related to a running process @@ -200,7 +200,7 @@ def increment_attempts(self): return self # pylint: disable=too-many-instance-attributes,too-many-statements -class HeronExecutor(object): +class HeronExecutor: """ Heron executor is a class that is responsible for running each of the process on a given container. Based on the container id and the instance distribution, it determines if the container is a master node or a worker node and it starts processes accordingly.""" diff --git a/heron/executor/tests/python/heron_executor_unittest.py b/heron/executor/tests/python/heron_executor_unittest.py index 061e0541118..f8b66a9220c 100644 --- a/heron/executor/tests/python/heron_executor_unittest.py +++ b/heron/executor/tests/python/heron_executor_unittest.py @@ -52,7 +52,7 @@ class CommandEncoder(json.JSONEncoder): def default(self, o): return o.cmd -class MockPOpen(object): +class MockPOpen: """fake subprocess.Popen object that we can use to mock processes and pids""" next_pid = 0 diff --git a/heron/instance/src/python/basics/base_instance.py b/heron/instance/src/python/basics/base_instance.py index c37e32a7fc7..ba114e5c08a 100644 --- a/heron/instance/src/python/basics/base_instance.py +++ b/heron/instance/src/python/basics/base_instance.py @@ -38,7 +38,7 @@ from heronpy.api.state.stateful_component import StatefulComponent # pylint: disable=too-many-instance-attributes -class BaseInstance(object): +class BaseInstance: """The base class for heron bolt/spout instance Implements the following functionality: @@ -83,7 +83,7 @@ def log(self, message, level=None): if level is None: _log_level = logging.INFO else: - if level == "trace" or level == "debug": + if level in ("trace", "debug"): _log_level = logging.DEBUG elif level == "info": _log_level = logging.INFO diff --git a/heron/instance/src/python/basics/bolt_instance.py b/heron/instance/src/python/basics/bolt_instance.py index 38164af2182..ad18bcd051a 100644 --- a/heron/instance/src/python/basics/bolt_instance.py +++ b/heron/instance/src/python/basics/bolt_instance.py @@ -128,8 +128,8 @@ def emit(self, tup, stream=Stream.DEFAULT_STREAM_ID, # Set the anchors for a tuple if anchors is not None: merged_roots = set() - for tup in [t for t in anchors if isinstance(t, HeronTuple) and t.roots is not None]: - merged_roots.update(tup.roots) + for tuple_ in [t for t in anchors if isinstance(t, HeronTuple) and t.roots is not None]: + merged_roots.update(tuple_.roots) for rt in merged_roots: to_add = data_tuple.roots.add() to_add.CopyFrom(rt) @@ -154,6 +154,7 @@ def emit(self, tup, stream=Stream.DEFAULT_STREAM_ID, if direct_task is not None: sent_task_ids.append(direct_task) return sent_task_ids + return None def process_incoming_tuples(self): """Should be called when tuple was buffered into in_stream @@ -184,7 +185,7 @@ def _read_tuples_and_execute(self): if isinstance(tuples, tuple_pb2.HeronTupleSet): if tuples.HasField("control"): raise RuntimeError("Bolt cannot get acks/fails from other components") - elif tuples.HasField("data"): + if tuples.HasField("data"): stream = tuples.data.stream for data_tuple in tuples.data.tuples: diff --git a/heron/instance/src/python/basics/spout_instance.py b/heron/instance/src/python/basics/spout_instance.py index 620de48305a..206dac5862c 100644 --- a/heron/instance/src/python/basics/spout_instance.py +++ b/heron/instance/src/python/basics/spout_instance.py @@ -173,6 +173,7 @@ def emit(self, tup, tup_id=None, stream=Stream.DEFAULT_STREAM_ID, if direct_task is not None: sent_task_ids.append(direct_task) return sent_task_ids + return None # pylint: disable=no-self-use def process_incoming_tuples(self): @@ -191,7 +192,7 @@ def _read_tuples(self): if isinstance(tuples, tuple_pb2.HeronTupleSet): if tuples.HasField("data"): raise RuntimeError("Spout cannot get incoming data tuples from other components") - elif tuples.HasField("control"): + if tuples.HasField("control"): for ack_tuple in tuples.control.acks: self._handle_ack_tuple(ack_tuple, True) for fail_tuple in tuples.control.fails: @@ -291,13 +292,12 @@ def _is_continue_to_work(self): if not self.acking_enabled and self.output_helper.is_out_queue_available(): return True - elif self.acking_enabled and self.output_helper.is_out_queue_available() and \ + if self.acking_enabled and self.output_helper.is_out_queue_available() and \ len(self.in_flight_tuples) < max_spout_pending: return True - elif self.acking_enabled and not self.in_stream.is_empty(): + if self.acking_enabled and not self.in_stream.is_empty(): return True - else: - return False + return False def _look_for_timeouts(self): spout_config = self.pplan_helper.context.get_cluster_config() diff --git a/heron/instance/src/python/instance/st_heron_instance.py b/heron/instance/src/python/instance/st_heron_instance.py index 24d3978f27f..1c91c495740 100644 --- a/heron/instance/src/python/instance/st_heron_instance.py +++ b/heron/instance/src/python/instance/st_heron_instance.py @@ -26,11 +26,10 @@ import resource import signal import traceback -from heron.common.src.python.utils import log import yaml +from heron.common.src.python.utils import log from heron.proto import physical_plan_pb2, tuple_pb2, ckptmgr_pb2, common_pb2 - from heron.instance.src.python.utils.misc import HeronCommunicator from heron.instance.src.python.utils.misc import SerializerHelper from heron.instance.src.python.utils.misc import PhysicalPlanHelper @@ -52,7 +51,7 @@ def set_resource_limit(max_ram): resource.setrlimit(resource.RLIMIT_RSS, (max_ram, max_ram)) # pylint: disable=too-many-instance-attributes -class SingleThreadHeronInstance(object): +class SingleThreadHeronInstance: """SingleThreadHeronInstance is an implementation of Heron Instance in python""" STREAM_MGR_HOST = "127.0.0.1" METRICS_MGR_HOST = "127.0.0.1" diff --git a/heron/instance/src/python/network/event_looper.py b/heron/instance/src/python/network/event_looper.py index e2af1a49cac..4d66cec698d 100644 --- a/heron/instance/src/python/network/event_looper.py +++ b/heron/instance/src/python/network/event_looper.py @@ -28,7 +28,7 @@ from heron.common.src.python.utils.log import Log -class EventLooper(object): +class EventLooper: """EventLooper is a Python implementation of WakeableLooper.java EventLooper is a class for scheduling recurring tasks that could: @@ -85,7 +85,6 @@ def on_exit(self): @abstractmethod def do_wait(self): """Blocking operation, should be implemented by a subclass""" - pass @abstractmethod def wake_up(self): @@ -93,7 +92,6 @@ def wake_up(self): Note that this method should be implemented in a thread-safe way. """ - pass def add_wakeup_task(self, task): """Add a wakeup task @@ -135,9 +133,8 @@ def _get_next_timeout_interval(self): """ if len(self.timer_tasks) == 0: return sys.maxsize - else: - next_timeout_interval = self.timer_tasks[0][0] - time.time() - return next_timeout_interval + next_timeout_interval = self.timer_tasks[0][0] - time.time() + return next_timeout_interval def _execute_wakeup_tasks(self): """Executes wakeup tasks, should only be called from loop()""" @@ -149,6 +146,7 @@ def _execute_wakeup_tasks(self): def _trigger_timers(self): """Triggers expired timers""" current = time.time() - while len(self.timer_tasks) > 0 and (self.timer_tasks[0][0] - current <= 0): + # pylint: disable=chained-comparison + while self.timer_tasks and (self.timer_tasks[0][0] - current <= 0): task = heappop(self.timer_tasks)[1] task() diff --git a/heron/instance/src/python/network/gateway_looper.py b/heron/instance/src/python/network/gateway_looper.py index a6b9a041938..b0f84ac98bd 100644 --- a/heron/instance/src/python/network/gateway_looper.py +++ b/heron/instance/src/python/network/gateway_looper.py @@ -105,8 +105,7 @@ def poll(self, timeout=0.0): Log.debug("Trivial error: " + str(err)) if err.args[0] != errno.EINTR: raise - else: - return + return Log.debug("Selected [r]: " + str(readable_lst) + " [w]: " + str(writable_lst) + " [e]: " + str(error_lst)) diff --git a/heron/instance/src/python/network/heron_client.py b/heron/instance/src/python/network/heron_client.py index 49d6a7e0856..5e63af0645a 100644 --- a/heron/instance/src/python/network/heron_client.py +++ b/heron/instance/src/python/network/heron_client.py @@ -158,6 +158,7 @@ def handle_write(self): write_batch_time_sec = self.socket_options.nw_write_batch_time_ms * constants.MS_TO_SEC write_batch_size_bytes = self.socket_options.nw_write_batch_size_bytes + # pylint: disable=chained-comparison while (time.time() - start_cycle_time - write_batch_time_sec) < 0 and \ bytes_written < write_batch_size_bytes and len(self.out_buffer) > 0: outgoing_pkt = self.out_buffer[0] @@ -320,7 +321,6 @@ def on_connect(self, status): Should be implemented by a subclass. """ - pass @abstractmethod def on_response(self, status, context, response): @@ -328,7 +328,6 @@ def on_response(self, status, context, response): Should be implemented by a subclass. """ - pass @abstractmethod def on_incoming_message(self, message): @@ -336,7 +335,6 @@ def on_incoming_message(self, message): Should be implemented by a subclass. """ - pass @abstractmethod def on_error(self): @@ -345,4 +343,3 @@ def on_error(self): Note that this method is not called when a connection is not yet established. In such a case, ``on_connect()`` with status == StatusCode.CONNECT_ERROR is called. """ - pass diff --git a/heron/instance/src/python/network/protocol.py b/heron/instance/src/python/network/protocol.py index 524d1bc0408..728421ce1e2 100644 --- a/heron/instance/src/python/network/protocol.py +++ b/heron/instance/src/python/network/protocol.py @@ -25,7 +25,7 @@ from heron.common.src.python.utils.log import Log -class HeronProtocol(object): +class HeronProtocol: """Heron's application level network protocol""" INT_PACK_FMT = ">I" HEADER_SIZE = 4 @@ -84,7 +84,7 @@ def decode_packet(packet): return typename, reqid, serialized_msg -class OutgoingPacket(object): +class OutgoingPacket: """Wrapper class for outgoing packet""" def __init__(self, raw_data): self.raw = str(raw_data) @@ -137,7 +137,7 @@ def send(self, dispatcher): sent = dispatcher.send(self.to_send) self.to_send = self.to_send[sent:] -class IncomingPacket(object): +class IncomingPacket: """Helper class for incoming packet""" def __init__(self): """Initializes IncomingPacket object""" @@ -218,7 +218,7 @@ def __str__(self): (str(self.id), self.is_header_read, self.is_complete) -class REQID(object): +class REQID: """Helper class for REQID""" REQID_SIZE = 32 @@ -259,10 +259,9 @@ def __hash__(self): def __str__(self): if self.is_zero(): return "ZERO" - else: - return ''.join([str(i) for i in list(self.bytes)]) + return ''.join([str(i) for i in list(self.bytes)]) -class StatusCode(object): +class StatusCode: """StatusCode for Response""" OK = 0 WRITE_ERROR = 1 diff --git a/heron/instance/src/python/utils/metrics/metrics_helper.py b/heron/instance/src/python/utils/metrics/metrics_helper.py index e9d49ae8479..8e6c0c806b0 100644 --- a/heron/instance/src/python/utils/metrics/metrics_helper.py +++ b/heron/instance/src/python/utils/metrics/metrics_helper.py @@ -29,7 +29,7 @@ from heronpy.api.metrics import (CountMetric, MultiCountMetric, MeanReducedMetric, ReducedMetric, MultiMeanReducedMetric, MultiReducedMetric) -class BaseMetricsHelper(object): +class BaseMetricsHelper: """Helper class for metrics management It registers metrics to the metrics collector and provides methods for @@ -325,7 +325,7 @@ def failed_tuple(self, stream_id, source_component, latency_in_ns): self.update_count(self.FAIL_COUNT, key=global_stream_id) self.update_reduced_metric(self.FAIL_LATENCY, latency_in_ns, global_stream_id) -class MetricsCollector(object): +class MetricsCollector: """Helper class for pushing metrics to Out-Metrics queue""" def __init__(self, looper, out_metrics): self.looper = looper @@ -378,7 +378,7 @@ def _gather_one_metric(self, name, message): if metric_value is None: return - elif isinstance(metric_value, dict): + if isinstance(metric_value, dict): for key, value in list(metric_value.items()): if key is not None and value is not None: self._add_data_to_message(message, name + "/" + str(key), value) diff --git a/heron/instance/src/python/utils/misc/communicator.py b/heron/instance/src/python/utils/misc/communicator.py index bf1b4802861..5f11cbbf89a 100644 --- a/heron/instance/src/python/utils/misc/communicator.py +++ b/heron/instance/src/python/utils/misc/communicator.py @@ -24,7 +24,7 @@ from heron.common.src.python.utils.log import Log -class HeronCommunicator(object): +class HeronCommunicator: """HeronCommunicator: a wrapper class for non-blocking queue in Heron. Note that this class does not yet implement the dynamic tuning of expected available capacity, diff --git a/heron/instance/src/python/utils/misc/custom_grouping_helper.py b/heron/instance/src/python/utils/misc/custom_grouping_helper.py index 335c3c37ba3..b149ceb3d7d 100644 --- a/heron/instance/src/python/utils/misc/custom_grouping_helper.py +++ b/heron/instance/src/python/utils/misc/custom_grouping_helper.py @@ -21,7 +21,7 @@ '''custom_grouping_helper.py''' from collections import namedtuple -class CustomGroupingHelper(object): +class CustomGroupingHelper: """Helper class for managing custom grouping""" def __init__(self): # map list(targets)> @@ -72,12 +72,11 @@ def choose_tasks(self, values): if not isinstance(ret, list): raise TypeError("Returned object after custom grouping's choose_tasks() " "needs to be a list, given: %s" % str(type(ret))) - else: - for i in ret: - if not isinstance(i, int): - raise TypeError("Returned object after custom grouping's choose_tasks() " - "contained non-integer: %s" % str(i)) - if i not in self.task_ids: - raise ValueError("Returned object after custom grouping's choose_tasks() contained " - "a task id that is not registered: %d" % i) - return ret + for i in ret: + if not isinstance(i, int): + raise TypeError("Returned object after custom grouping's choose_tasks() " + "contained non-integer: %s" % str(i)) + if i not in self.task_ids: + raise ValueError("Returned object after custom grouping's choose_tasks() contained " + "a task id that is not registered: %d" % i) + return ret diff --git a/heron/instance/src/python/utils/misc/outgoing_tuple_helper.py b/heron/instance/src/python/utils/misc/outgoing_tuple_helper.py index 77bfc379d5e..53c3556c5f6 100644 --- a/heron/instance/src/python/utils/misc/outgoing_tuple_helper.py +++ b/heron/instance/src/python/utils/misc/outgoing_tuple_helper.py @@ -29,7 +29,7 @@ # pylint: disable=too-many-instance-attributes # pylint: disable=no-value-for-parameter -class OutgoingTupleHelper(object): +class OutgoingTupleHelper: """Helper class for preparing and pushing tuples to Out-Stream This is a Python implementation of OutgoingTupleCollection.java diff --git a/heron/instance/src/python/utils/misc/pplan_helper.py b/heron/instance/src/python/utils/misc/pplan_helper.py index f7729f325c2..62989698ad2 100644 --- a/heron/instance/src/python/utils/misc/pplan_helper.py +++ b/heron/instance/src/python/utils/misc/pplan_helper.py @@ -32,7 +32,7 @@ from .custom_grouping_helper import CustomGroupingHelper # pylint: disable=too-many-instance-attributes -class PhysicalPlanHelper(object): +class PhysicalPlanHelper: """Helper class for accessing Physical Plan :ivar pplan: Physical Plan protobuf message @@ -121,7 +121,7 @@ def check_output_schema(self, stream_id, tup): if size is None: raise RuntimeError("%s emitting to stream %s but was not declared in output fields" % (self.my_component_name, stream_id)) - elif size != len(tup): + if size != len(tup): raise RuntimeError("Number of fields emitted in stream %s does not match what's expected. " "Expected: %s, Observed: %s" % (stream_id, size, len(tup))) @@ -129,15 +129,13 @@ def get_my_spout(self): """Returns spout instance, or ``None`` if bolt is assigned""" if self.is_spout: return self._my_spbl - else: - return None + return None def get_my_bolt(self): """Returns bolt instance, or ``None`` if spout is assigned""" if self.is_spout: return None - else: - return self._my_spbl + return self._my_spbl def get_topology_state(self): """Returns the current topology state""" @@ -159,8 +157,7 @@ def get_topology_config(self): """Returns the topology config""" if self.pplan.topology.HasField("topology_config"): return self._get_dict_from_config(self.pplan.topology.topology_config) - else: - return {} + return {} def set_topology_context(self, metrics_collector): """Sets a new topology context""" @@ -192,9 +189,10 @@ def _get_dict_from_config(topology_config): if PhysicalPlanHelper._is_number(kv.value): config[kv.key] = PhysicalPlanHelper._get_number(kv.value) elif kv.value.lower() in ("true", "false"): - config[kv.key] = True if kv.value.lower() == "true" else False + config[kv.key] = kv.value.lower() == "true" else: config[kv.key] = kv.value + # pytlint: disable=simplifiable-if-expression elif kv.HasField("serialized_value") and \ kv.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"): # deserialize that diff --git a/heron/instance/src/python/utils/misc/serializer_helper.py b/heron/instance/src/python/utils/misc/serializer_helper.py index 1a9af24524d..3004c35887f 100644 --- a/heron/instance/src/python/utils/misc/serializer_helper.py +++ b/heron/instance/src/python/utils/misc/serializer_helper.py @@ -25,7 +25,7 @@ from heronpy.api.serializer import PythonSerializer import heronpy.api.api_constants as constants -class SerializerHelper(object): +class SerializerHelper: """Helper class for getting serializer for component""" @staticmethod def get_serializer(context): @@ -34,13 +34,12 @@ def get_serializer(context): serializer_clsname = cluster_config.get(constants.TOPOLOGY_SERIALIZER_CLASSNAME, None) if serializer_clsname is None: return PythonSerializer() - else: - try: - topo_pex_path = context.get_topology_pex_path() - pex_loader.load_pex(topo_pex_path) - serializer_cls = pex_loader.import_and_get_class(topo_pex_path, serializer_clsname) - serializer = serializer_cls() - return serializer - except Exception as e: - raise RuntimeError("Error with loading custom serializer class: %s, with error message: %s" - % (serializer_clsname, str(e))) + try: + topo_pex_path = context.get_topology_pex_path() + pex_loader.load_pex(topo_pex_path) + serializer_cls = pex_loader.import_and_get_class(topo_pex_path, serializer_clsname) + serializer = serializer_cls() + return serializer + except Exception as e: + raise RuntimeError("Error with loading custom serializer class: %s, with error message: %s" + % (serializer_clsname, str(e))) diff --git a/heron/instance/src/python/utils/topology/topology_context_impl.py b/heron/instance/src/python/utils/topology/topology_context_impl.py index 4372295c3bb..8d3c4c133dc 100644 --- a/heron/instance/src/python/utils/topology/topology_context_impl.py +++ b/heron/instance/src/python/utils/topology/topology_context_impl.py @@ -100,8 +100,7 @@ def get_sources(self, component_id): key = StreamId(id=istream.stream.id, component_name=istream.stream.component_name) ret[key] = istream.gtype return ret - else: - return None + return None def get_this_sources(self): return self.get_sources(self.get_component_id()) diff --git a/heron/instance/src/python/utils/tuple.py b/heron/instance/src/python/utils/tuple.py index f8946d9c7e6..bd608d9d2df 100644 --- a/heron/instance/src/python/utils/tuple.py +++ b/heron/instance/src/python/utils/tuple.py @@ -51,7 +51,7 @@ class RootTupleInfo(namedtuple('RootTupleInfo', 'stream_id tuple_id insertion_ti def is_expired(self, current_time, timeout_sec): return self.insertion_time + timeout_sec - current_time <= 0 -class TupleHelper(object): +class TupleHelper: """Tuple Helper, returns Heron Tuple compatible tuple""" TICK_TUPLE_ID = "__tick" TICK_SOURCE_COMPONENT = "__system" diff --git a/heron/instance/tests/python/network/mock_generator_client.py b/heron/instance/tests/python/network/mock_generator_client.py index 0f88d6db5aa..d9bf527bed0 100644 --- a/heron/instance/tests/python/network/mock_generator_client.py +++ b/heron/instance/tests/python/network/mock_generator_client.py @@ -95,7 +95,7 @@ def get_fail_packet(): packet.is_complete = False return packet -class MockDispatcher(object): +class MockDispatcher: """Mock asyncore.dispatcher class, supporting only recv() and send() method This dispatcher provides several options as to how to prepare its receive buffer. diff --git a/heron/shell/src/python/handlers/filedatahandler.py b/heron/shell/src/python/handlers/filedatahandler.py index c0baa7b7fa5..d9531c0a174 100644 --- a/heron/shell/src/python/handlers/filedatahandler.py +++ b/heron/shell/src/python/handlers/filedatahandler.py @@ -41,7 +41,7 @@ def get(self, path): self.write("Only relative paths are allowed") self.set_status(403) self.finish() - return + return None offset = self.get_argument("offset", default=-1) length = self.get_argument("length", default=-1) @@ -50,3 +50,4 @@ def get(self, path): data = utils.read_chunk(path, offset=offset, length=length, escape_data=True) self.write(json.dumps(data)) self.finish() + return None diff --git a/heron/shell/src/python/handlers/filehandler.py b/heron/shell/src/python/handlers/filehandler.py index ab26247a780..1648c81eded 100644 --- a/heron/shell/src/python/handlers/filehandler.py +++ b/heron/shell/src/python/handlers/filehandler.py @@ -54,3 +54,4 @@ def get(self, path): ) self.write(t.generate(**args)) self.finish() + return diff --git a/heron/shell/src/python/handlers/killexecutorhandler.py b/heron/shell/src/python/handlers/killexecutorhandler.py index 65eb1e3790a..1dadff0731b 100644 --- a/heron/shell/src/python/handlers/killexecutorhandler.py +++ b/heron/shell/src/python/handlers/killexecutorhandler.py @@ -20,6 +20,7 @@ ''' killexecutorhandler.py ''' +# pylint: disable=wrong-import-order from future.standard_library import install_aliases install_aliases() @@ -67,11 +68,11 @@ def kill_parent(): fh = open(filepath) firstLine = int(fh.readline()) fh.close() - logger.info("Killing process " + instanceId + " " + str(firstLine)) + logger.info("Killing process %s %s", instanceId, firstLine) os.kill(firstLine, signal.SIGTERM) status_finish(200) else: # instance_id not found - logger.info(filepath + " not found") + logger.info("%s not found", filepath) status_finish(422) else: # instance_id not given, which means kill the container kill_parent() diff --git a/heron/shell/src/python/utils.py b/heron/shell/src/python/utils.py index bf9c1bdd5f2..2d0975d78e8 100644 --- a/heron/shell/src/python/utils.py +++ b/heron/shell/src/python/utils.py @@ -46,10 +46,9 @@ def stat_type(md): ''' stat type''' if stat.S_ISDIR(md): return 'd' - elif stat.S_ISSOCK(md): + if stat.S_ISSOCK(md): return 's' - else: - return '-' + return '-' def triple(md): ''' triple ''' diff --git a/heron/tools/admin/src/python/main.py b/heron/tools/admin/src/python/main.py index 3283eaacb07..79231801042 100644 --- a/heron/tools/admin/src/python/main.py +++ b/heron/tools/admin/src/python/main.py @@ -100,9 +100,8 @@ def run(handlers, command, parser, command_args, unknown_args): if command in handlers: return handlers[command].run(command, parser, command_args, unknown_args) - else: - err_context = 'Unknown subcommand: %s' % command - return result.SimpleResult(result.Status.InvocationError, err_context) + err_context = 'Unknown subcommand: %s' % command + return result.SimpleResult(result.Status.InvocationError, err_context) def cleanup(files): ''' diff --git a/heron/tools/admin/src/python/standalone.py b/heron/tools/admin/src/python/standalone.py index b7afdff7bf5..bfddfbfd358 100644 --- a/heron/tools/admin/src/python/standalone.py +++ b/heron/tools/admin/src/python/standalone.py @@ -41,7 +41,7 @@ # pylint: disable=unused-argument # pylint: disable=too-many-branches -class Action(object): +class Action: SET = "set" CLUSTER = "cluster" TEMPLATE = "template" @@ -50,17 +50,17 @@ class Action(object): TYPE = "type" -class Role(object): +class Role: ZOOKEEPERS = "zookeepers" MASTERS = "masters" SLAVES = "slaves" CLUSTER = "cluster" -class Cluster(object): +class Cluster: START = "start" STOP = "stop" -class Get(object): +class Get: SERVICE_URL = "service-url" HERON_TRACKER_URL = "heron-tracker-url" HERON_UI_URL = "heron-ui-url" @@ -581,8 +581,7 @@ def wait_for_job_to_start(single_master, job): r = requests.get("http://%s:4646/v1/job/%s" % (single_master, job)) if r.status_code == 200 and r.json()["Status"] == "running": break - else: - raise RuntimeError() + raise RuntimeError() except: Log.debug(sys.exc_info()[0]) Log.info("Waiting for %s to come up... %s" % (job, i)) @@ -860,12 +859,12 @@ def get_hostname(ip_addr, cl_args): def check_sure(cl_args, prompt): yes = input("%s" % prompt + ' (yes/no): ') - if yes == "y" or yes == "yes": + if yes in ("y", "yes"): return True - elif yes == "n" or yes == "no": + if yes in ("n", "no"): return False - else: - print('Invalid input. Please input "yes" or "no"') + print('Invalid input. Please input "yes" or "no"') + return None def get_jobs(cl_args, nomad_addr): r = requests.get("http://%s:4646/v1/jobs" % nomad_addr) diff --git a/heron/tools/cli/src/python/cli_helper.py b/heron/tools/cli/src/python/cli_helper.py index 51195fa53d6..da228aa1f10 100644 --- a/heron/tools/cli/src/python/cli_helper.py +++ b/heron/tools/cli/src/python/cli_helper.py @@ -152,5 +152,4 @@ def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]): def run(command, cl_args, action, extra_lib_jars=[]): if cl_args['deploy_mode'] == config.SERVER_MODE: return run_server(command, cl_args, action, extra_args=dict()) - else: - return run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=extra_lib_jars) + return run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=extra_lib_jars) diff --git a/heron/tools/cli/src/python/config.py b/heron/tools/cli/src/python/config.py index 870ef037d9f..137a96969e8 100644 --- a/heron/tools/cli/src/python/config.py +++ b/heron/tools/cli/src/python/config.py @@ -134,7 +134,6 @@ def run(command, parser, cl_args, unknown_args): configcommand = cl_args.get('configcommand', None) if configcommand == 'set': return _set(cl_args) - elif configcommand == 'unset': + if configcommand == 'unset': return _unset(cl_args) - else: - return _list(cl_args) + return _list(cl_args) diff --git a/heron/tools/cli/src/python/execute.py b/heron/tools/cli/src/python/execute.py index 99c4e28645d..4bca88e634b 100644 --- a/heron/tools/cli/src/python/execute.py +++ b/heron/tools/cli/src/python/execute.py @@ -121,6 +121,7 @@ def heron_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines): return heron_class(class_name, lib_jars, extra_jars, arguments, java_defines) def heron_pex(topology_pex, topology_class_name, args=None): + """Use a topology defined in a PEX.""" Log.debug("Importing %s from %s", topology_class_name, topology_pex) if topology_class_name == '-': # loading topology by running its main method (if __name__ == "__main__") @@ -134,26 +135,27 @@ def heron_pex(topology_pex, topology_class_name, args=None): # invoke the command with subprocess and print error message, if any process = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1) + # pylint: disable=fixme # todo(rli): improve python topology submission workflow return ProcessResult(process) - else: - try: - # loading topology from Topology's subclass (no main method) - # to support specifying the name of topology - Log.debug("args: %s", args) - if args is not None and isinstance(args, (list, tuple)) and len(args) > 0: - opts.set_config('cmdline.topology.name', args[0]) - os.environ["HERON_OPTIONS"] = opts.get_heron_config() - Log.debug("Heron options: {%s}", os.environ["HERON_OPTIONS"]) - pex_loader.load_pex(topology_pex) - topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name) - topology_class.write() - return SimpleResult(Status.Ok) - except Exception as ex: - Log.debug(traceback.format_exc()) - err_context = "Topology %s failed to be loaded from the given pex: %s" %\ - (topology_class_name, ex) - return SimpleResult(Status.HeronError, err_context) + try: + # loading topology from Topology's subclass (no main method) + # to support specifying the name of topology + Log.debug("args: %s", args) + if args is not None and isinstance(args, (list, tuple)) and len(args) > 0: + opts.set_config('cmdline.topology.name', args[0]) + os.environ["HERON_OPTIONS"] = opts.get_heron_config() + Log.debug("Heron options: {%s}", os.environ["HERON_OPTIONS"]) + pex_loader.load_pex(topology_pex) + topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name) + topology_class.write() + return SimpleResult(Status.Ok) + except Exception as ex: + Log.debug(traceback.format_exc()) + err_context = "Topology %s failed to be loaded from the given pex: %s" %\ + (topology_class_name, ex) + return SimpleResult(Status.HeronError, err_context) + return None # pylint: disable=superfluous-parens def heron_cpp(topology_binary, args=None): diff --git a/heron/tools/cli/src/python/help.py b/heron/tools/cli/src/python/help.py index a535de83ec7..6c9d35ad4dd 100644 --- a/heron/tools/cli/src/python/help.py +++ b/heron/tools/cli/src/python/help.py @@ -69,6 +69,5 @@ def run(command, parser, args, unknown_args): if subparser: print(subparser.format_help()) return SimpleResult(Status.Ok) - else: - Log.error("Unknown subcommand \'%s\'", command_help) - return SimpleResult(Status.InvocationError) + Log.error("Unknown subcommand \'%s\'", command_help) + return SimpleResult(Status.InvocationError) diff --git a/heron/tools/cli/src/python/main.py b/heron/tools/cli/src/python/main.py index 91f67c2823f..e6ae363f4d5 100644 --- a/heron/tools/cli/src/python/main.py +++ b/heron/tools/cli/src/python/main.py @@ -127,9 +127,8 @@ def run(handlers, command, parser, command_args, unknown_args): if command in handlers: return handlers[command].run(command, parser, command_args, unknown_args) - else: - err_context = 'Unknown subcommand: %s' % command - return result.SimpleResult(result.Status.InvocationError, err_context) + err_context = 'Unknown subcommand: %s' % command + return result.SimpleResult(result.Status.InvocationError, err_context) def cleanup(files): ''' diff --git a/heron/tools/cli/src/python/restart.py b/heron/tools/cli/src/python/restart.py index ab8802ef007..06f42d37a14 100644 --- a/heron/tools/cli/src/python/restart.py +++ b/heron/tools/cli/src/python/restart.py @@ -73,6 +73,5 @@ def run(command, parser, cl_args, unknown_args): if cl_args['deploy_mode'] == config.SERVER_MODE: dict_extra_args = {"container_id": str(container_id)} return cli_helper.run_server(command, cl_args, message, extra_args=dict_extra_args) - else: - list_extra_args = ["--container_id", str(container_id)] - return cli_helper.run_direct(command, cl_args, message, extra_args=list_extra_args) + list_extra_args = ["--container_id", str(container_id)] + return cli_helper.run_direct(command, cl_args, message, extra_args=list_extra_args) diff --git a/heron/tools/cli/src/python/result.py b/heron/tools/cli/src/python/result.py index c8e47ff02c9..df652934867 100644 --- a/heron/tools/cli/src/python/result.py +++ b/heron/tools/cli/src/python/result.py @@ -52,14 +52,13 @@ class Status(Enum): def status_type(status_code): if status_code == 0: return Status.Ok - elif status_code < 100: + if status_code < 100: return Status.InvocationError - elif status_code == 200: + if status_code == 200: return Status.DryRun - else: - return Status.HeronError + return Status.HeronError -class Result(object): +class Result: """Result class""" def __init__(self, status=None, err_context=None, succ_context=None): self.status = status @@ -89,7 +88,7 @@ def _log_context(self): self._do_log(Log.error, self.err_context) else: raise RuntimeError( - "Unknown status type of value %d. Expected value: %s", self.status.value, list(Status)) + "Unknown status type of value %d. Expected value: %s" % (self.status.value, list(Status))) def add_context(self, err_context, succ_context=None): """ Prepend msg to add some context information @@ -181,13 +180,12 @@ def render(results): for r in results: r.render() else: - raise RuntimeError("Unknown result instance: %s", str(results.__class__)) + raise RuntimeError("Unknown result instance: %s" % (str(results.__class__),)) # check if all results are successful def is_successful(results): if isinstance(results, list): return all([is_successful(result) for result in results]) - elif isinstance(results, Result): + if isinstance(results, Result): return results.status == Status.Ok or results.status == Status.DryRun - else: - raise RuntimeError("Unknown result instance: %s", str(results.__class__)) + raise RuntimeError("Unknown result instance: %s" % (str(results.__class__),)) diff --git a/heron/tools/cli/src/python/submit.py b/heron/tools/cli/src/python/submit.py index 83e9e7f1c56..5e6be8d8643 100644 --- a/heron/tools/cli/src/python/submit.py +++ b/heron/tools/cli/src/python/submit.py @@ -19,6 +19,7 @@ # under the License. ''' submit.py ''' +# pylint: disable=wrong-import-order from future.standard_library import install_aliases install_aliases() @@ -399,6 +400,7 @@ def download(uri, cluster): for f in os.listdir(tmp_dir): if f.endswith(suffix): return os.path.join(tmp_dir, f) + return None ################################################################################ # pylint: disable=unused-argument @@ -478,9 +480,8 @@ def run(command, parser, cl_args, unknown_args): # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) - elif tar_type: + if tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) - elif cpp_type: + if cpp_type: return submit_cpp(cl_args, unknown_args, tmp_dir) - else: - return submit_pex(cl_args, unknown_args, tmp_dir) + return submit_pex(cl_args, unknown_args, tmp_dir) diff --git a/heron/tools/cli/src/python/update.py b/heron/tools/cli/src/python/update.py index cc3e05815ae..8dddb18f11d 100644 --- a/heron/tools/cli/src/python/update.py +++ b/heron/tools/cli/src/python/update.py @@ -179,7 +179,6 @@ def run(command, parser, cl_args, unknown_args): # Execute if cl_args['deploy_mode'] == config.SERVER_MODE: return cli_helper.run_server(command, cl_args, action, dict_extra_args) - else: - # Convert extra argument to commandline format and then execute - list_extra_args = convert_args_dict_to_list(dict_extra_args) - return cli_helper.run_direct(command, cl_args, action, list_extra_args, extra_lib_jars) + # Convert extra argument to commandline format and then execute + list_extra_args = convert_args_dict_to_list(dict_extra_args) + return cli_helper.run_direct(command, cl_args, action, list_extra_args, extra_lib_jars) diff --git a/heron/tools/common/src/python/access/query.py b/heron/tools/common/src/python/access/query.py index 11aa2e88bae..ba874bcbdc8 100644 --- a/heron/tools/common/src/python/access/query.py +++ b/heron/tools/common/src/python/access/query.py @@ -20,10 +20,10 @@ ''' query.py ''' -class QueryHandler(object): +class QueryHandler: ''' QueryHandler ''' - def fetch(self, cluster, metric, topology, component, instance, timerange, envirn=None): + def fetch(self, cluster, metric, topology, component, instance, timerange, environ=None): ''' :param cluster: :param metric: @@ -31,12 +31,11 @@ def fetch(self, cluster, metric, topology, component, instance, timerange, envir :param component: :param instance: :param timerange: - :param envirn: + :param environ: :return: ''' - pass - def fetch_max(self, cluster, metric, topology, component, instance, timerange, envirn=None): + def fetch_max(self, cluster, metric, topology, component, instance, timerange, environ=None): ''' :param cluster: :param metric: @@ -44,10 +43,9 @@ def fetch_max(self, cluster, metric, topology, component, instance, timerange, e :param component: :param instance: :param timerange: - :param envirn: + :param environ: :return: ''' - pass def fetch_backpressure(self, cluster, metric, topology, component, instance, \ timerange, is_max, environ=None): @@ -62,4 +60,3 @@ def fetch_backpressure(self, cluster, metric, topology, component, instance, \ :param environ: :return: ''' - pass diff --git a/heron/tools/common/src/python/utils/classpath.py b/heron/tools/common/src/python/utils/classpath.py index 2fc734d2916..712ae1fff32 100644 --- a/heron/tools/common/src/python/utils/classpath.py +++ b/heron/tools/common/src/python/utils/classpath.py @@ -39,11 +39,10 @@ def valid_path(path): Log.debug('Checking classpath entry as directory: %s', path) if os.path.isdir(path): return True - else: - # check if the classpath entry is a file - Log.debug('Checking classpath entry as file: %s', path) - if os.path.isfile(path): - return True + # check if the classpath entry is a file + Log.debug('Checking classpath entry as file: %s', path) + if os.path.isfile(path): + return True return False diff --git a/heron/tools/common/src/python/utils/config.py b/heron/tools/common/src/python/utils/config.py index 1d5198a3d53..64534dbd138 100644 --- a/heron/tools/common/src/python/utils/config.py +++ b/heron/tools/common/src/python/utils/config.py @@ -289,16 +289,14 @@ def parse_cluster_role_env(cluster_role_env, config_path): if (ROLE_REQUIRED in cli_confs) and (cli_confs[ROLE_REQUIRED] is True): raise Exception("role required but not provided (cluster/role/env = %s). See %s in %s" % (cluster_role_env, ROLE_REQUIRED, cli_conf_file)) - else: - parts.append(getpass.getuser()) + parts.append(getpass.getuser()) # if environ is required but not provided, raise exception if len(parts) == 2: if (ENV_REQUIRED in cli_confs) and (cli_confs[ENV_REQUIRED] is True): raise Exception("environ required but not provided (cluster/role/env = %s). See %s in %s" % (cluster_role_env, ENV_REQUIRED, cli_conf_file)) - else: - parts.append(ENVIRON) + parts.append(ENVIRON) # if cluster or role or environ is empty, print if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0: @@ -342,12 +340,14 @@ def direct_mode_cluster_role_env(cluster_role_env, config_path): return True # if role is required but not provided, raise exception - role_present = True if len(cluster_role_env[1]) > 0 else False + role_present = bool(cluster_role_env[1]) + # pylint: disable=simplifiable-if-expression if ROLE_REQUIRED in client_confs and client_confs[ROLE_REQUIRED] and not role_present: raise Exception("role required but not provided (cluster/role/env = %s). See %s in %s" % (cluster_role_env, ROLE_REQUIRED, cli_conf_file)) # if environ is required but not provided, raise exception + # pylint: disable=simplifiable-if-expression environ_present = True if len(cluster_role_env[2]) > 0 else False if ENV_REQUIRED in client_confs and client_confs[ENV_REQUIRED] and not environ_present: raise Exception("environ required but not provided (cluster/role/env = %s). See %s in %s" @@ -362,13 +362,15 @@ def server_mode_cluster_role_env(cluster_role_env, config_map): cmap = config_map[cluster_role_env[0]] # if role is required but not provided, raise exception - role_present = True if len(cluster_role_env[1]) > 0 else False + role_present = bool(cluster_role_env[1]) + # pylint: disable=simplifiable-if-expression if ROLE_KEY in cmap and cmap[ROLE_KEY] and not role_present: raise Exception("role required but not provided (cluster/role/env = %s)."\ % (cluster_role_env)) # if environ is required but not provided, raise exception environ_present = True if len(cluster_role_env[2]) > 0 else False + # pylint: disable=simplifiable-if-expression if ENVIRON_KEY in cmap and cmap[ENVIRON_KEY] and not environ_present: raise Exception("environ required but not provided (cluster/role/env = %s)."\ % (cluster_role_env)) diff --git a/heron/tools/common/src/python/utils/heronparser.py b/heron/tools/common/src/python/utils/heronparser.py index 29cb7a8018a..cfc05af75ba 100755 --- a/heron/tools/common/src/python/utils/heronparser.py +++ b/heron/tools/common/src/python/utils/heronparser.py @@ -78,8 +78,8 @@ def _replacer(match): # it means we have captured a non-quoted (real) comment string. if match.group(1) is not None: return "" # so we will return empty to remove the comment - else: # otherwise, we will return the 1st group - return match.group(1) # captured quoted-string + # otherwise, we will return the 1st group + return match.group(1) # captured quoted-string return regex.sub(_replacer, string) @classmethod @@ -101,6 +101,7 @@ def clear(cls): @classmethod def initializeFromRC(cls, rcfile): + """Initialise.""" if len(cls.cmdmap) > 0: return effective_rc = (rcfile, HERON_RC)[rcfile is None] diff --git a/heron/tools/explorer/src/python/help.py b/heron/tools/explorer/src/python/help.py index 6eaf116415a..6995849cc2f 100644 --- a/heron/tools/explorer/src/python/help.py +++ b/heron/tools/explorer/src/python/help.py @@ -60,6 +60,5 @@ def run(command, parser, args, unknown_args): if subparser: print(subparser.format_help()) return True - else: - Log.error("Unknown subcommand \'%s\'" % command_help) - return False + Log.error("Unknown subcommand \'%s\'" % command_help) + return False diff --git a/heron/tools/explorer/src/python/main.py b/heron/tools/explorer/src/python/main.py index 49006fde069..6aea942c294 100644 --- a/heron/tools/explorer/src/python/main.py +++ b/heron/tools/explorer/src/python/main.py @@ -100,29 +100,29 @@ def run(command, *args): return clusters.run(command, *args) # show topologies - elif command == 'topologies': + if command == 'topologies': return topologies.run(command, *args) # physical plan - elif command == 'containers': + if command == 'containers': return physicalplan.run_containers(command, *args) - elif command == 'metrics': + if command == 'metrics': return physicalplan.run_metrics(command, *args) # logical plan - elif command == 'components': + if command == 'components': return logicalplan.run_components(command, *args) - elif command == 'spouts': + if command == 'spouts': return logicalplan.run_spouts(command, *args) - elif command == 'bolts': + if command == 'bolts': return logicalplan.run_bolts(command, *args) # help - elif command == 'help': + if command == 'help': return help.run(command, *args) # version - elif command == 'version': + if command == 'version': return version.run(command, *args) return 1 diff --git a/heron/tools/explorer/src/python/topologies.py b/heron/tools/explorer/src/python/topologies.py index 137f29086d9..ecb0dfcd698 100644 --- a/heron/tools/explorer/src/python/topologies.py +++ b/heron/tools/explorer/src/python/topologies.py @@ -50,8 +50,7 @@ def to_table(result): count += 1 if count > max_count: continue - else: - table.append([role, env, topo]) + table.append([role, env, topo]) header = ['role', 'env', 'topology'] rest_count = 0 if count <= max_count else count - max_count return table, header, rest_count @@ -123,10 +122,9 @@ def run(command, parser, cl_args, unknown_args): location = cl_args['cluster/[role]/[env]'].split('/') if len(location) == 1: return show_cluster(cl_args, *location) - elif len(location) == 2: + if len(location) == 2: return show_cluster_role(cl_args, *location) - elif len(location) == 3: + if len(location) == 3: return show_cluster_role_env(cl_args, *location) - else: - Log.error('Invalid topologies selection') - return False + Log.error('Invalid topologies selection') + return False diff --git a/heron/tools/tracker/src/python/config.py b/heron/tools/tracker/src/python/config.py index f19182bf975..b449cec47a0 100644 --- a/heron/tools/tracker/src/python/config.py +++ b/heron/tools/tracker/src/python/config.py @@ -28,7 +28,7 @@ EXTRA_LINK_FORMATTER_KEY = "formatter" EXTRA_LINK_URL_KEY = "url" -class Config(object): +class Config: """ Responsible for reading the yaml config file and exposing various tracker configs. diff --git a/heron/tools/tracker/src/python/graph.py b/heron/tools/tracker/src/python/graph.py index be94778ecca..b73f0849a8d 100644 --- a/heron/tools/tracker/src/python/graph.py +++ b/heron/tools/tracker/src/python/graph.py @@ -22,7 +22,7 @@ ################################################################################ -class Graph(object): +class Graph: ''' Adjacency list of edges in graph. This will correspond to the streams in a topology DAG. ''' diff --git a/heron/tools/tracker/src/python/handlers/containerfilehandler.py b/heron/tools/tracker/src/python/handlers/containerfilehandler.py index 3132567640d..749d0d5fd15 100644 --- a/heron/tools/tracker/src/python/handlers/containerfilehandler.py +++ b/heron/tools/tracker/src/python/handlers/containerfilehandler.py @@ -103,6 +103,7 @@ def initialize(self, tracker): @tornado.gen.coroutine def get(self): + """Serve a GET request.""" try: cluster = self.get_argument_cluster() role = self.get_argument_role() diff --git a/heron/tools/tracker/src/python/javaobj.py b/heron/tools/tracker/src/python/javaobj.py index ed962a01b7e..1587d05d0a2 100644 --- a/heron/tools/tracker/src/python/javaobj.py +++ b/heron/tools/tracker/src/python/javaobj.py @@ -78,7 +78,7 @@ def dumps(obj): "java.lang.Integer", "java.lang.Long"]) -class JavaClass(object): +class JavaClass: """Java class representation""" def __init__(self): self.name = None @@ -95,7 +95,7 @@ def __repr__(self): return "[%s:0x%X]" % (self.name, self.serialVersionUID) -class JavaObject(object): +class JavaObject: """Java object representation""" def __init__(self): self.classdesc = None @@ -132,7 +132,7 @@ def copy(self, new_object): for name in self.classdesc.fields_names: new_object.__setattr__(name, getattr(self, name)) -class JavaObjectConstants(object): +class JavaObjectConstants: """class about Java object constants""" STREAM_MAGIC = 0xaced @@ -222,7 +222,7 @@ def readObject(self): position_bak = self.object_stream.tell() the_rest = self.object_stream.read() - if len(the_rest): + if the_rest: log_error("Warning!!!!: Stream still has %s bytes left.\ Enable debug mode of logging to see the hexdump." % len(the_rest)) log_debug(self._create_hexdump(the_rest)) @@ -457,7 +457,7 @@ def do_array(self, parent=None, ident=0): assert type_char == self.TYPE_ARRAY type_char = classdesc.name[1] - if type_char == self.TYPE_OBJECT or type_char == self.TYPE_ARRAY: + if type_char in (self.TYPE_OBJECT, self.TYPE_ARRAY): for _ in range(size): _, res = self._read_and_exec_opcode(ident=ident+1) log_debug("Object value: %s" % str(res), ident) @@ -519,7 +519,7 @@ def _read_value(self, field_type, ident, name=""): (res, ) = self._readStruct(">f") elif field_type == self.TYPE_DOUBLE: (res, ) = self._readStruct(">d") - elif field_type == self.TYPE_OBJECT or field_type == self.TYPE_ARRAY: + elif field_type in (self.TYPE_OBJECT, self.TYPE_ARRAY): _, res = self._read_and_exec_opcode(ident=ident+1) else: raise RuntimeError("Unknown typecode: %s" % field_type) @@ -533,8 +533,7 @@ def _convert_char_to_type(self, type_char): if typecode in self.TYPECODES_LIST: return typecode - else: - raise RuntimeError("Typecode %s (%s) isn't supported." % (type_char, typecode)) + raise RuntimeError("Typecode %s (%s) isn't supported." % (type_char, typecode)) def _add_reference(self, obj): self.references.append(obj) @@ -545,7 +544,7 @@ def _oops_dump_state(self): log_error("Stream seeking back at -16 byte (2nd line is an actual position!):") self.object_stream.seek(-16, mode=1) the_rest = self.object_stream.read() - if len(the_rest): + if the_rest: log_error("Warning!!!!: Stream still has %s bytes left." % len(the_rest)) log_error(self._create_hexdump(the_rest)) log_error("=" * 30) @@ -597,7 +596,7 @@ def write_object(self, obj, parent=None): self._writeStruct(">B", 1, (self.TC_OBJECT, )) self._writeStruct(">B", 1, (self.TC_CLASSDESC, )) -class DefaultObjectTransformer(object): +class DefaultObjectTransformer: class JavaList(list, JavaObject): pass @@ -619,7 +618,6 @@ def transform(self, obj): obj.copy(new_object) new_object.extend(obj.annotations[1:]) return new_object - # pylint: disable=redefined-variable-type if obj.get_class().name == "java.util.HashMap": new_object = self.JavaMap() obj.copy(new_object) diff --git a/heron/tools/tracker/src/python/pyutils.py b/heron/tools/tracker/src/python/pyutils.py index 71af4a90c3e..c7cb2777e05 100644 --- a/heron/tools/tracker/src/python/pyutils.py +++ b/heron/tools/tracker/src/python/pyutils.py @@ -27,5 +27,4 @@ def is_str_instance(obj): if isPY3: return isinstance(obj, str) - else: - return str(type(obj)) == "" or str(type(obj)) == "" + return str(type(obj)) == "" or str(type(obj)) == "" diff --git a/heron/tools/tracker/src/python/query.py b/heron/tools/tracker/src/python/query.py index ceaa23ad5b1..5d89a525284 100644 --- a/heron/tools/tracker/src/python/query.py +++ b/heron/tools/tracker/src/python/query.py @@ -30,7 +30,7 @@ #################################################################### # pylint: disable=no-self-use -class Query(object): +class Query: """Execute the query for metrics. Uses Tracker to get individual metrics that are part of the query. Example usage: @@ -109,8 +109,7 @@ def parse_query_string(self, query): # This must be the last index, since this was an NOP starting brace if index != len(query) - 1: raise Exception("Invalid syntax") - else: - return self.parse_query_string(query[1:-1]) + return self.parse_query_string(query[1:-1]) start_index = query.find("(") # There must be a ( in the query if start_index < 0: diff --git a/heron/tools/tracker/src/python/query_operators.py b/heron/tools/tracker/src/python/query_operators.py index 216f1f5a1fa..8102b3b54b7 100644 --- a/heron/tools/tracker/src/python/query_operators.py +++ b/heron/tools/tracker/src/python/query_operators.py @@ -32,13 +32,12 @@ def is_str_instance(obj): if isPY3: return isinstance(obj, str) - else: - return str(type(obj)) == "" or str(type(obj)) == "" + return str(type(obj)) == "" or str(type(obj)) == "" ##################################################################### # Data Structure for fetched Metrics ##################################################################### -class Metrics(object): +class Metrics: """Represents a univariate timeseries. Multivariate timeseries is simply a list of this.""" def __init__(self, componentName, metricName, instance, start, end, timeline): @@ -81,7 +80,7 @@ def setDefault(self, constant, start, end): ################################################################ # pylint: disable=no-self-use -class Operator(object): +class Operator: """Base class for all operators""" def __init__(self, _): raise Exception("Not implemented exception") @@ -438,7 +437,7 @@ def execute(self, tracker, tmaster, start, end): allMetrics.append(met) raise tornado.gen.Return(allMetrics) # If first is univariate - elif len(metrics) == 1 and "" in metrics: + if len(metrics) == 1 and "" in metrics: allMetrics = [] for key, metric in list(metrics2.items()): # Initialize with first metrics timeline, but second metric's instance @@ -452,19 +451,17 @@ def execute(self, tracker, tmaster, start, end): allMetrics.append(met) raise tornado.gen.Return(allMetrics) # If second is univariate - else: - allMetrics = [] - for key, metric in list(metrics.items()): - # Initialize with first metrics timeline and its instance - met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline)) - for timestamp in list(met.timeline.keys()): - if timestamp not in metrics2[""].timeline or metrics2[""].timeline[timestamp] == 0: - met.timeline.pop(timestamp) - else: - met.timeline[timestamp] /= metrics2[""].timeline[timestamp] - allMetrics.append(met) - raise tornado.gen.Return(allMetrics) - raise Exception("This should not be generated.") + allMetrics = [] + for key, metric in list(metrics.items()): + # Initialize with first metrics timeline and its instance + met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline)) + for timestamp in list(met.timeline.keys()): + if timestamp not in metrics2[""].timeline or metrics2[""].timeline[timestamp] == 0: + met.timeline.pop(timestamp) + else: + met.timeline[timestamp] /= metrics2[""].timeline[timestamp] + allMetrics.append(met) + raise tornado.gen.Return(allMetrics) class Multiply(Operator): """Multiply Operator. Has same conditions as division operator. @@ -563,7 +560,7 @@ def execute(self, tracker, tmaster, start, end): allMetrics.append(met) raise tornado.gen.Return(allMetrics) # If first is univariate - elif len(metrics) == 1 and "" in metrics: + if len(metrics) == 1 and "" in metrics: allMetrics = [] for key, metric in list(metrics2.items()): # Initialize with first metrics timeline, but second metric's instance @@ -577,19 +574,17 @@ def execute(self, tracker, tmaster, start, end): allMetrics.append(met) raise tornado.gen.Return(allMetrics) # If second is univariate - else: - allMetrics = [] - for key, metric in list(metrics.items()): - # Initialize with first metrics timeline and its instance - met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline)) - for timestamp in list(met.timeline.keys()): - if timestamp not in metrics2[""].timeline: - met.timeline.pop(timestamp) - else: - met.timeline[timestamp] *= metrics2[""].timeline[timestamp] - allMetrics.append(met) - raise tornado.gen.Return(allMetrics) - raise Exception("This should not be generated.") + allMetrics = [] + for key, metric in list(metrics.items()): + # Initialize with first metrics timeline and its instance + met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline)) + for timestamp in list(met.timeline.keys()): + if timestamp not in metrics2[""].timeline: + met.timeline.pop(timestamp) + else: + met.timeline[timestamp] *= metrics2[""].timeline[timestamp] + allMetrics.append(met) + raise tornado.gen.Return(allMetrics) class Subtract(Operator): """Subtract Operator. Has same conditions as division operator. @@ -686,7 +681,7 @@ def execute(self, tracker, tmaster, start, end): allMetrics.append(met) raise tornado.gen.Return(allMetrics) # If first is univariate - elif len(metrics) == 1 and "" in metrics: + if len(metrics) == 1 and "" in metrics: allMetrics = [] for key, metric in list(metrics2.items()): # Initialize with first metrics timeline, but second metric's instance @@ -700,19 +695,17 @@ def execute(self, tracker, tmaster, start, end): allMetrics.append(met) raise tornado.gen.Return(allMetrics) # If second is univariate - else: - allMetrics = [] - for key, metric in list(metrics.items()): - # Initialize with first metrics timeline and its instance - met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline)) - for timestamp in list(met.timeline.keys()): - if timestamp not in metrics2[""].timeline: - met.timeline.pop(timestamp) - else: - met.timeline[timestamp] -= metrics2[""].timeline[timestamp] - allMetrics.append(met) - raise tornado.gen.Return(allMetrics) - raise Exception("This should not be generated.") + allMetrics = [] + for key, metric in list(metrics.items()): + # Initialize with first metrics timeline and its instance + met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline)) + for timestamp in list(met.timeline.keys()): + if timestamp not in metrics2[""].timeline: + met.timeline.pop(timestamp) + else: + met.timeline[timestamp] -= metrics2[""].timeline[timestamp] + allMetrics.append(met) + raise tornado.gen.Return(allMetrics) class Rate(Operator): """Rate Operator. This operator is used to find rate of change for all timeseries. diff --git a/heron/tools/tracker/src/python/topology.py b/heron/tools/tracker/src/python/topology.py index 798b43d5003..ab513746285 100644 --- a/heron/tools/tracker/src/python/topology.py +++ b/heron/tools/tracker/src/python/topology.py @@ -27,7 +27,7 @@ from heronpy.api import api_constants # pylint: disable=too-many-instance-attributes -class Topology(object): +class Topology: """ Class Topology Contains all the relevant information about @@ -242,9 +242,8 @@ def get_status(self): if status == 1: return "Running" - elif status == 2: + if status == 2: return "Paused" - elif status == 3: + if status == 3: return "Killed" - else: - return "Unknown" + return "Unknown" diff --git a/heron/tools/tracker/src/python/tracker.py b/heron/tools/tracker/src/python/tracker.py index 5cdbea40818..81cc06b56c4 100644 --- a/heron/tools/tracker/src/python/tracker.py +++ b/heron/tools/tracker/src/python/tracker.py @@ -81,8 +81,7 @@ def _convert_java_value(kv, include_non_primitives=True): Log.exception("Failed to parse data as java object") if include_non_primitives: return _raw_value(kv) - else: - return None + return None def _raw_value(kv): return { @@ -91,7 +90,7 @@ def _raw_value(kv): 'raw' : utils.hex_escape(kv.serialized_value)} -class Tracker(object): +class Tracker: """ Tracker is a stateless cache of all the topologies for the given state managers. It watches for @@ -129,7 +128,6 @@ def synch_topologies(self): traceback.print_exc() sys.exit(1) - # pylint: disable=deprecated-lambda def on_topologies_watch(state_manager, topologies): """watch topologies""" Log.info("State watch triggered for topologies.") @@ -157,7 +155,6 @@ def stop_sync(self): for state_manager in self.state_managers: state_manager.stop() - # pylint: disable=deprecated-lambda def getTopologyByClusterRoleEnvironAndName(self, cluster, role, environ, topologyName): """ Find and return the topology given its cluster, environ, topology name, and @@ -172,9 +169,8 @@ def getTopologyByClusterRoleEnvironAndName(self, cluster, role, environ, topolog if role is not None: raise Exception("Topology not found for {0}, {1}, {2}, {3}".format( cluster, role, environ, topologyName)) - else: - raise Exception("Topology not found for {0}, {1}, {2}".format( - cluster, environ, topologyName)) + raise Exception("Topology not found for {0}, {1}, {2}".format( + cluster, environ, topologyName)) # There is only one topology which is returned. return topologies[0] @@ -316,14 +312,10 @@ def extract_metadata(self, topology): @staticmethod def extract_runtime_state(topology): runtime_state = {} - runtime_state["has_physical_plan"] = \ - True if topology.physical_plan else False - runtime_state["has_packing_plan"] = \ - True if topology.packing_plan else False - runtime_state["has_tmaster_location"] = \ - True if topology.tmaster else False - runtime_state["has_scheduler_location"] = \ - True if topology.scheduler_location else False + runtime_state["has_physical_plan"] = bool(topology.physical_plan) + runtime_state["has_packing_plan"] = bool(topology.packing_plan) + runtime_state["has_tmaster_location"] = bool(topology.tmaster) + runtime_state["has_scheduler_location"] = bool(topology.scheduler_location) # "stmgrs" listed runtime state for each stream manager # however it is possible that physical plan is not complete # yet and we do not know how many stmgrs there are. That said, diff --git a/heron/tools/tracker/src/python/utils.py b/heron/tools/tracker/src/python/utils.py index 1b61232b671..a6f912975f2 100644 --- a/heron/tools/tracker/src/python/utils.py +++ b/heron/tools/tracker/src/python/utils.py @@ -76,8 +76,7 @@ def make_shell_logfiles_url(host, shell_port, _, instance_id=None): return None if not instance_id: return "http://%s:%d/browse/log-files" % (host, shell_port) - else: - return "http://%s:%d/file/log-files/%s.log.0" % (host, shell_port, instance_id) + return "http://%s:%d/file/log-files/%s.log.0" % (host, shell_port, instance_id) def make_shell_logfile_data_url(host, shell_port, instance_id, offset, length): """ diff --git a/heron/tools/tracker/tests/python/mock_proto.py b/heron/tools/tracker/tests/python/mock_proto.py index b92662bb009..82cd4def0bb 100644 --- a/heron/tools/tracker/tests/python/mock_proto.py +++ b/heron/tools/tracker/tests/python/mock_proto.py @@ -23,7 +23,7 @@ import heron.proto.topology_pb2 as protoTopology # pylint: disable=no-self-use, missing-docstring -class MockProto(object): +class MockProto: ''' Mocking Proto''' topology_name = "mock_topology_name" topology_id = "mock_topology_id" diff --git a/heron/tools/ui/src/python/handlers/api/topology.py b/heron/tools/ui/src/python/handlers/api/topology.py index 6dd20dbcf53..c519bc5a903 100644 --- a/heron/tools/ui/src/python/handlers/api/topology.py +++ b/heron/tools/ui/src/python/handlers/api/topology.py @@ -58,13 +58,13 @@ def get(self, cluster, environ, topology, comp_name): else: comp_names = [comp_name] exception_infos = dict() - for comp_name in comp_names: - exception_infos[comp_name] = yield access.get_component_exceptionsummary( - cluster, environ, topology, comp_name) + for comp_name_ in comp_names: + exception_infos[comp_name_] = yield access.get_component_exceptionsummary( + cluster, environ, topology, comp_name_) # Combine exceptions from multiple component aggregate_exceptions = dict() - for comp_name, exception_logs in list(exception_infos.items()): + for comp_name_, exception_logs in list(exception_infos.items()): for exception_log in exception_logs: class_name = exception_log['class_name'] if class_name != '': diff --git a/heronpy/api/api_constants.py b/heronpy/api/api_constants.py index d4cbf653f67..0d16717675e 100644 --- a/heronpy/api/api_constants.py +++ b/heronpy/api/api_constants.py @@ -24,7 +24,7 @@ ########################### Constants for topology configuration ################################## #################################################################################################### -class TopologyReliabilityMode(object): +class TopologyReliabilityMode: ATMOST_ONCE = "ATMOST_ONCE" ATLEAST_ONCE = "ATLEAST_ONCE" EFFECTIVELY_ONCE = "EFFECTIVELY_ONCE" diff --git a/heronpy/api/bolt/bolt.py b/heronpy/api/bolt/bolt.py index 89bef6ab017..0f522edeb3c 100644 --- a/heronpy/api/bolt/bolt.py +++ b/heronpy/api/bolt/bolt.py @@ -42,7 +42,6 @@ def initialize(self, config, context): topology, including the task id and component id of this task, input and output information, etc. """ - pass @abstractmethod def process(self, tup): @@ -77,4 +76,3 @@ def process_tick(self, tup): :type tup: :class:`Tuple` :param tup: the tick tuple to be processed """ - pass diff --git a/heronpy/api/bolt/window_bolt.py b/heronpy/api/bolt/window_bolt.py index 892994c2f19..6e29eaba6a5 100644 --- a/heronpy/api/bolt/window_bolt.py +++ b/heronpy/api/bolt/window_bolt.py @@ -59,7 +59,6 @@ def processWindow(self, window_info, tuples): :type tuples: :class:`list of Tuples` :param tuples: The list of tuples in this window """ - pass # pylint: disable=unused-argument def initialize(self, config, context): @@ -109,8 +108,8 @@ def process_tick(self, tup): curtime = int(time.time()) window_info = WindowContext(curtime - self.window_duration, curtime) tuple_batch = [] - for (tup, tm) in self.current_tuples: - tuple_batch.append(tup) + for (tuple_, tm) in self.current_tuples: + tuple_batch.append(tuple_) self.processWindow(window_info, tuple_batch) self._expire(curtime) @@ -145,7 +144,6 @@ def processWindow(self, window_info, tuples): :type tuples: :class:`list of Tuples` :param tuples: The list of tuples in this window """ - pass # pylint: disable=unused-argument def initialize(self, config, context): @@ -178,6 +176,6 @@ def process_tick(self, tup): curtime = int(time.time()) window_info = WindowContext(curtime - self.window_duration, curtime) self.processWindow(window_info, list(self.current_tuples)) - for tup in self.current_tuples: - self.ack(tup) + for tuple_ in self.current_tuples: + self.ack(tuple_) self.current_tuples.clear() diff --git a/heronpy/api/cloudpickle.py b/heronpy/api/cloudpickle.py index eb3bfefb9d5..88e94c5ab22 100644 --- a/heronpy/api/cloudpickle.py +++ b/heronpy/api/cloudpickle.py @@ -511,7 +511,7 @@ def __getitem__(self, item): def save_attrgetter(self, obj): """attrgetter serializer""" - class Dummy(object): + class Dummy: def __init__(self, attrs, index=None): self.attrs = attrs self.index = index @@ -795,7 +795,7 @@ def _load_class(cls, d): if typ == 'property': v = property(*v) elif typ == 'staticmethod': - v = staticmethod(v) # pylint: disable=redefined-variable-type + v = staticmethod(v) elif typ == 'classmethod': v = classmethod(v) setattr(cls, k, v) diff --git a/heronpy/api/component/base_component.py b/heronpy/api/component/base_component.py index 47810d81d76..48a14b9c953 100644 --- a/heronpy/api/component/base_component.py +++ b/heronpy/api/component/base_component.py @@ -13,7 +13,7 @@ # limitations under the License. '''base_component.py''' -class BaseComponent(object): +class BaseComponent: """Base component for Heron spout and bolt""" def __init__(self, delegate): """Initializes BaseComponent diff --git a/heronpy/api/component/component_spec.py b/heronpy/api/component/component_spec.py index dc00aadadc3..29269ada78a 100644 --- a/heronpy/api/component/component_spec.py +++ b/heronpy/api/component/component_spec.py @@ -28,7 +28,7 @@ from heronpy.api.stream import Stream, Grouping # pylint: disable=too-many-instance-attributes -class HeronComponentSpec(object): +class HeronComponentSpec: """Class to specify the information and location of components in a topology This class is generated by the ``spec()`` method of Spout and Bolt class and @@ -67,8 +67,7 @@ def get_protobuf(self): """Returns protobuf message (Spout or Bolt) of this component""" if self.is_spout: return self._get_spout() - else: - return self._get_bolt() + return self._get_bolt() def _get_spout(self): """Returns Spout protobuf message""" @@ -191,7 +190,7 @@ def _sanitize_inputs(self): """Sanitizes input fields and returns a map Grouping>""" ret = {} if self.inputs is None: - return + return None if isinstance(self.inputs, dict): # inputs are dictionary, must be either Grouping> or @@ -248,7 +247,7 @@ def _sanitize_outputs(self): """Sanitizes output fields and returns a map list of output fields>""" ret = {} if self.outputs is None: - return + return None if not isinstance(self.outputs, (list, tuple)): raise TypeError("Argument to outputs must be either list or tuple, given: %s" @@ -314,7 +313,7 @@ def _get_stream_schema(fields): return stream_schema -class GlobalStreamId(object): +class GlobalStreamId: """Wrapper class to define stream_id and its component name Constructor method is compatible with StreamParse's GlobalStreamId class, although @@ -358,11 +357,10 @@ def component_id(self): # appropriate this case. return "" % self._component_id.uuid return self._component_id.name - elif isinstance(self._component_id, str): + if isinstance(self._component_id, str): return self._component_id - else: - raise ValueError("Component Id for this GlobalStreamId is not properly set: <%s:%s>" - % (str(type(self._component_id)), str(self._component_id))) + raise ValueError("Component Id for this GlobalStreamId is not properly set: <%s:%s>" + % (str(type(self._component_id)), str(self._component_id))) def __eq__(self, other): return hasattr(other, 'component_id') and self.component_id == other.component_id \ diff --git a/heronpy/api/custom_grouping.py b/heronpy/api/custom_grouping.py index 4654c90d209..2d21ed95993 100644 --- a/heronpy/api/custom_grouping.py +++ b/heronpy/api/custom_grouping.py @@ -21,7 +21,7 @@ '''custom_grouping.py: interface module for custom grouping''' from abc import abstractmethod -class ICustomGrouping(object): +class ICustomGrouping: '''Interface for custom grouping class''' @abstractmethod @@ -36,7 +36,6 @@ def prepare(self, context, component, stream, target_tasks): :type target_tasks: list of int :param target_tasks: list of target task ids """ - pass @abstractmethod def choose_tasks(self, values): @@ -46,4 +45,3 @@ def choose_tasks(self, values): :rtype: list of int :return: list of task ids to which these values are emitted """ - pass diff --git a/heronpy/api/metrics.py b/heronpy/api/metrics.py index 848d972b120..6eabf477f3c 100644 --- a/heronpy/api/metrics.py +++ b/heronpy/api/metrics.py @@ -23,12 +23,11 @@ # pylint: disable=attribute-defined-outside-init -class IMetric(object): +class IMetric: """Interface for Heron Metric""" @abstractmethod def get_value_and_reset(self): """Returns the current value and reset""" - pass class CountMetric(IMetric): """Counter for a single value""" @@ -68,22 +67,19 @@ def get_value_and_reset(self): return ret # Reducer metric -class IReducer(object): +class IReducer: """Interface for Reducer""" @abstractmethod def init(self): """Called when this reducer is initialized/reinitialized""" - pass @abstractmethod def reduce(self, value): """Called to reduce the value""" - pass @abstractmethod def extract(self): """Called to extract the current value""" - pass class MeanReducer(IReducer): """Mean Reducer""" @@ -98,8 +94,7 @@ def reduce(self, value): def extract(self): if self.count > 0: return float(self.sum)/self.count - else: - return None + return None class ReducedMetric(IMetric): """Reduced Metric""" diff --git a/heronpy/api/serializer.py b/heronpy/api/serializer.py index 4441c43cd03..4f9759e567e 100644 --- a/heronpy/api/serializer.py +++ b/heronpy/api/serializer.py @@ -28,12 +28,11 @@ import heronpy.api.cloudpickle as cloudpickle -class IHeronSerializer(object): +class IHeronSerializer: """Serializer interface for Heron""" @abstractmethod def initialize(self, config): """Initializes the serializer""" - pass @abstractmethod def serialize(self, obj): @@ -42,7 +41,6 @@ def serialize(self, obj): :param obj: The object to be serialized :returns: Serialized object as byte string """ - pass @abstractmethod def deserialize(self, input_str): @@ -51,7 +49,6 @@ def deserialize(self, input_str): :param input_str: Serialized object as byte string :returns: Deserialized object """ - pass class PythonSerializer(IHeronSerializer): """Default serializer""" diff --git a/heronpy/api/spout/spout.py b/heronpy/api/spout/spout.py index 504f0820fb8..6e86182fa42 100644 --- a/heronpy/api/spout/spout.py +++ b/heronpy/api/spout/spout.py @@ -52,7 +52,6 @@ def initialize(self, config, context): topology, including the task id and component id of this task, input and output information, etc. """ - pass @abstractmethod def close(self): @@ -60,7 +59,6 @@ def close(self): There is no guarantee that close() will be called. """ - pass @abstractmethod def next_tuple(self): @@ -92,7 +90,6 @@ def ack(self, tup_id): :param tup_id: the ID of the HeronTuple that has been fully acknowledged. """ - pass @abstractmethod def fail(self, tup_id): @@ -109,7 +106,6 @@ def fail(self, tup_id): :param tup_id: the ID of the HeronTuple that has failed either due to a bolt calling ``fail()`` or timeout """ - pass @abstractmethod def activate(self): @@ -119,7 +115,6 @@ def activate(self): after having been deactivated when the topology is manipulated using the `heron` client. """ - pass @abstractmethod def deactivate(self): @@ -128,4 +123,3 @@ def deactivate(self): next_tuple() will not be called while a spout is deactivated. The spout may or may not be reactivated in the future. """ - pass diff --git a/heronpy/api/state/state.py b/heronpy/api/state/state.py index 43a9d289b2c..bea9a90e3a4 100644 --- a/heronpy/api/state/state.py +++ b/heronpy/api/state/state.py @@ -21,7 +21,7 @@ '''state.py''' from abc import abstractmethod -class State(object): +class State: """State represents the state interface as seen by stateful bolts and spouts. In Heron, state gives a notional Key/Value interface along with the ability to iterate over the key/values @@ -32,7 +32,6 @@ def put(self, key, value): :param key: The key to get back the value :param value: The value associated with the key """ - pass @abstractmethod def get(self, key): @@ -40,20 +39,17 @@ def get(self, key): :param key: The key whose value we want back :return: The value associated with the key """ - pass @abstractmethod def enumerate(self): """Allows one to enumerate over the state. :return: The enumerate object """ - pass @abstractmethod def clear(self): """Clears the state to empty state """ - pass class HashMapState(State): """HashMapState represents default implementation of the State interface @@ -61,14 +57,11 @@ class HashMapState(State): def __init__(self): self._dict = {} - def put(self, k, v): - self._dict[k] = v + def put(self, key, value): + self._dict[key] = value - def get(self, k): - if k in self._dict: - return self._dict[k] - else: - return None + def get(self, key): + return self._dict.get(key) def enumerate(self): return enumerate(self._dict) diff --git a/heronpy/api/state/stateful_component.py b/heronpy/api/state/stateful_component.py index 7df55655cf7..d61ce6e0ddc 100644 --- a/heronpy/api/state/stateful_component.py +++ b/heronpy/api/state/stateful_component.py @@ -21,7 +21,7 @@ '''stateful_component.py''' from abc import abstractmethod -class StatefulComponent(object): +class StatefulComponent: """Defines a component that saves its internal state using the State interface When running under effectively once semantics, the state is periodically checkpointed and is replayed when errors occur to a globally consistent checkpoint. @@ -36,11 +36,9 @@ def init_state(self, state): Note that init_state() is called before initialize() :param state: the previously saved state of the component. """ - pass @abstractmethod def pre_save(self): """This is a hook for the component to perform some actions just before the framework saves its state. """ - pass diff --git a/heronpy/api/stream.py b/heronpy/api/stream.py index b959d03dcae..828f55342a3 100644 --- a/heronpy/api/stream.py +++ b/heronpy/api/stream.py @@ -19,7 +19,7 @@ from heronpy.api.custom_grouping import ICustomGrouping from heronpy.proto import topology_pb2 -class Stream(object): +class Stream: """Heron output stream It is compatible with StreamParse API. @@ -50,7 +50,7 @@ def __init__(self, fields=None, name=DEFAULT_STREAM_ID, direct=False): if name is None: raise TypeError("Stream's name cannot be None") - elif isinstance(name, str): + if isinstance(name, str): self.stream_id = name else: raise TypeError("Stream name must be a string, given: %s" % str(name)) @@ -62,7 +62,7 @@ def __init__(self, fields=None, name=DEFAULT_STREAM_ID, direct=False): else: raise TypeError("'direct' must be either True or False, given: %s" % str(direct)) -class Grouping(object): +class Grouping: """Helper class for defining Grouping for Python topology""" SHUFFLE = topology_pb2.Grouping.Value("SHUFFLE") ALL = topology_pb2.Grouping.Value("ALL") @@ -83,18 +83,17 @@ class Grouping(object): @classmethod def is_grouping_sane(cls, gtype): """Checks if a given gtype is sane""" - if gtype == cls.SHUFFLE or gtype == cls.ALL or gtype == cls.LOWEST or gtype == cls.NONE: + if gtype in (cls.SHUFFLE, cls.ALL, cls.LOWEST, cls.NONE): return True - elif isinstance(gtype, cls.FIELDS): + if isinstance(gtype, cls.FIELDS): return gtype.gtype == topology_pb2.Grouping.Value("FIELDS") and \ gtype.fields is not None - elif isinstance(gtype, cls.CUSTOM): + if isinstance(gtype, cls.CUSTOM): return gtype.gtype == topology_pb2.Grouping.Value("CUSTOM") and \ gtype.python_serialized is not None - else: - #pylint: disable=fixme - #TODO: DIRECT are not supported yet - return False + #pylint: disable=fixme + #TODO: DIRECT are not supported yet + return False @classmethod def fields(cls, *fields): @@ -148,9 +147,8 @@ def custom_serialized(cls, serialized, is_java=True): if not is_java: return cls.CUSTOM(gtype=topology_pb2.Grouping.Value("CUSTOM"), python_serialized=serialized) - else: - raise NotImplementedError("Custom grouping implemented in Java for Python topology" - "is not yet supported.") + raise NotImplementedError("Custom grouping implemented in Java for Python topology" + "is not yet supported.") @classmethod def custom_object(cls, java_class_name, arg_list): diff --git a/heronpy/api/task_hook.py b/heronpy/api/task_hook.py index 91a863c4eb4..19fedcb48d3 100644 --- a/heronpy/api/task_hook.py +++ b/heronpy/api/task_hook.py @@ -22,7 +22,7 @@ from collections import namedtuple from abc import abstractmethod -class ITaskHook(object): +class ITaskHook: """ITaskHook is an interface for defining task hooks for a topology""" @abstractmethod @@ -32,12 +32,10 @@ def prepare(self, conf, context): :param conf: component-specific configuration passed to the topology :param context: topology context """ - pass @abstractmethod def clean_up(self): """Called just before the spout/bolt's cleanup method is called""" - pass @abstractmethod def emit(self, emit_info): @@ -45,7 +43,6 @@ def emit(self, emit_info): :param emit_info: EmitInfo object """ - pass @abstractmethod def spout_ack(self, spout_ack_info): @@ -53,7 +50,6 @@ def spout_ack(self, spout_ack_info): :param spout_ack_info: SpoutAckInfo object """ - pass @abstractmethod def spout_fail(self, spout_fail_info): @@ -61,7 +57,6 @@ def spout_fail(self, spout_fail_info): :param spout_fail_info: SpoutFailInfo object """ - pass @abstractmethod def bolt_execute(self, bolt_execute_info): @@ -69,7 +64,6 @@ def bolt_execute(self, bolt_execute_info): :param bolt_execute_info: BoltExecuteInfo object """ - pass @abstractmethod def bolt_ack(self, bolt_ack_info): @@ -77,7 +71,6 @@ def bolt_ack(self, bolt_ack_info): :param bolt_ack_info: BoltAckInfo object """ - pass @abstractmethod def bolt_fail(self, bolt_fail_info): @@ -85,7 +78,6 @@ def bolt_fail(self, bolt_fail_info): :param bolt_fail_info: BoltFailInfo object """ - pass ################################################################################## diff --git a/heronpy/api/tests/python/component_unittest.py b/heronpy/api/tests/python/component_unittest.py index f8b5f46f38f..a93baac5e07 100644 --- a/heronpy/api/tests/python/component_unittest.py +++ b/heronpy/api/tests/python/component_unittest.py @@ -175,7 +175,6 @@ def test_sanitize_inputs(self): with self.assertRaises(ValueError): spec._sanitize_inputs() - # pylint: disable=redefined-variable-type # pylint: disable=pointless-statement def test_sanitize_outputs(self): # outputs is None (no argument to outputs) diff --git a/heronpy/api/topology.py b/heronpy/api/topology.py index 5de7057a887..a628224d2f3 100644 --- a/heronpy/api/topology.py +++ b/heronpy/api/topology.py @@ -80,8 +80,7 @@ def class_dict_to_specs(mcs, class_dict): spec.name = name if spec.name in specs: raise ValueError("Duplicate component name: %s" % spec.name) - else: - specs[spec.name] = spec + specs[spec.name] = spec return specs @classmethod @@ -258,7 +257,7 @@ def _sanitize_config(custom_config): return sanitized @six.add_metaclass(TopologyType) -class Topology(object): +class Topology: """Topology is an abstract class for defining a topology Topology writers can define their custom topology by inheriting this class. @@ -304,7 +303,7 @@ def write(cls): with open(path, 'wb') as f: f.write(cls.protobuf_topology.SerializeToString()) -class TopologyBuilder(object): +class TopologyBuilder: """Builder for heronpy.api.src.python topology This class dynamically creates a subclass of `Topology` with given spouts and diff --git a/heronpy/api/topology_context.py b/heronpy/api/topology_context.py index 1a054594c9c..4f6083e9661 100644 --- a/heronpy/api/topology_context.py +++ b/heronpy/api/topology_context.py @@ -21,7 +21,7 @@ '''topology_context.py''' from abc import abstractmethod -class TopologyContext(object): +class TopologyContext: """Topology Context is the means for spouts/bolts to get information about the running topology. This file just is the interface to be used by spouts/bolts @@ -33,14 +33,12 @@ def get_task_id(self): """Gets the task id of this component :return: the task_id of this component """ - pass @abstractmethod def get_component_id(self): """Gets the component id of this component :return: the component_id of this component """ - pass @abstractmethod def get_cluster_config(self): @@ -48,14 +46,12 @@ def get_cluster_config(self): Note that the returned config is auto-typed map: any Python object>. :return: the dict of key -> value """ - pass @abstractmethod def get_topology_name(self): """Returns the name of the topology :return: the name of the topology """ - pass @abstractmethod def register_metric(self, name, metric, time_bucket_in_sec): @@ -64,7 +60,6 @@ def register_metric(self, name, metric, time_bucket_in_sec): :param metric: The IMetric that needs to be registered :param time_bucket_in_sec: The interval in seconds to do getValueAndReset """ - pass @abstractmethod def get_sources(self, component_id): @@ -74,7 +69,6 @@ def get_sources(self, component_id): :return: map gtype>, or None if not found """ - pass def get_this_sources(self): """Returns the declared inputs to this component @@ -90,7 +84,6 @@ def get_component_tasks(self, component_id): :return: list of task_ids or None if not found """ - pass @abstractmethod def add_task_hook(self, task_hook): @@ -99,4 +92,3 @@ def add_task_hook(self, task_hook): :type task_hook: ITaskHook :param task_hook: Implementation of ITaskHook """ - pass diff --git a/heronpy/api/tuple.py b/heronpy/api/tuple.py index e1d07d22fc5..6841135ed60 100644 --- a/heronpy/api/tuple.py +++ b/heronpy/api/tuple.py @@ -37,7 +37,7 @@ :type values: tuple """ -class TupleHelper(object): +class TupleHelper: """Tuple generator, returns StreamParse compatible tuple""" TICK_TUPLE_ID = "__tick" TICK_SOURCE_COMPONENT = "__system" diff --git a/heronpy/streamlet/builder.py b/heronpy/streamlet/builder.py index 06a27e3f052..050bc06dee8 100644 --- a/heronpy/streamlet/builder.py +++ b/heronpy/streamlet/builder.py @@ -26,7 +26,7 @@ from heronpy.streamlet.impl.supplierspout import SupplierStreamlet from heronpy.streamlet.impl.generatorspout import GeneratorStreamlet -class Builder(object): +class Builder: """A Builder object is used to build the functional API DAG in Heron.""" def __init__(self): """ diff --git a/heronpy/streamlet/config.py b/heronpy/streamlet/config.py index bbf10fd6da9..7432735ef9c 100644 --- a/heronpy/streamlet/config.py +++ b/heronpy/streamlet/config.py @@ -23,7 +23,7 @@ import heronpy.api.api_constants as api_constants from heronpy.streamlet.resources import Resources -class Config(object): +class Config: """Config is the way users configure the execution of the topology. Things like tuple delivery semantics, resources used, as well as user defined key/value pairs are passed on to the runner via diff --git a/heronpy/streamlet/context.py b/heronpy/streamlet/context.py index bcfbae2d5aa..c8967f46f33 100644 --- a/heronpy/streamlet/context.py +++ b/heronpy/streamlet/context.py @@ -22,7 +22,7 @@ from abc import abstractmethod -class Context(object): +class Context: """Context is the information available at runtime for operators like transform. It contains basic things like config, runtime information like task, the stream that it is operating on, ProcessState, etc. @@ -32,39 +32,32 @@ class Context(object): def get_task_id(self): """Fetches the task id of the current instance of the operator """ - pass @abstractmethod def get_config(self): """Fetches the config of the computation """ - pass @abstractmethod def get_stream_name(self): """Fetches the stream name that we are operating on """ - pass @abstractmethod def get_num_partitions(self): """Fetches the number of partitions of the stream we are operating on """ - pass def get_partition_index(self): """Fetches the partition of the stream that we are operating on """ - pass @abstractmethod def get_state(self): """The state where components can store any of their local state """ - pass @abstractmethod def emit(self, values): """Emits the values in the output stream """ - pass diff --git a/heronpy/streamlet/generator.py b/heronpy/streamlet/generator.py index 194bc2dc25f..2eed5c98939 100644 --- a/heronpy/streamlet/generator.py +++ b/heronpy/streamlet/generator.py @@ -14,7 +14,7 @@ '''generator.py: API for defining generic sources in python''' from abc import abstractmethod -class Generator(object): +class Generator: """API for defining a generic source for Heron in the Python Streamlet API """ diff --git a/heronpy/streamlet/impl/joinbolt.py b/heronpy/streamlet/impl/joinbolt.py index ba708567a64..1dde5276987 100644 --- a/heronpy/streamlet/impl/joinbolt.py +++ b/heronpy/streamlet/impl/joinbolt.py @@ -73,6 +73,7 @@ def initialize(self, config, context): self._join_type = config[JoinBolt.JOINTYPE] def processWindow(self, window_config, tuples): + """Process a window""" # our temporary map mymap = {} for tup in tuples: diff --git a/heronpy/streamlet/impl/repartitionbolt.py b/heronpy/streamlet/impl/repartitionbolt.py index 78fc49e223f..1e70c91044b 100644 --- a/heronpy/streamlet/impl/repartitionbolt.py +++ b/heronpy/streamlet/impl/repartitionbolt.py @@ -33,6 +33,8 @@ # pylint: disable=unused-argument class RepartitionCustomGrouping(ICustomGrouping): + """Implementation of repartitioning grouping""" + def __init__(self, repartition_function): self._repartition_function = repartition_function diff --git a/heronpy/streamlet/impl/streamletboltbase.py b/heronpy/streamlet/impl/streamletboltbase.py index b4fbd7e9168..99dea5abfe9 100644 --- a/heronpy/streamlet/impl/streamletboltbase.py +++ b/heronpy/streamlet/impl/streamletboltbase.py @@ -21,7 +21,7 @@ """module for base streamlet bolt: StreamletBoltBase""" from heronpy.api.stream import Stream -class StreamletBoltBase(object): +class StreamletBoltBase: """StreamletBoltBase""" # output declarer outputs = [Stream(fields=['_output_'], name='output')] diff --git a/heronpy/streamlet/impl/streamletspoutbase.py b/heronpy/streamlet/impl/streamletspoutbase.py index 260957c20b7..6822717938d 100644 --- a/heronpy/streamlet/impl/streamletspoutbase.py +++ b/heronpy/streamlet/impl/streamletspoutbase.py @@ -21,7 +21,7 @@ """module for base streamlet API spout: StreamletSpoutBase""" from heronpy.api.stream import Stream -class StreamletSpoutBase(object): +class StreamletSpoutBase: """StreamletSpoutBase""" # output declarer outputs = [Stream(fields=['_output_'], name='output')] diff --git a/heronpy/streamlet/keyedwindow.py b/heronpy/streamlet/keyedwindow.py index 51c84f1ccb9..495dfd24477 100644 --- a/heronpy/streamlet/keyedwindow.py +++ b/heronpy/streamlet/keyedwindow.py @@ -22,7 +22,7 @@ from heronpy.streamlet.window import Window -class KeyedWindow(object): +class KeyedWindow: """Transformation depending on Windowing pass on the window/key information using this class """ diff --git a/heronpy/streamlet/resources.py b/heronpy/streamlet/resources.py index e4be526b0d9..7052995cab7 100644 --- a/heronpy/streamlet/resources.py +++ b/heronpy/streamlet/resources.py @@ -20,7 +20,7 @@ '''resources.py: module for defining resources''' -class Resources(object): +class Resources: """Resources needed by the topology are encapsulated in this class. Currently we deal with CPU and RAM. Others can be added later. """ diff --git a/heronpy/streamlet/runner.py b/heronpy/streamlet/runner.py index a955b42ecda..c0b9e407672 100644 --- a/heronpy/streamlet/runner.py +++ b/heronpy/streamlet/runner.py @@ -24,13 +24,12 @@ from heronpy.streamlet.builder import Builder from heronpy.streamlet.config import Config -class Runner(object): +class Runner: """Runner is used to run a topology that is built by the builder. It exports a sole function called run that takes care of constructing the topology """ def __init__(self): """Nothing really""" - pass # pylint: disable=protected-access, no-self-use def run(self, name, config, builder): diff --git a/heronpy/streamlet/streamlet.py b/heronpy/streamlet/streamlet.py index 4fd83f6f005..ef6f9322aaa 100644 --- a/heronpy/streamlet/streamlet.py +++ b/heronpy/streamlet/streamlet.py @@ -24,7 +24,7 @@ from heronpy.streamlet.impl.streamletboltbase import StreamletBoltBase # pylint: disable=too-many-instance-attributes, protected-access -class Streamlet(object): +class Streamlet: """A Streamlet is a (potentially unbounded) ordered collection of tuples Streamlets originate from pub/sub systems(such Pulsar/Kafka), or from static data(such as csv files, HDFS files), or for that matter any other source. They are also created by @@ -59,6 +59,7 @@ def get_num_partitions(self): def map(self, map_function): """Return a new Streamlet by applying map_function to each element of this Streamlet. """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.mapbolt import MapStreamlet map_streamlet = MapStreamlet(map_function, self) self._add_child(map_streamlet) @@ -68,6 +69,7 @@ def flat_map(self, flatmap_function): """Return a new Streamlet by applying map_function to each element of this Streamlet and flattening the result """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.flatmapbolt import FlatMapStreamlet fm_streamlet = FlatMapStreamlet(flatmap_function, self) self._add_child(fm_streamlet) @@ -76,6 +78,7 @@ def flat_map(self, flatmap_function): def filter(self, filter_function): """Return a new Streamlet containing only the elements that satisfy filter_function """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.filterbolt import FilterStreamlet filter_streamlet = FilterStreamlet(filter_function, self) self._add_child(filter_streamlet) @@ -90,6 +93,7 @@ def repartition(self, num_partitions, repartition_function=None): It could also return a list of partitions if it wants to send it to multiple partitions. """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.repartitionbolt import RepartitionStreamlet if repartition_function is None: repartition_function = lambda x: x @@ -113,6 +117,7 @@ def reduce_by_window(self, window_config, reduce_function): reduce_function takes two element at one time and reduces them to one element that is used in the subsequent operations. """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.reducebywindowbolt import ReduceByWindowStreamlet reduce_streamlet = ReduceByWindowStreamlet(window_config, reduce_function, self) self._add_child(reduce_streamlet) @@ -122,6 +127,7 @@ def reduce_by_window(self, window_config, reduce_function): def union(self, other_streamlet): """Returns a new Streamlet that consists of elements of both this and other_streamlet """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.unionbolt import UnionStreamlet union_streamlet = UnionStreamlet(self, other_streamlet) self._add_child(union_streamlet) @@ -134,6 +140,7 @@ def transform(self, transform_operator): Before starting to cycle over the Streamlet, the open function of the transform_operator is called. This allows the transform_operator to do any kind of initialization/loading, etc. """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.transformbolt import TransformStreamlet transform_streamlet = TransformStreamlet(transform_operator, self) self._add_child(transform_streamlet) @@ -142,22 +149,23 @@ def transform(self, transform_operator): def log(self): """Logs all elements of this streamlet. This returns nothing """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.logbolt import LogStreamlet log_streamlet = LogStreamlet(self) self._add_child(log_streamlet) - return def consume(self, consume_function): """Calls consume_function for each element of this streamlet. This function returns nothing """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.consumebolt import ConsumeStreamlet consume_streamlet = ConsumeStreamlet(consume_function, self) self._add_child(consume_streamlet) - return def join(self, join_streamlet, window_config, join_function): """Return a new Streamlet by joining join_streamlet with this streamlet """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt join_streamlet_result = JoinStreamlet(JoinBolt.INNER, window_config, join_function, self, join_streamlet) @@ -168,6 +176,7 @@ def join(self, join_streamlet, window_config, join_function): def outer_right_join(self, join_streamlet, window_config, join_function): """Return a new Streamlet by outer right join_streamlet with this streamlet """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt join_streamlet_result = JoinStreamlet(JoinBolt.OUTER_RIGHT, window_config, join_function, self, join_streamlet) @@ -178,6 +187,7 @@ def outer_right_join(self, join_streamlet, window_config, join_function): def outer_left_join(self, join_streamlet, window_config, join_function): """Return a new Streamlet by left join_streamlet with this streamlet """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt join_streamlet_result = JoinStreamlet(JoinBolt.OUTER_LEFT, window_config, join_function, self, join_streamlet) @@ -188,6 +198,7 @@ def outer_left_join(self, join_streamlet, window_config, join_function): def outer_join(self, join_streamlet, window_config, join_function): """Return a new Streamlet by outer join_streamlet with this streamlet """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt join_streamlet_result = JoinStreamlet(JoinBolt.OUTER, window_config, @@ -200,6 +211,7 @@ def reduce_by_key_and_window(self, window_config, reduce_function): """Return a new Streamlet in which each (key, value) pair of this Streamlet are collected over the time_window and then reduced using the reduce_function """ + # pylint: disable=import-outside-toplevel from heronpy.streamlet.impl.reducebykeyandwindowbolt import ReduceByKeyAndWindowStreamlet reduce_streamlet = ReduceByKeyAndWindowStreamlet(window_config, reduce_function, self) self._add_child(reduce_streamlet) diff --git a/heronpy/streamlet/transformoperator.py b/heronpy/streamlet/transformoperator.py index 66b99650747..ba7a651ca31 100644 --- a/heronpy/streamlet/transformoperator.py +++ b/heronpy/streamlet/transformoperator.py @@ -14,7 +14,7 @@ '''transformoperator.py: API for defining generic transformer in python''' from abc import abstractmethod -class TransformOperator(object): +class TransformOperator: """API for defining a generic transformer for Heron in the Python Streamlet API """ diff --git a/heronpy/streamlet/window.py b/heronpy/streamlet/window.py index 4113bc4c070..8938ba2dbb1 100644 --- a/heronpy/streamlet/window.py +++ b/heronpy/streamlet/window.py @@ -20,7 +20,7 @@ '''window.py: module for defining Window''' -class Window(object): +class Window: """Window is a container containing information about a particular window. Transformations that depend on Windowing, pass the window information inside their streamlets using this container. diff --git a/heronpy/streamlet/windowconfig.py b/heronpy/streamlet/windowconfig.py index f0ce7d8e493..ef0a75dea35 100644 --- a/heronpy/streamlet/windowconfig.py +++ b/heronpy/streamlet/windowconfig.py @@ -22,7 +22,7 @@ import datetime -class WindowConfig(object): +class WindowConfig: """WindowConfig allows streamlet API users to program window configuration for operations that rely on windowing. Currently we only support time/count based sliding/tumbling windows. diff --git a/integration_test/src/python/common/status.py b/integration_test/src/python/common/status.py index 876b3e84faa..423cc2b0cdf 100644 --- a/integration_test/src/python/common/status.py +++ b/integration_test/src/python/common/status.py @@ -31,7 +31,7 @@ def __init__(self, message, error=None): else: logging.error(message) -class TestSuccess(object): +class TestSuccess: def __init__(self, message=None): if message: logging.info(message) diff --git a/integration_test/src/python/local_test_runner/test_template.py b/integration_test/src/python/local_test_runner/test_template.py index 9f0d3ba9548..f793875d53c 100644 --- a/integration_test/src/python/local_test_runner/test_template.py +++ b/integration_test/src/python/local_test_runner/test_template.py @@ -54,7 +54,7 @@ HERON_STMGR_CMD = os.path.join(HERON_SANDBOX_HOME, HERON_CORE, HERON_BIN, HERON_STMGR) ProcessTuple = namedtuple('ProcessTuple', 'pid cmd') -class TestTemplate(object): +class TestTemplate: """ Class that encapsulates the template used for integration tests. Intended to be abstract and subclassed for specific tests. """ diff --git a/integration_test/src/python/test_runner/main.py b/integration_test/src/python/test_runner/main.py index a09803a2b8e..351c1399184 100644 --- a/integration_test/src/python/test_runner/main.py +++ b/integration_test/src/python/test_runner/main.py @@ -42,7 +42,7 @@ successes = [] failures = [] -class FileBasedExpectedResultsHandler(object): +class FileBasedExpectedResultsHandler: def __init__(self, file_path): self.file_path = file_path @@ -57,7 +57,7 @@ def fetch_results(self): except Exception as e: raise status.TestFailure("Failed to read expected result file %s" % self.file_path, e) -class HttpBasedExpectedResultsHandler(object): +class HttpBasedExpectedResultsHandler: def __init__(self, server_host_port, topology_name, task_count): self.server_host_port = server_host_port self.topology_name = topology_name @@ -87,7 +87,7 @@ def fetch_results(self): raise status.TestFailure( "Fetching expected result failed for %s topology" % self.topology_name, e) -class HttpBasedActualResultsHandler(object): +class HttpBasedActualResultsHandler: def __init__(self, server_host_port, topology_name): self.server_host_port = server_host_port self.topology_name = topology_name @@ -100,7 +100,7 @@ def fetch_results(self): raise status.TestFailure("Fetching result failed for %s topology" % self.topology_name, e) # pylint: disable=unnecessary-lambda -class ExactlyOnceResultsChecker(object): +class ExactlyOnceResultsChecker: """Compares what results we found against what was expected. Verifies and exact match""" def __init__(self, topology_name, expected_results_handler, actual_results_handler): diff --git a/integration_test/src/python/topology_test_runner/main.py b/integration_test/src/python/topology_test_runner/main.py index 2ba818466a6..8e5ec456140 100644 --- a/integration_test/src/python/topology_test_runner/main.py +++ b/integration_test/src/python/topology_test_runner/main.py @@ -44,7 +44,7 @@ successes = [] failures = [] -class TopologyStructureResultChecker(object): +class TopologyStructureResultChecker: """ Validate topology graph structure """ @@ -224,7 +224,7 @@ def _parse_instance_id(self, input): return output -class FileBasedExpectedResultsHandler(object): +class FileBasedExpectedResultsHandler: """ Get expected topology graph structure result from local file """ @@ -245,7 +245,7 @@ def fetch_results(self): raise status.TestFailure("Failed to read expected result file %s" % self.file_path, e) -class ZkFileBasedActualResultsHandler(object): +class ZkFileBasedActualResultsHandler: """ Get actual topology graph structure result from zk """ @@ -294,7 +294,7 @@ def stop_state_mgr(self): self.state_mgr.stop() -class HttpBasedActualResultsHandler(object): +class HttpBasedActualResultsHandler: """ Get actually loaded instance states TODO(yaoli): complete this class when stateful processing is ready diff --git a/third_party/python/cpplint/cpplint.py b/third_party/python/cpplint/cpplint.py index 4e196e8ea3f..a7be857eb8f 100755 --- a/third_party/python/cpplint/cpplint.py +++ b/third_party/python/cpplint/cpplint.py @@ -794,7 +794,7 @@ def _IsSourceExtension(s): return s in GetNonHeaderExtensions() -class _IncludeState(object): +class _IncludeState: """Tracks line numbers for includes, and the order in which includes appear. include_list contains list of lists of (header, line number) pairs. @@ -961,7 +961,7 @@ def CheckNextIncludeOrder(self, header_type): return '' -class _CppLintState(object): +class _CppLintState: """Maintains module-wide state..""" def __init__(self): @@ -1183,7 +1183,7 @@ def _RestoreFilters(): """ Restores filters previously backed up.""" _cpplint_state.RestoreFilters() -class _FunctionState(object): +class _FunctionState: """Tracks current function name and the number of lines in its body.""" _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. @@ -1247,7 +1247,7 @@ class _IncludeError(Exception): pass -class FileInfo(object): +class FileInfo: """Provides utility functions for filenames. FileInfo provides easy access to the components of a file's path @@ -1591,7 +1591,7 @@ def CleanseComments(line): return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) -class CleansedLines(object): +class CleansedLines: """Holds 4 copies of all lines with different preprocessing applied to them. 1) elided member contains lines without strings and comments. @@ -2310,7 +2310,7 @@ def IsForwardClassDeclaration(clean_lines, linenum): return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) -class _BlockInfo(object): +class _BlockInfo: """Stores information about a generic block of code.""" def __init__(self, linenum, seen_open_brace): @@ -2497,7 +2497,7 @@ def CheckEnd(self, filename, clean_lines, linenum, error): 'Anonymous namespace should be terminated with "// namespace"') -class _PreprocessorInfo(object): +class _PreprocessorInfo: """Stores checkpoints of nesting stacks when #if/#else is seen.""" def __init__(self, stack_before_if): @@ -2511,7 +2511,7 @@ def __init__(self, stack_before_if): self.seen_else = False -class NestingState(object): +class NestingState: """Holds states related to parsing braces.""" def __init__(self): From b3bb9d2d358ac89c7a361c935cfac8cb88b3b78f Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 9 May 2020 14:20:31 +0100 Subject: [PATCH 04/22] update setuptools --- WORKSPACE | 10 +++++----- heron/proto/BUILD | 2 +- heronpy/proto/BUILD | 2 +- tools/rules/pex/BUILD | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index e204e5ca1d4..69d03bd85a5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -180,7 +180,7 @@ PYTEST_WHEEL = "https://pypi.python.org/packages/fd/3e/d326a05d083481746a769fc05 REQUESTS_SRC = "https://pypi.python.org/packages/d9/03/155b3e67fe35fe5b6f4227a8d9e96a14fda828b18199800d161bcefc1359/requests-2.12.3.tar.gz" -SETUPTOOLS_SRC = "https://pypi.python.org/packages/68/13/1bfbfbd86560e61fa9803d241084fff41a775bf56ee8b3ad72fc9e550dad/setuptools-31.0.0.tar.gz" +SETUPTOOLS_WHEEL = "https://pypi.python.org/packages/a0/df/635cdb901ee4a8a42ec68e480c49f85f4c59e8816effbf57d9e6ee8b3588/setuptools-46.1.3-py3-none-any.whl" VIRTUALENV_SRC = "https://pypi.python.org/packages/d4/0c/9840c08189e030873387a73b90ada981885010dd9aea134d6de30cd24cb8/virtualenv-15.1.0.tar.gz" @@ -224,10 +224,10 @@ http_file( ) http_file( - name = "setuptools_src", - downloaded_file_path = "setuptools-31.0.0.tar.gz", - sha256 = "0818cc0de692c3a5c83ca83aa7ec7ba6bc206f278735f1e0267b8d0e095cfe7a", - urls = [SETUPTOOLS_SRC], + name = "setuptools_wheel", + downloaded_file_path = "setuptools-46.1.3-py3-none-any.whl", + sha256 = "4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee", + urls = [SETUPTOOLS_WHEEL], ) http_archive( diff --git a/heron/proto/BUILD b/heron/proto/BUILD index 5da290884b7..7bab2b7e2ec 100644 --- a/heron/proto/BUILD +++ b/heron/proto/BUILD @@ -182,7 +182,7 @@ pex_library( name = "proto-py", reqs = [ "protobuf==3.8.0", - "setuptools==18.8.1", + "setuptools==46.1.3", ], deps = [ ":proto_ckptmgr_py", diff --git a/heronpy/proto/BUILD b/heronpy/proto/BUILD index b0f52461de5..5eb56958e10 100644 --- a/heronpy/proto/BUILD +++ b/heronpy/proto/BUILD @@ -28,7 +28,7 @@ pex_library( srcs = glob(["**/*.py"]), reqs = [ "protobuf==3.8.0", - "setuptools==18.8.1", + "setuptools==46.1.3", ], deps = [ ":proto_ckptmgr_py", diff --git a/tools/rules/pex/BUILD b/tools/rules/pex/BUILD index 510481c45d3..26dca5888c7 100644 --- a/tools/rules/pex/BUILD +++ b/tools/rules/pex/BUILD @@ -34,7 +34,7 @@ POST_EXECUTE = [ --quiet --no-cache-dir --no-index --build $(@D)/pexbuild \ --find-links $$(dirname $(location @pex_src//file)) \ --find-links $$(dirname $(location @wheel_src//file)) \ - --find-links $$(dirname $(location @setuptools_src//file))', + --find-links $$(dirname $(location @setuptools_wheel//file))', '# Work around setuptools insistance on writing to the source directory,', '# which is discouraged by Bazel (and annoying)', @@ -46,7 +46,7 @@ POST_EXECUTE = [ --entry-point=pex_wrapper \ --output-file=$@ \ --find-links $$(dirname $(location @pex_src//file)) \ - --find-links $$(dirname $(location @setuptools_src//file)) \ + --find-links $$(dirname $(location @setuptools_wheel//file)) \ --find-links $$(dirname $(location @requests_src//file)) \ --find-links $$(dirname $(location @wheel_src//file))', ] @@ -57,7 +57,7 @@ genrule( "wrapper/setup.py", "wrapper/pex_wrapper.py", "wrapper/README", - "@setuptools_src//file", + "@setuptools_wheel//file", "@wheel_src//file", "@pex_src//file", "@requests_src//file", From 38d0b95543c2b700f9a15667e7c17249a92f5a26 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 9 May 2020 14:55:36 +0100 Subject: [PATCH 05/22] Make pex_pytest non-zip-safe --- tools/rules/pex/pex_rules.bzl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/rules/pex/pex_rules.bzl b/tools/rules/pex/pex_rules.bzl index 2dbe33347a3..0ad7f198107 100644 --- a/tools/rules/pex/pex_rules.bzl +++ b/tools/rules/pex/pex_rules.bzl @@ -195,6 +195,8 @@ def _pex_binary_impl(ctx): "--output-file", deploy_pex.path, "--disable-cache", + "--python-shebang", "#!/usr/bin/env python3", + "--no-compile", manifest_file.path, ] #EXTRA_PEX_ARGS# @@ -321,7 +323,7 @@ pex_bin_attrs = _dmerge(pex_attrs, { "pex_verbosity": attr.int(default = 0), "resources": attr.label_list(allow_files = True), "zip_safe": attr.bool( - default = True, + default = False, mandatory = False, ), }) From 6edf005d46dc84e0d6a38228670387742e23bc8b Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Mon, 18 May 2020 20:51:36 +0100 Subject: [PATCH 06/22] Rough proto_library fix The issue encountered was https://github.com/protocolbuffers/protobuf/issues/1491 which may be fixed by a pending PR to protoc, or with a switch to the official protobuf rules and the import_prefix parameter to proto_library. --- tools/rules/genproto.bzl | 5 ++ tools/rules/proto.bzl | 183 --------------------------------------- 2 files changed, 5 insertions(+), 183 deletions(-) delete mode 100644 tools/rules/proto.bzl diff --git a/tools/rules/genproto.bzl b/tools/rules/genproto.bzl index 64574151633..385a1e427aa 100644 --- a/tools/rules/genproto.bzl +++ b/tools/rules/genproto.bzl @@ -165,6 +165,11 @@ def proto_library( proto_cmd = "$(location %s) --python_out=$(@D) %s" % (protoc, proto_path) else: proto_cmd = "$(location %s) %s --python_out=$(@D) %s" % (protoc, proto_include_paths, proto_path) + # hack to work around not having import_prefix from the official proto rules which is needed to sort out imports + # and without having https://github.com/protocolbuffers/protobuf/pull/7470 + # import common_pb2 as common__pb2 -> import .common_pb2 as common__pb2 + proto_cmd += "\nfind $(@D) -type f -name '*_pb2.py' -exec sed -i.bak -E 's/^(import )([^ .]+_pb2)/from . import \\2/' {} \\;" + py_deps = [] proto_deps = [src, protoc] for dep in deps: diff --git a/tools/rules/proto.bzl b/tools/rules/proto.bzl deleted file mode 100644 index 11308bdd0e5..00000000000 --- a/tools/rules/proto.bzl +++ /dev/null @@ -1,183 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -load("@rules_java//java:defs.bzl", "java_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("pex_rules", "pex_library") - -def proto_package_impl(ctx): - return struct(proto_src = ctx.file.src) - -genproto_base_attrs = { - "src": attr.label( - allow_files = [".proto"], - allow_single_file = True, - ), - "deps": attr.label_list( - allow_files = False, - providers = ["proto_src"], - ), -} - -proto_package = rule( - proto_package_impl, - attrs = genproto_base_attrs, -) - -def genproto_java_impl(ctx): - src = ctx.file.src - protoc = ctx.file._protoc - - srcjar = ctx.actions.declare_file(ctx.configuration.genfiles_dir, ctx.label.name + ".srcjar") - java_srcs = srcjar.path + ".srcs" - - inputs = [src, protoc] - java_cmd = "\n".join([ - "set -e", - "rm -rf " + java_srcs, - "mkdir " + java_srcs, - protoc.path + " -I heron/proto --java_out=" + java_srcs + " " + src.path, - "jar cMf " + srcjar.path + " -C " + java_srcs + " .", - "rm -rf " + java_srcs, - ]) - ctx.actions.run( - inputs = inputs, - outputs = [srcjar], - mnemonic = "ProtocJava", - command = java_cmd, - use_default_shell_env = True, - ) - - return struct(files = set([srcjar])) - -genproto_java = rule( - genproto_java_impl, - attrs = genproto_base_attrs.update({ - "_protoc": attr.label( - default = Label("//third_party/protobuf:protoc"), - allow_files = True, - allow_single_file = True, - ), - }), -) - -def proto_library( - name, - src = None, - includes = [], - deps = [], - visibility = None, - gen_java = False, - gen_cc = False, - gen_py = False): - if not src: - if name.endswith("_proto"): - src = name[:-6] + ".proto" - else: - src = name + ".proto" - proto_package(name = name, src = src, deps = deps) - - if gen_java: - genproto_java( - name = name + "_java_src", - src = src, - deps = deps, - visibility = ["//visibility:private"], - ) - java_deps = ["@com_google_protobuf//:protobuf_java"] - for dep in deps: - java_deps.append(dep + "_java") - java_library( - name = name + "_java", - srcs = [name + "_java_src"], - deps = java_deps, - visibility = visibility, - ) - - if not includes: - proto_include_paths = "" - else: - proto_include_paths = "".join(["-I " + incl for incl in includes]) - - if gen_cc: - # We'll guess that the repository is set up such that a .proto in - # //foo/bar has the package foo.bar. `location` is substituted with the - # relative path to its label from the workspace root. - proto_path = "$(location %s)" % src - proto_hdr = src[:-6] + ".pb.h" - proto_src = src[:-6] + ".pb.cc" - proto_srcgen_rule = name + "_cc_src" - proto_lib = name + "_cc" - protoc = "//third_party/protobuf:protoc" - if not includes: - proto_cmd = "$(location %s) --cpp_out=$(@D) %s" % (protoc, proto_path) - else: - proto_cmd = "$(location %s) %s --cpp_out=$(@D) %s" % (protoc, proto_include_paths, proto_path) - - cc_deps = ["//third_party/protobuf:protobuf-cxx"] - proto_deps = [src, protoc] - for dep in deps: - cc_deps.append(dep + "_cc") - proto_deps.append(dep) - native.genrule( - name = proto_srcgen_rule, - visibility = visibility, - outs = [proto_hdr, proto_src], - srcs = proto_deps, - cmd = proto_cmd, - ) - cc_library( - name = proto_lib, - visibility = visibility, - hdrs = [proto_hdr], - srcs = [":" + proto_srcgen_rule], - defines = ["GOOGLE_PROTOBUF_NO_RTTI"], - deps = cc_deps, - linkstatic = 1, - ) - - if gen_py: - # We'll guess that the repository is set up such that a .proto in - # //foo/bar has the package foo.bar. `location` is substituted with the - # relative path to its label from the workspace root. - proto_path = "$(location %s)" % src - proto_src = src[:-6] + "_pb2.py" - proto_srcgen_rule = name + "_py_src" - proto_lib = name + "_py" - protoc = "//third_party/protobuf:protoc" - if not includes: - proto_cmd = "$(location %s) --python_out=$(@D) %s" % (protoc, proto_path) - else: - proto_cmd = "$(location %s) %s --python_out=$(@D) %s" % (protoc, proto_include_paths, proto_path) - py_deps = [] - proto_deps = [src, protoc] - for dep in deps: - py_deps.append(dep + "_py") - proto_deps.append(dep) - native.genrule( - name = proto_srcgen_rule, - visibility = visibility, - outs = [proto_src], - srcs = proto_deps, - cmd = proto_cmd, - ) - pex_library( - name = proto_lib, - visibility = visibility, - srcs = [proto_src], - deps = py_deps, - ) From 3a61cb18f504e6814604da49d8da71bf06d4956a Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Mon, 18 May 2020 21:43:45 +0100 Subject: [PATCH 07/22] WIP: Fix python3 incompatibilities * bytes vs str issues * update kazoo * order of processes in executor test changed due to dict ordering? * some places needed / switched to // - may be more not caught by tests * add travis_wait as some stages going over 10 minutes without output in CI TODO: * make sure the kazoo upgrade is correct, it was done only by updating package versoin --- .travis.yml | 2 +- heron/executor/src/python/heron_executor.py | 28 ++++++++++------- .../tests/python/heron_executor_unittest.py | 30 +++++++++++-------- .../src/python/network/gateway_looper.py | 2 +- heron/instance/src/python/network/protocol.py | 14 ++++----- .../python/network/mock_generator_client.py | 2 +- .../tests/python/network/protocol_unittest.py | 2 +- heron/statemgrs/src/python/BUILD | 2 +- .../tracker/src/python/query_operators.py | 6 ++-- scripts/travis/build.sh | 4 +-- 10 files changed, 52 insertions(+), 40 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0d8ebe8e550..eda03912032 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,4 +44,4 @@ script: - python -V - which python3 - python3 -V - - scripts/travis/ci.sh + - stdbuf -i0 -o0 -e0 travis_wait scripts/travis/ci.sh diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py index a79bf1ca256..e3756b88272 100755 --- a/heron/executor/src/python/heron_executor.py +++ b/heron/executor/src/python/heron_executor.py @@ -180,14 +180,14 @@ def __eq__(self, other): return self.cmd == other.cmd class ProcessInfo: + """ + Container for info related to a running process + :param process: the process POpen object + :param name: the logical (i.e., unique) name of the process + :param command: an array of strings comprising the command and it's args + :param attempts: how many times the command has been run (defaults to 1) + """ def __init__(self, process, name, command, attempts=1): - """ - Container for info related to a running process - :param process: the process POpen object - :param name: the logical (i.e., unique) name of the process - :param command: an array of strings comprising the command and it's args - :param attempts: how many times the command has been run (defaults to 1) - """ self.process = process self.pid = process.pid self.name = name @@ -199,6 +199,12 @@ def increment_attempts(self): self.attempts += 1 return self + def __repr__(self): + return ( + "ProcessInfo(pid=%(pid)r, name=%(name)r, command=%(command)r, attempts=%(attempts)r)" + % vars(self) + ) + # pylint: disable=too-many-instance-attributes,too-many-statements class HeronExecutor: """ Heron executor is a class that is responsible for running each of the process on a given @@ -223,8 +229,8 @@ def init_from_parsed_args(self, parsed_args): # escaping is still left there for reference and backward compatibility purposes (to be # removed after no topology needs it) self.instance_jvm_opts =\ - base64.b64decode(parsed_args.instance_jvm_opts.lstrip('"'). - rstrip('"').replace('(61)', '=').replace('=', '=')) + base64.b64decode(parsed_args.instance_jvm_opts. + strip('"').replace('(61)', '=').replace('=', '=')).decode() self.classpath = parsed_args.classpath # Needed for Docker environments since the hostname of a docker container is the container's # id within docker, rather than the host's hostname. NOTE: this 'HOST' env variable is not @@ -255,11 +261,11 @@ def init_from_parsed_args(self, parsed_args): # removed after no topology needs it) component_jvm_opts_in_json =\ base64.b64decode(parsed_args.component_jvm_opts. - lstrip('"').rstrip('"').replace('(61)', '=').replace('=', '=')) + strip('"').replace('(61)', '=').replace('=', '=')).decode() if component_jvm_opts_in_json != "": for (k, v) in list(json.loads(component_jvm_opts_in_json).items()): # In json, the component name and JVM options are still in base64 encoding - self.component_jvm_opts[base64.b64decode(k)] = base64.b64decode(v) + self.component_jvm_opts[base64.b64decode(k).decode()] = base64.b64decode(v).decode() self.pkg_type = parsed_args.pkg_type self.topology_binary_file = parsed_args.topology_binary_file diff --git a/heron/executor/tests/python/heron_executor_unittest.py b/heron/executor/tests/python/heron_executor_unittest.py index f8b66a9220c..823898ce18f 100644 --- a/heron/executor/tests/python/heron_executor_unittest.py +++ b/heron/executor/tests/python/heron_executor_unittest.py @@ -24,6 +24,8 @@ import unittest2 as unittest import json +from pprint import pprint + from heron.executor.src.python.heron_executor import ProcessInfo from heron.executor.src.python.heron_executor import HeronExecutor from heron.proto.packing_plan_pb2 import PackingPlan @@ -180,10 +182,10 @@ def get_expected_instance_command(component_name, instance_id, container_id): '--metrics_sinks_yaml=metrics_sinks_config_file ' '--metricsmgr_port=metricsmgr_port ' '--ckptmgr_port=ckptmgr-port' % (HOSTNAME, INTERNAL_CONF_PATH, OVERRIDE_PATH)), - ProcessInfo(MockPOpen(), 'heron-shell-0', get_expected_shell_command(0)), - ProcessInfo(MockPOpen(), 'metricsmgr-0', get_expected_metricsmgr_command(0)), ProcessInfo(MockPOpen(), 'heron-metricscache', get_expected_metricscachemgr_command()), ProcessInfo(MockPOpen(), 'heron-healthmgr', get_expected_healthmgr_command()), + ProcessInfo(MockPOpen(), 'metricsmgr-0', get_expected_metricsmgr_command(0)), + ProcessInfo(MockPOpen(), 'heron-shell-0', get_expected_shell_command(0)), ] MockPOpen.set_next_pid(37) @@ -199,20 +201,17 @@ def get_expected_instance_command(component_name, instance_id, container_id): '--ckptmgr_port=ckptmgr-port --ckptmgr_id=ckptmgr-1 ' '--metricscachemgr_mode=cluster' % (HOSTNAME, INTERNAL_CONF_PATH, OVERRIDE_PATH)), + ProcessInfo(MockPOpen(), 'metricsmgr-1', get_expected_metricsmgr_command(1)), ProcessInfo(MockPOpen(), 'container_1_word_3', get_expected_instance_command('word', 3, 1)), - ProcessInfo(MockPOpen(), 'container_1_exclaim1_1', - get_expected_instance_command('exclaim1', 1, 1)), ProcessInfo(MockPOpen(), 'container_1_exclaim1_2', get_expected_instance_command('exclaim1', 2, 1)), + ProcessInfo(MockPOpen(), 'container_1_exclaim1_1', + get_expected_instance_command('exclaim1', 1, 1)), ProcessInfo(MockPOpen(), 'heron-shell-1', get_expected_shell_command(1)), - ProcessInfo(MockPOpen(), 'metricsmgr-1', get_expected_metricsmgr_command(1)), ] MockPOpen.set_next_pid(37) expected_processes_container_7 = [ - ProcessInfo(MockPOpen(), 'container_7_word_11', get_expected_instance_command('word', 11, 7)), - ProcessInfo(MockPOpen(), 'container_7_exclaim1_210', - get_expected_instance_command('exclaim1', 210, 7)), ProcessInfo(MockPOpen(), 'stmgr-7', 'stmgr_binary --topology_name=topname --topology_id=topid ' '--topologydefn_file=topdefnfile --zkhostportlist=zknode --zkroot=zkroot ' @@ -225,6 +224,9 @@ def get_expected_instance_command(component_name, instance_id, container_id): '--metricscachemgr_mode=cluster' % (HOSTNAME, INTERNAL_CONF_PATH, OVERRIDE_PATH)), ProcessInfo(MockPOpen(), 'metricsmgr-7', get_expected_metricsmgr_command(7)), + ProcessInfo(MockPOpen(), 'container_7_word_11', get_expected_instance_command('word', 11, 7)), + ProcessInfo(MockPOpen(), 'container_7_exclaim1_210', + get_expected_instance_command('exclaim1', 210, 7)), ProcessInfo(MockPOpen(), 'heron-shell-7', get_expected_shell_command(7)), ] @@ -330,12 +332,16 @@ def do_test_launch(self, executor, expected_processes): found_monitored = list([(pinfo[0], pinfo[1].name, pinfo[1].command_str) for pinfo in list(monitored_processes.items())]) found_processes.sort(key=lambda tuple: tuple[0]) found_monitored.sort(key=lambda tuple: tuple[0]) - print("do_test_commands - found_processes: %s found_monitored: %s" \ - % (found_processes, found_monitored)) + print("found_processes:") + pprint(found_processes) + print("found_monitored:") + pprint(found_monitored) self.assertEqual(found_processes, found_monitored) - print("do_test_commands - expected_processes: %s monitored_processes: %s" \ - % (expected_processes, monitored_processes)) + print("expected_processes:") + pprint(expected_processes) + print("monitored_processes:") + pprint(monitored_processes) self.assert_processes(expected_processes, monitored_processes) def test_change_instance_dist_container_1(self): diff --git a/heron/instance/src/python/network/gateway_looper.py b/heron/instance/src/python/network/gateway_looper.py index b0f84ac98bd..4bd7b443e85 100644 --- a/heron/instance/src/python/network/gateway_looper.py +++ b/heron/instance/src/python/network/gateway_looper.py @@ -66,7 +66,7 @@ def do_wait(self): self.poll(timeout=0.0) def wake_up(self): - os.write(self.pipe_w, "\n") + os.write(self.pipe_w, b"\n") Log.debug("Wake up called") def on_exit(self): diff --git a/heron/instance/src/python/network/protocol.py b/heron/instance/src/python/network/protocol.py index 728421ce1e2..2e356920264 100644 --- a/heron/instance/src/python/network/protocol.py +++ b/heron/instance/src/python/network/protocol.py @@ -71,7 +71,7 @@ def decode_packet(packet): len_typename = HeronProtocol.unpack_int(data[:4]) data = data[4:] - typename = data[:len_typename] + typename = data[:len_typename].decode() data = data[len_typename:] reqid = REQID.unpack(data[:REQID.REQID_SIZE]) @@ -87,8 +87,8 @@ def decode_packet(packet): class OutgoingPacket: """Wrapper class for outgoing packet""" def __init__(self, raw_data): - self.raw = str(raw_data) - self.to_send = str(raw_data) + self.raw = bytes(raw_data) + self.to_send = bytes(raw_data) def __len__(self): return len(self.raw) @@ -101,7 +101,7 @@ def create_packet(reqid, message): :param message: protocol buffer object """ assert message.IsInitialized() - packet = '' + packet = b'' # calculate the totla size of the packet incl. header typename = message.DESCRIPTOR.full_name @@ -114,7 +114,7 @@ def create_packet(reqid, message): # next write the type string packet += HeronProtocol.pack_int(len(typename)) - packet += typename + packet += typename.encode() # reqid packet += reqid.pack() @@ -141,8 +141,8 @@ class IncomingPacket: """Helper class for incoming packet""" def __init__(self): """Initializes IncomingPacket object""" - self.header = '' - self.data = '' + self.header = b'' + self.data = b'' self.is_header_read = False self.is_complete = False # for debugging identification purposes diff --git a/heron/instance/tests/python/network/mock_generator_client.py b/heron/instance/tests/python/network/mock_generator_client.py index d9bf527bed0..dad89ceeda5 100644 --- a/heron/instance/tests/python/network/mock_generator_client.py +++ b/heron/instance/tests/python/network/mock_generator_client.py @@ -102,7 +102,7 @@ class MockDispatcher: """ PARTIAL_DATA_SIZE = 4 def __init__(self): - self.to_be_received = "" + self.to_be_received = b"" self.eagain_test = False self.fatal_error_test = False diff --git a/heron/instance/tests/python/network/protocol_unittest.py b/heron/instance/tests/python/network/protocol_unittest.py index f408795ba1e..d42ddb82c5a 100644 --- a/heron/instance/tests/python/network/protocol_unittest.py +++ b/heron/instance/tests/python/network/protocol_unittest.py @@ -73,7 +73,7 @@ def test_read(self): pkt.read(header_dispatcher) self.assertTrue(pkt.is_header_read) self.assertFalse(pkt.is_complete) - self.assertEqual(pkt.data, "") + self.assertEqual(pkt.data, b"") # an incomplete data packet is prepared partial_data_dispatcher = mock_generator.MockDispatcher() diff --git a/heron/statemgrs/src/python/BUILD b/heron/statemgrs/src/python/BUILD index e2421df0953..6c0a382f2d3 100644 --- a/heron/statemgrs/src/python/BUILD +++ b/heron/statemgrs/src/python/BUILD @@ -5,7 +5,7 @@ pex_library( srcs = glob(["**/*.py"]), reqs = [ "PyYAML==3.13", - "kazoo==1.3.1", + "kazoo==2.7.0", "zope.interface==4.0.5", ], deps = [ diff --git a/heron/tools/tracker/src/python/query_operators.py b/heron/tools/tracker/src/python/query_operators.py index 8102b3b54b7..dd460c5aca9 100644 --- a/heron/tools/tracker/src/python/query_operators.py +++ b/heron/tools/tracker/src/python/query_operators.py @@ -54,17 +54,17 @@ def floorTimestamps(self, start, end, timeline): """ floor timestamp """ ret = {} for timestamp, value in list(timeline.items()): - ts = timestamp / 60 * 60 + ts = timestamp // 60 * 60 if start <= ts <= end: ret[ts] = value return ret def setDefault(self, constant, start, end): """ set default time """ - starttime = start / 60 * 60 + starttime = start // 60 * 60 if starttime < start: starttime += 60 - endtime = end / 60 * 60 + endtime = end // 60 * 60 while starttime <= endtime: # STREAMCOMP-1559 # Second check is a work around, because the response from tmaster diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh index b0cc412ee66..25986a5abfc 100755 --- a/scripts/travis/build.sh +++ b/scripts/travis/build.sh @@ -37,7 +37,7 @@ fi # verify that eggs have not been added to the repo set +e -EGGS=`find . -name "*.egg"` +#EGGS=`find . -name "*.egg"` set -e if [ "$EGGS" ]; then echo 'ERROR: The following eggs were found in the repo, '\ @@ -49,7 +49,7 @@ fi # verify that wheels have not been added to the repo set +e -WHEELS=`find . -name "*.whl"` +#WHEELS=`find . -name "*.whl"` set -e if [ "$WHEELS" ]; then echo 'ERROR: The following wheels were found in the repo, '\ From 72445322c37a5f2078f43dd6fb518cf7dc7343b2 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Tue, 19 May 2020 02:22:48 +0100 Subject: [PATCH 08/22] Try fixing build time issue in travis --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eda03912032..c6eb9fcd02a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,10 @@ before_install: - chmod +x bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh - ./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh --user +install: + - sudo apt-get install python3-pip python3-setuptools + - pip3 install travis-wait-improved + script: - which gcc - gcc --version @@ -44,4 +48,4 @@ script: - python -V - which python3 - python3 -V - - stdbuf -i0 -o0 -e0 travis_wait scripts/travis/ci.sh + - travis-wait-improved --timeout=180m scripts/travis/ci.sh From ae49c65379fa5698a65bdab9669eb36fa73bc040 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Thu, 21 May 2020 17:59:00 +0100 Subject: [PATCH 09/22] Upgrade docker rules --- WORKSPACE | 13 +++++++++---- tools/rules/genproto.bzl | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 69d03bd85a5..72c1b1f73a5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -368,11 +368,12 @@ http_archive( # end helm # for docker image building +DOCKER_RULES_VERSION = "0.14.1" http_archive( name = "io_bazel_rules_docker", - sha256 = "aed1c249d4ec8f703edddf35cbe9dfaca0b5f5ea6e4cd9e83e99f3b0d1136c3d", - strip_prefix = "rules_docker-0.7.0", - urls = ["https://github.com/bazelbuild/rules_docker/archive/v0.7.0.tar.gz"], + sha256 = "dc97fccceacd4c6be14e800b2a00693d5e8d07f69ee187babfd04a80a9f8e250", + strip_prefix = "rules_docker-%s" % DOCKER_RULES_VERSION, + urls = ["https://github.com/bazelbuild/rules_docker/archive/v%s.tar.gz" % DOCKER_RULES_VERSION], ) load( @@ -382,6 +383,10 @@ load( container_repositories() +load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps") + +container_deps() + load( "@io_bazel_rules_docker//container:container.bzl", "container_pull", @@ -392,7 +397,7 @@ container_pull( digest = "sha256:495800e9eb001dfd2fb41d1941155203bb9be06b716b0f8b1b0133eb12ea813c", registry = "index.docker.io", repository = "heron/base", - tag = "0.4.0", + tag = "0.5.0", ) # end docker image building diff --git a/tools/rules/genproto.bzl b/tools/rules/genproto.bzl index 385a1e427aa..9e4d3fcbd30 100644 --- a/tools/rules/genproto.bzl +++ b/tools/rules/genproto.bzl @@ -168,7 +168,7 @@ def proto_library( # hack to work around not having import_prefix from the official proto rules which is needed to sort out imports # and without having https://github.com/protocolbuffers/protobuf/pull/7470 # import common_pb2 as common__pb2 -> import .common_pb2 as common__pb2 - proto_cmd += "\nfind $(@D) -type f -name '*_pb2.py' -exec sed -i.bak -E 's/^(import )([^ .]+_pb2)/from . import \\2/' {} \\;" + proto_cmd += "\nfind $(@D) -ignore_readdir_race -type f -name '*_pb2.py' -exec sed -i.bak -E 's/^(import )([^ .]+_pb2)/from . import \\2/' {} \\;" py_deps = [] proto_deps = [src, protoc] From 2641d1bf91b93e78bad9079f02472ec4ceaf76b9 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Thu, 21 May 2020 21:30:56 +0100 Subject: [PATCH 10/22] Upgrade to python3 in CI --- scripts/applatix/build.sh | 10 +++++----- scripts/applatix/test.sh | 4 ++-- scripts/applatix/testutils.sh | 4 ++-- scripts/shutils/save-logs.py | 13 +++++++------ scripts/travis/build.sh | 12 ++++++------ scripts/travis/test.sh | 8 ++++---- 6 files changed, 26 insertions(+), 25 deletions(-) diff --git a/scripts/applatix/build.sh b/scripts/applatix/build.sh index d7c9900bc7e..f39bb3ea3ff 100755 --- a/scripts/applatix/build.sh +++ b/scripts/applatix/build.sh @@ -79,14 +79,14 @@ echo "Using $PLATFORM platform" # build heron T="heron build" start_timer "$T" -python ${UTILS}/save-logs.py "heron_build.txt" bazel\ +${UTILS}/save-logs.py "heron_build.txt" bazel\ --bazelrc=tools/applatix/bazel.rc build --config=$PLATFORM heron/... end_timer "$T" # run heron unit tests T="heron test non-flaky" start_timer "$T" -python ${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel\ +${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel\ --bazelrc=tools/applatix/bazel.rc test\ --test_summary=detailed --test_output=errors\ --config=$PLATFORM --test_tag_filters=-flaky heron/... @@ -96,7 +96,7 @@ end_timer "$T" # which should be fixed. For now, run them serially T="heron test flaky" start_timer "$T" -python ${UTILS}/save-logs.py "heron_test_flaky.txt" bazel\ +${UTILS}/save-logs.py "heron_test_flaky.txt" bazel\ --bazelrc=tools/applatix/bazel.rc test\ --test_summary=detailed --test_output=errors\ --config=$PLATFORM --test_tag_filters=flaky --jobs=1 heron/... @@ -104,14 +104,14 @@ end_timer "$T" T="heron build binpkgs" start_timer "$T" -python ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ +${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ --bazelrc=tools/applatix/bazel.rc build\ --config=$PLATFORM scripts/packages:binpkgs end_timer "$T" T="heron build testpkgs" start_timer "$T" -python ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ +${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ --bazelrc=tools/applatix/bazel.rc build\ --config=$PLATFORM scripts/packages:testpkgs end_timer "$T" diff --git a/scripts/applatix/test.sh b/scripts/applatix/test.sh index 78c023a9a7d..d0619b9fdf1 100755 --- a/scripts/applatix/test.sh +++ b/scripts/applatix/test.sh @@ -39,13 +39,13 @@ SCALA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/scala-integration-tests.jar # install clients and tools T="heron clients/tools install" start_timer "$T" -python ${UTILS}/save-logs.py "heron_install.txt" ./heron-install.sh --user +${UTILS}/save-logs.py "heron_install.txt" ./heron-install.sh --user end_timer "$T" # install tests T="heron tests install" start_timer "$T" -python ${UTILS}/save-logs.py "heron_tests_install.txt" ./heron-tests-install.sh --user +${UTILS}/save-logs.py "heron_tests_install.txt" ./heron-tests-install.sh --user end_timer "$T" # initialize http-server for integration tests diff --git a/scripts/applatix/testutils.sh b/scripts/applatix/testutils.sh index f526f2f4bc0..988e9841999 100755 --- a/scripts/applatix/testutils.sh +++ b/scripts/applatix/testutils.sh @@ -34,13 +34,13 @@ export PATH=${HOME}/bin:$PATH # install clients and tools T="heron clients/tools install" start_timer "$T" -python ${UTILS}/save-logs.py "heron_install.txt" ./heron-install.sh --user +${UTILS}/save-logs.py "heron_install.txt" ./heron-install.sh --user end_timer "$T" # install tests T="heron tests install" start_timer "$T" -python ${UTILS}/save-logs.py "heron_tests_install.txt" ./heron-tests-install.sh --user +${UTILS}/save-logs.py "heron_tests_install.txt" ./heron-tests-install.sh --user end_timer "$T" print_timer_summary diff --git a/scripts/shutils/save-logs.py b/scripts/shutils/save-logs.py index 5ac5b670f29..2e25d3dae1a 100755 --- a/scripts/shutils/save-logs.py +++ b/scripts/shutils/save-logs.py @@ -58,7 +58,7 @@ def main(file, cmd): sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count)) sys.stdout.flush() nextPrint = datetime.now() + timedelta(seconds=10) - out.write(line) + out.write(line.decode()) line = pout.readline() out.close() errcode = process.wait() @@ -72,9 +72,10 @@ def main(file, cmd): return errcode if __name__ == "__main__": - if sys.argv < 1: - print("Usage: %s [file info]" % sys.argv[0]) - sys.exit(1) - file = sys.argv[1] - cmd = sys.argv[2:] + try: + _, file, *cmd = sys.argv + except ValueError: + print("Usage: %s [file info]" % sys.argv[0]) + sys.exit(1) + main(file, cmd) diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh index 25986a5abfc..f3c5f0b4539 100755 --- a/scripts/travis/build.sh +++ b/scripts/travis/build.sh @@ -77,7 +77,7 @@ echo "Using $PLATFORM platform" # build heron T="heron build" start_timer "$T" -python ${UTILS}/save-logs.py "heron_build.txt" bazel\ +${UTILS}/save-logs.py "heron_build.txt" bazel\ --bazelrc=tools/travis/bazel.rc build --config=$PLATFORM heron/... \ heronpy/... examples/... storm-compatibility-examples/... \ eco-storm-examples/... eco-heron-examples/... contrib/... @@ -86,7 +86,7 @@ end_timer "$T" # run heron unit tests T="heron test non-flaky" start_timer "$T" -python ${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel\ +${UTILS}/save-logs.py "heron_test_non_flaky.txt" bazel\ --bazelrc=tools/travis/bazel.rc test\ --test_summary=detailed --test_output=errors\ --config=$PLATFORM --test_tag_filters=-flaky heron/... \ @@ -98,7 +98,7 @@ end_timer "$T" # which should be fixed. For now, run them serially T="heron test flaky" start_timer "$T" -python ${UTILS}/save-logs.py "heron_test_flaky.txt" bazel\ +${UTILS}/save-logs.py "heron_test_flaky.txt" bazel\ --bazelrc=tools/travis/bazel.rc test\ --test_summary=detailed --test_output=errors\ --config=$PLATFORM --test_tag_filters=flaky --jobs=1 heron/... \ @@ -109,21 +109,21 @@ end_timer "$T" # build packages T="heron build tarpkgs" start_timer "$T" -python ${UTILS}/save-logs.py "heron_build_tarpkgs.txt" bazel\ +${UTILS}/save-logs.py "heron_build_tarpkgs.txt" bazel\ --bazelrc=tools/travis/bazel.rc build\ --config=$PLATFORM scripts/packages:tarpkgs end_timer "$T" T="heron build binpkgs" start_timer "$T" -python ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ +${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ --bazelrc=tools/travis/bazel.rc build\ --config=$PLATFORM scripts/packages:binpkgs end_timer "$T" T="heron build docker images" start_timer "$T" -python ${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ +${UTILS}/save-logs.py "heron_build_binpkgs.txt" bazel\ --bazelrc=tools/travis/bazel.rc build\ --config=$PLATFORM scripts/images:heron.tar end_timer "$T" diff --git a/scripts/travis/test.sh b/scripts/travis/test.sh index 142b1975132..e1915fe3d5a 100755 --- a/scripts/travis/test.sh +++ b/scripts/travis/test.sh @@ -37,19 +37,19 @@ SCALA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/scala-integration-tests.jar # build test related jar T="heron build integration_test" start_timer "$T" -python ${UTILS}/save-logs.py "heron_build_integration_test.txt" bazel --bazelrc=tools/travis/bazel.rc build --config=$PLATFORM integration_test/src/... +${UTILS}/save-logs.py "heron_build_integration_test.txt" bazel --bazelrc=tools/travis/bazel.rc build --config=$PLATFORM integration_test/src/... end_timer "$T" # install heron T="heron install" start_timer "$T" -python ${UTILS}/save-logs.py "heron_install.txt" bazel --bazelrc=tools/travis/bazel.rc run --config=$PLATFORM -- scripts/packages:heron-install.sh --user +${UTILS}/save-logs.py "heron_install.txt" bazel --bazelrc=tools/travis/bazel.rc run --config=$PLATFORM -- scripts/packages:heron-install.sh --user end_timer "$T" # install tests T="heron tests install" start_timer "$T" -python ${UTILS}/save-logs.py "heron_tests_install.txt" bazel --bazelrc=tools/travis/bazel.rc run --config=$PLATFORM -- scripts/packages:heron-tests-install.sh --user +${UTILS}/save-logs.py "heron_tests_install.txt" bazel --bazelrc=tools/travis/bazel.rc run --config=$PLATFORM -- scripts/packages:heron-tests-install.sh --user end_timer "$T" pathadd ${HOME}/bin/ @@ -57,7 +57,7 @@ pathadd ${HOME}/bin/ # run local integration test T="heron integration_test local" start_timer "$T" -python ./bazel-bin/integration_test/src/python/local_test_runner/local-test-runner +./bazel-bin/integration_test/src/python/local_test_runner/local-test-runner end_timer "$T" # initialize http-server for integration tests From f61314539dc2b986f4381cb02b9d422cd093dce4 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Fri, 22 May 2020 00:33:05 +0100 Subject: [PATCH 11/22] Fix python integration tests --- .../src/python/local_test_runner/main.py | 14 +++++++------- .../src/python/local_test_runner/test_kill_bolt.py | 2 +- .../local_test_runner/test_kill_metricsmgr.py | 2 +- .../python/local_test_runner/test_kill_stmgr.py | 2 +- .../test_kill_stmgr_metricsmgr.py | 2 +- .../python/local_test_runner/test_kill_tmaster.py | 2 +- .../src/python/local_test_runner/test_scale_up.py | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/integration_test/src/python/local_test_runner/main.py b/integration_test/src/python/local_test_runner/main.py index cf45651b0d5..0957dec8a1c 100644 --- a/integration_test/src/python/local_test_runner/main.py +++ b/integration_test/src/python/local_test_runner/main.py @@ -34,12 +34,12 @@ from heron.common.src.python.utils import log # import test_kill_bolt -import test_kill_metricsmgr -import test_kill_stmgr -import test_kill_stmgr_metricsmgr -import test_kill_tmaster -import test_scale_up -import test_template +from . import test_kill_metricsmgr +from . import test_kill_stmgr +from . import test_kill_stmgr_metricsmgr +from . import test_kill_tmaster +from . import test_scale_up +from . import test_template TEST_CLASSES = [ test_template.TestTemplate, @@ -110,7 +110,7 @@ def main(): # Read the configuration file from package conf_file = DEFAULT_TEST_CONF_FILE - conf_string = pkgutil.get_data(__name__, conf_file) + conf_string = pkgutil.get_data(__name__, conf_file).decode() decoder = json.JSONDecoder(strict=False) # Convert the conf file to a json format diff --git a/integration_test/src/python/local_test_runner/test_kill_bolt.py b/integration_test/src/python/local_test_runner/test_kill_bolt.py index 92489611512..7a4eb16d610 100644 --- a/integration_test/src/python/local_test_runner/test_kill_bolt.py +++ b/integration_test/src/python/local_test_runner/test_kill_bolt.py @@ -21,7 +21,7 @@ """test_kill_bolt.py""" import logging -import test_template +from . import test_template NON_TMASTER_SHARD = 1 HERON_BOLT = 'identity-bolt_3' diff --git a/integration_test/src/python/local_test_runner/test_kill_metricsmgr.py b/integration_test/src/python/local_test_runner/test_kill_metricsmgr.py index e8cc9292172..71c06f588fe 100644 --- a/integration_test/src/python/local_test_runner/test_kill_metricsmgr.py +++ b/integration_test/src/python/local_test_runner/test_kill_metricsmgr.py @@ -20,7 +20,7 @@ """test_kill_metricsmgr.py""" -import test_template +from . import test_template class TestKillMetricsMgr(test_template.TestTemplate): diff --git a/integration_test/src/python/local_test_runner/test_kill_stmgr.py b/integration_test/src/python/local_test_runner/test_kill_stmgr.py index 0b34e63d14c..f6a7bfa3055 100644 --- a/integration_test/src/python/local_test_runner/test_kill_stmgr.py +++ b/integration_test/src/python/local_test_runner/test_kill_stmgr.py @@ -20,7 +20,7 @@ """test_kill_stmgr.py""" -import test_template +from . import test_template class TestKillStmgr(test_template.TestTemplate): diff --git a/integration_test/src/python/local_test_runner/test_kill_stmgr_metricsmgr.py b/integration_test/src/python/local_test_runner/test_kill_stmgr_metricsmgr.py index ac54d3511a2..34f9409f722 100644 --- a/integration_test/src/python/local_test_runner/test_kill_stmgr_metricsmgr.py +++ b/integration_test/src/python/local_test_runner/test_kill_stmgr_metricsmgr.py @@ -20,7 +20,7 @@ """test_kill_stmgr_metricsmgr.py""" -import test_template +from . import test_template class TestKillStmgrMetricsMgr(test_template.TestTemplate): diff --git a/integration_test/src/python/local_test_runner/test_kill_tmaster.py b/integration_test/src/python/local_test_runner/test_kill_tmaster.py index 573519b6238..0e0b49ae3d3 100644 --- a/integration_test/src/python/local_test_runner/test_kill_tmaster.py +++ b/integration_test/src/python/local_test_runner/test_kill_tmaster.py @@ -22,7 +22,7 @@ """test_kill_tmaster.py""" import logging import subprocess -import test_template +from . import test_template TMASTER_SHARD = 0 diff --git a/integration_test/src/python/local_test_runner/test_scale_up.py b/integration_test/src/python/local_test_runner/test_scale_up.py index 330f62e9836..d79837837f5 100644 --- a/integration_test/src/python/local_test_runner/test_scale_up.py +++ b/integration_test/src/python/local_test_runner/test_scale_up.py @@ -24,7 +24,7 @@ import subprocess from ..common import status -import test_template +from . import test_template class TestScaleUp(test_template.TestTemplate): From 20b19ce08fb6c2deb5783a942b774263629c9ef6 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Fri, 22 May 2020 18:47:54 +0100 Subject: [PATCH 12/22] Fix more bytes vs str errors + update vagrant --- heron/common/src/python/utils/proc.py | 2 +- heron/executor/src/python/heron_executor.py | 6 +- heron/shell/src/python/utils.py | 3 +- heron/tools/admin/src/python/standalone.py | 7 ++ heron/tools/cli/src/python/execute.py | 11 +-- heron/tools/common/src/python/utils/config.py | 19 ++--- heron/tools/tracker/src/python/utils.py | 9 ++- integration_test/src/python/common/status.py | 2 +- .../src/python/local_test_runner/main.py | 2 +- .../python/local_test_runner/test_template.py | 6 +- .../src/python/test_runner/main.py | 2 +- .../src/python/topology_test_runner/main.py | 6 +- scripts/shutils/common.sh | 2 +- scripts/shutils/save-logs.py | 10 ++- scripts/travis/build.sh | 1 - vagrant/README.md | 6 +- vagrant/Vagrantfile | 27 ++++++- vagrant/init.sh | 71 ++++++------------- 18 files changed, 100 insertions(+), 92 deletions(-) diff --git a/heron/common/src/python/utils/proc.py b/heron/common/src/python/utils/proc.py index 706c4bfe01f..897c68189d9 100644 --- a/heron/common/src/python/utils/proc.py +++ b/heron/common/src/python/utils/proc.py @@ -86,7 +86,7 @@ def __init__(self): self.end = False self.strs = [] - def add(self, line): + def add(self, line: bytes): if not line: self.end = True else: diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py index e3756b88272..2bf5d3f9b3e 100755 --- a/heron/executor/src/python/heron_executor.py +++ b/heron/executor/src/python/heron_executor.py @@ -697,7 +697,7 @@ def _get_jvm_version(self): if not self.jvm_version: cmd = [os.path.join(self.heron_java_home, 'bin/java'), '-cp', self.instance_classpath, 'org.apache.heron.instance.util.JvmVersion'] - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) (process_stdout, process_stderr) = process.communicate() if process.returncode != 0: Log.error("Failed to determine JVM version. Exiting. Output of %s: %s", @@ -911,7 +911,7 @@ def _run_process(self, name, cmd): # stderr is redirected to stdout so that it can more easily be logged. stderr has a max buffer # size and can cause the child process to deadlock if it fills up process = subprocess.Popen(cmd.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - env=cmd.env, bufsize=1) + env=cmd.env, text=True, bufsize=1) proc.async_stream_process_stdout(process, stdout_log_fn(name)) except Exception: Log.info("Exception running command %s", cmd) @@ -925,7 +925,7 @@ def _run_blocking_process(self, cmd, is_shell=False): # stderr is redirected to stdout so that it can more easily be logged. stderr has a max buffer # size and can cause the child process to deadlock if it fills up process = subprocess.Popen(cmd.cmd, shell=is_shell, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=cmd.env) + stderr=subprocess.STDOUT, text=True, env=cmd.env) # wait for termination self._wait_process_std_out_err(cmd.cmd, process) diff --git a/heron/shell/src/python/utils.py b/heron/shell/src/python/utils.py index 2d0975d78e8..60c55331389 100644 --- a/heron/shell/src/python/utils.py +++ b/heron/shell/src/python/utils.py @@ -169,7 +169,8 @@ def str_cmd(cmd, cwd, env): Runs the command and returns its stdout and stderr. """ process = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, cwd=cwd, env=env) + stderr=subprocess.PIPE, cwd=cwd, + env=env, text=True) stdout_builder, stderr_builder = proc.async_stdout_stderr_builder(process) process.wait() stdout, stderr = stdout_builder.result(), stderr_builder.result() diff --git a/heron/tools/admin/src/python/standalone.py b/heron/tools/admin/src/python/standalone.py index bfddfbfd358..65523739d44 100644 --- a/heron/tools/admin/src/python/standalone.py +++ b/heron/tools/admin/src/python/standalone.py @@ -423,6 +423,7 @@ def stop_cluster(cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -437,6 +438,7 @@ def stop_cluster(cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -495,6 +497,7 @@ def start_api_server(masters, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -524,6 +527,7 @@ def start_heron_tools(masters, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -611,6 +615,7 @@ def scp_package(package_file, destinations, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pids.append({"pid": pid, "dest": dest}) @@ -652,6 +657,7 @@ def start_master_nodes(masters, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pids.append({"pid": pid, "dest": master}) @@ -686,6 +692,7 @@ def start_slave_nodes(slaves, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pids.append({"pid": pid, "dest": slave}) diff --git a/heron/tools/cli/src/python/execute.py b/heron/tools/cli/src/python/execute.py index 4bca88e634b..86aeaa87f1e 100644 --- a/heron/tools/cli/src/python/execute.py +++ b/heron/tools/cli/src/python/execute.py @@ -22,6 +22,7 @@ import contextlib import os import subprocess +import shlex import tarfile import tempfile import traceback @@ -61,7 +62,7 @@ def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=N java_path = config.get_java_path() if java_path is None: - err_context = "Neither JAVA_BIN or JAVA_HOME are set" + err_context = "Unable to find java command" return SimpleResult(Status.InvocationError, err_context) # Construct the command line for the sub process to run @@ -78,12 +79,12 @@ def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=N heron_env['HERON_OPTIONS'] = opts.get_heron_config() # print the verbose message - Log.debug("Invoking class using command: ``%s''", ' '.join(all_args)) + Log.debug("Invoking class using command: `%s`", ' '.join(shlex.quote(a) for a in all_args)) Log.debug("Heron options: {%s}", str(heron_env["HERON_OPTIONS"])) # invoke the command with subprocess and print error message, if any process = subprocess.Popen(all_args, env=heron_env, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, bufsize=1) + stderr=subprocess.PIPE, text=True, bufsize=1) # stdout message has the information Java program sends back # stderr message has extra information, such as debugging message return ProcessResult(process) @@ -134,7 +135,7 @@ def heron_pex(topology_pex, topology_class_name, args=None): Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS'])) # invoke the command with subprocess and print error message, if any process = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, bufsize=1) + stderr=subprocess.PIPE, text=True, bufsize=1) # pylint: disable=fixme # todo(rli): improve python topology submission workflow return ProcessResult(process) @@ -171,5 +172,5 @@ def heron_cpp(topology_binary, args=None): print('Heron options: {%s}' % str(heron_env['HERON_OPTIONS'])) # invoke the command with subprocess and print error message, if any proc = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, bufsize=1) + stderr=subprocess.PIPE, text=True, bufsize=1) return ProcessResult(proc) diff --git a/heron/tools/common/src/python/utils/config.py b/heron/tools/common/src/python/utils/config.py index 64534dbd138..a6a0c547a15 100644 --- a/heron/tools/common/src/python/utils/config.py +++ b/heron/tools/common/src/python/utils/config.py @@ -25,9 +25,11 @@ import getpass import os import sys +import shutil import subprocess import tarfile import tempfile +from pathlib import Path import yaml from heron.common.src.python.utils.log import Log @@ -112,7 +114,7 @@ def cygpath(x): normalized class path on cygwin ''' command = ['cygpath', '-wp', x] - p = subprocess.Popen(command, stdout=subprocess.PIPE) + p = subprocess.Popen(command, stdout=subprocess.PIPE, text=True) result = p.communicate() output = result[0] lines = output.split("\n") @@ -141,6 +143,9 @@ def get_classpath(jars): ''' return ':'.join(map(normalized_class_path, jars)) +def _get_heron_dir(): + # assuming the tool runs from $HERON_ROOT/bin/ + return normalized_class_path(str(Path(sys.argv[0]).resolve(strict=True).parent.parent)) def get_heron_dir(): """ @@ -155,9 +160,7 @@ def get_heron_dir(): :return: root location of the .pex file """ - go_above_dirs = 9 - path = "/".join(os.path.realpath(__file__).split('/')[:-go_above_dirs]) - return normalized_class_path(path) + return _get_heron_dir() def get_zipped_heron_dir(): """ @@ -174,9 +177,7 @@ def get_zipped_heron_dir(): :return: root location of the .pex file. """ - go_above_dirs = 7 - path = "/".join(os.path.realpath(__file__).split('/')[:-go_above_dirs]) - return normalized_class_path(path) + return _get_heron_dir() ################################################################################ # Get the root of heron dir and various sub directories depending on platform @@ -432,8 +433,8 @@ def get_java_path(): java_home = os.environ.get("JAVA_HOME") if java_home: return os.path.join(java_home, BIN_DIR, "java") - # this could use shutil.which("java") when python2 support is dropped - return None + + return shutil.which("java") def check_release_file_exists(): diff --git a/heron/tools/tracker/src/python/utils.py b/heron/tools/tracker/src/python/utils.py index a6f912975f2..d2443f403b6 100644 --- a/heron/tools/tracker/src/python/utils.py +++ b/heron/tools/tracker/src/python/utils.py @@ -27,8 +27,10 @@ import string import sys import subprocess +from pathlib import Path import yaml + # directories for heron tools distribution BIN_DIR = "bin" CONF_DIR = "conf" @@ -116,7 +118,7 @@ def cygpath(x): :return: the path in windows """ command = ['cygpath', '-wp', x] - p = subprocess.Popen(command, stdout=subprocess.PIPE) + p = subprocess.Popen(command, stdout=subprocess.PIPE, text=True) output, _ = p.communicate() lines = output.split("\n") return lines[0] @@ -138,8 +140,9 @@ def get_heron_tracker_dir(): This will extract heron tracker directory from .pex file. :return: root location for heron-tools. """ - path = "/".join(os.path.realpath(__file__).split('/')[:-8]) - return normalized_class_path(path) + # assuming the tracker runs from $HERON_ROOT/bin/heron-tracker + root = Path(sys.argv[0]).resolve(strict=True).parent.parent + return normalized_class_path(str(root)) def get_heron_tracker_bin_dir(): """ diff --git a/integration_test/src/python/common/status.py b/integration_test/src/python/common/status.py index 423cc2b0cdf..a1f3277e9fd 100644 --- a/integration_test/src/python/common/status.py +++ b/integration_test/src/python/common/status.py @@ -27,7 +27,7 @@ class TestFailure(Exception): def __init__(self, message, error=None): Exception.__init__(self, message, error) if error: - logging.error("%s :: %s", message, traceback.format_exc(error)) + logging.error("%s :: %s", message, error, exc_info=True) else: logging.error(message) diff --git a/integration_test/src/python/local_test_runner/main.py b/integration_test/src/python/local_test_runner/main.py index 0957dec8a1c..9c4d9c8b57d 100644 --- a/integration_test/src/python/local_test_runner/main.py +++ b/integration_test/src/python/local_test_runner/main.py @@ -82,7 +82,7 @@ def run_tests(test_classes, args): failures += [testname] except Exception as e: - logging.error("Exception thrown while running tests: %s", str(e)) + logging.error("Exception thrown while running tests: %s", str(e), exc_info=True) finally: tracker_process.kill() diff --git a/integration_test/src/python/local_test_runner/test_template.py b/integration_test/src/python/local_test_runner/test_template.py index f793875d53c..cc2e6c0c141 100644 --- a/integration_test/src/python/local_test_runner/test_template.py +++ b/integration_test/src/python/local_test_runner/test_template.py @@ -92,9 +92,9 @@ def run_test(self): return result except status.TestFailure as e: - raise e + raise except Exception as e: - raise status.TestFailure("Exception thrown during test", e) + raise status.TestFailure("Exception thrown during test", e) from e finally: if topology_submitted: self.cleanup_test() @@ -329,7 +329,7 @@ def _get_processes(): """ # pylint: disable=fixme # TODO: if the submit fails before we get here (e.g., Topology already exists), this hangs - processes = subprocess.check_output(['ps', '-o', 'pid,args']) + processes = subprocess.check_output(['ps', '-o', 'pid,args'], text=True) processes = processes.split('\n') processes = processes[1:] # remove first line, which is name of columns process_list = [] diff --git a/integration_test/src/python/test_runner/main.py b/integration_test/src/python/test_runner/main.py index 351c1399184..c348ced8cf4 100644 --- a/integration_test/src/python/test_runner/main.py +++ b/integration_test/src/python/test_runner/main.py @@ -425,7 +425,7 @@ def main(): log.configure(level=logging.DEBUG) conf_file = DEFAULT_TEST_CONF_FILE # Read the configuration file from package - conf_string = pkgutil.get_data(__name__, conf_file) + conf_string = pkgutil.get_data(__name__, conf_file).decode() decoder = json.JSONDecoder(strict=False) # Convert the conf file to a json format conf = decoder.decode(conf_string) diff --git a/integration_test/src/python/topology_test_runner/main.py b/integration_test/src/python/topology_test_runner/main.py index 8e5ec456140..a83ed894560 100644 --- a/integration_test/src/python/topology_test_runner/main.py +++ b/integration_test/src/python/topology_test_runner/main.py @@ -240,7 +240,7 @@ def fetch_results(self): raise status.TestFailure("Expected results file %s does not exist" % self.file_path) else: with open(self.file_path, "r") as expected_result_file: - return expected_result_file.read().rstrip() + return expected_result_file.read().decode().rstrip() except Exception as e: raise status.TestFailure("Failed to read expected result file %s" % self.file_path, e) @@ -316,10 +316,10 @@ def fetch_from_server(self, server_host_port, topology_name, data_name, path): logging.info("Fetching %s for topology %s, retry count: %d", data_name, topology_name, i) response = self.get_http_response(server_host_port, path) if response.status == 200: - return response.read() + return response.read().decode() elif i != RETRY_ATTEMPTS: logging.info("Fetching %s failed with status: %s; reason: %s; body: %s", - data_name, response.status, response.reason, response.read()) + data_name, response.status, response.reason, response.read().decode()) time.sleep(RETRY_INTERVAL) raise status.TestFailure("Failed to fetch %s after %d attempts" % (data_name, RETRY_ATTEMPTS)) diff --git a/scripts/shutils/common.sh b/scripts/shutils/common.sh index 6413777119e..3c4857bca7b 100755 --- a/scripts/shutils/common.sh +++ b/scripts/shutils/common.sh @@ -92,7 +92,7 @@ function print_timer_summary { # Discover the platform that we are running on function discover_platform { - discover=`python -mplatform` + discover="${PLATFORM-$(python3 -mplatform)}" if [[ $discover =~ ^.*centos.*$ ]]; then echo "centos" elif [[ $discover =~ ^.*Ubuntu.*$ ]]; then diff --git a/scripts/shutils/save-logs.py b/scripts/shutils/save-logs.py index 2e25d3dae1a..cc50c9910c7 100755 --- a/scripts/shutils/save-logs.py +++ b/scripts/shutils/save-logs.py @@ -21,6 +21,7 @@ import os import subprocess import sys +import shlex from datetime import datetime, timedelta @@ -35,12 +36,15 @@ def tail(filename, n): n -= 1 if n == -1: break - return fm[i + 1 if i else 0:].splitlines() + return fm[i + 1 if i else 0:].decode().splitlines() finally: fm.close() +def shell_cmd(cmd): + return " ".join(shlex.quote(c) for c in cmd) + def main(file, cmd): - print("%s writing to: %s" % (cmd, file)) + print("%s > %s" % (shell_cmd(cmd),file)) with open(file, "w") as out: count = 0 process = subprocess.Popen(cmd, @@ -64,7 +68,7 @@ def main(file, cmd): errcode = process.wait() diff = datetime.now() - start sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count)) - print("\n %s finished with errcode: %d" % (cmd, errcode)) + print("\n `%s` finished with errcode: %d" % (shell_cmd(cmd), errcode)) if errcode != 0: lines = tail(file, 1000) print('\n'.join(lines)) diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh index f3c5f0b4539..55ff069dabc 100755 --- a/scripts/travis/build.sh +++ b/scripts/travis/build.sh @@ -19,7 +19,6 @@ # Script to kick off the travis CI build. We want the build to fail-fast if any # of the below commands fail so we need to chain them in this script. # - set -e DIR=`dirname $0` diff --git a/vagrant/README.md b/vagrant/README.md index 08a6c48abc9..662362e3d6d 100644 --- a/vagrant/README.md +++ b/vagrant/README.md @@ -16,6 +16,8 @@ specific language governing permissions and limitations under the License. --> -Vagrant VM to build and run Heron +Vagrant VM for CI and debugging ================================= -vagrant up \ No newline at end of file +Running `vagrant up master` will bring up an environment similar to the one used by Travis for CI. If the build fails, it can be inspected by entering the machine with `vagrant ssh master`. When you're down with the VM, you can clean up with `vagrant destroy -f`. + +The advantage of this is you don't need to worry about the potential environment pollution, and others can reproduce the results from other platforms. diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 7161cf7fa4b..df5cef9d30d 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -15,7 +15,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -SLAVES=1 +SLAVES=0 NET_PREFIX="192.168.25." NODES={"master" => NET_PREFIX + "5"} @@ -29,12 +29,33 @@ File.open('.vagrant/hosts', 'w') do |file| end Vagrant.configure(2) do |config| - config.vm.box = "ubuntu/trusty64" + config.vm.box = "ubuntu/focal64" + # config.vm.box = "bento/ubuntu-20.04" config.vm.synced_folder "../", "/vagrant" + config.vm.boot_timeout = 600 config.vm.define "master" do |master| master.vm.provider "virtualbox" do |v| - v.memory = 4096 + host = RbConfig::CONFIG['host_os'] + mem_ratio = 1.0/2 + cpu_exec_cap = 75 + # Give VM 1/2 system memory & access to all cpu cores on the host + if host =~ /darwin/ + cpus = `sysctl -n hw.ncpu`.to_i + # sysctl returns Bytes and we need to convert to MB + mem = `sysctl -n hw.memsize`.to_i / 1024^2 * mem_ratio + elsif host =~ /linux/ + cpus = `nproc`.to_i + # meminfo shows KB and we need to convert to MB + mem = `grep 'MemTotal' /proc/meminfo | sed -E -e 's/MemTotal:\\s+//' -e 's/ kB//'`.to_i / 1024 * mem_ratio + else # Windows folks + cpus = `wmic cpu get NumberOfCores`.split("\n")[2].to_i + mem = `wmic OS get TotalVisibleMemorySize`.split("\n")[2].to_i / 1024 * mem_ratio + end + mem = mem.to_i + v.customize ["modifyvm", :id, "--cpuexecutioncap", cpu_exec_cap] + v.memory = mem + v.cpus = cpus end master.vm.hostname = "master" diff --git a/vagrant/init.sh b/vagrant/init.sh index c5186b2d3be..5f4f0925bd7 100644 --- a/vagrant/init.sh +++ b/vagrant/init.sh @@ -1,4 +1,5 @@ -#!/bin/bash -ex +#!/bin/bash +set -o errexit -o nounset -o pipefail # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -25,7 +26,7 @@ install_mesos() { ip=$(cat /etc/hosts | grep `hostname` | grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}") echo $ip > "/etc/mesos-$mode/ip" - if [ $mode == "master" ]; then + if [ "$mode" == "master" ]; then ln -s /lib/init/upstart-job /etc/init.d/mesos-master service mesos-master start else @@ -33,6 +34,7 @@ install_mesos() { fi ln -s /lib/init/upstart-job /etc/init.d/mesos-slave + echo 'docker,mesos' > /etc/mesos-slave/containerizers service mesos-slave start } @@ -41,69 +43,40 @@ install_marathon() { service marathon start } -install_docker() { - apt-get install -qy lxc-docker - echo 'docker,mesos' > /etc/mesos-slave/containerizers - service mesos-slave restart -} - -install_jdk8() { - apt-get install -y software-properties-common python-software-properties - add-apt-repository -y ppa:webupd8team/java - apt-get -y update - /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections - apt-get -y install oracle-java8-installer oracle-java8-set-default vim wget screen git -} - bazelVersion=3.0.0 bazel_install() { - install_jdk8 - apt-get install -y g++ automake cmake gcc-4.8 g++-4.8 zlib1g-dev zip pkg-config wget libssl-dev + apt-get install -y automake cmake gcc g++ zlib1g-dev zip pkg-config wget libssl-dev libunwind-dev mkdir -p /opt/bazel pushd /opt/bazel - pushd /tmp - wget http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz - tar xvfz libunwind-1.1.tar.gz - cd libunwind-1.1 && ./configure --prefix=/usr && make install - popd wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/${bazelVersion}/bazel-${bazelVersion}-installer-linux-x86_64.sh chmod +x /tmp/bazel.sh - /tmp/bazel.sh --user + /tmp/bazel.sh popd } build_heron() { - pushd /tmp - wget http://ftpmirror.gnu.org/libtool/libtool-2.4.6.tar.gz - tar xf libtool* - cd libtool-2.4.6 - sh configure --prefix /usr/local - make install - popd pushd /vagrant - export CC=gcc-4.8 - export CXX=g++-4.8 - export PATH=/sbin:$PATH - ~/bin/bazel clean + bazel clean ./bazel_configure.py - ~/bin/bazel --bazelrc=tools/travis/bazel.rc build --config=ubuntu heron/... + bazel --bazelrc=tools/travis/bazel.rc build --config=ubuntu heron/... popd } -if [[ $1 != "master" && $1 != "slave" ]]; then +if [[ "$1" != "master" && $1 != "slave" ]]; then echo "Usage: $0 master|slave" exit 1 fi -mode=$1 +mode="$1" cd /vagrant/vagrant # name resolution cp .vagrant/hosts /etc/hosts +# XXX: not needed? # ssh key key=".vagrant/ssh_key.pub" -if [ -f $key ]; then +if [ -f "$key" ]; then cat $key >> /home/vagrant/.ssh/authorized_keys fi @@ -118,27 +91,23 @@ if [ -f ".vagrant/apt-proxy" ]; then echo "Acquire::http::Proxy \"$apt_proxy\";" > /etc/apt/apt.conf.d/90-apt-proxy.conf fi +:<<'REMOVED' # add mesosphere repo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E56151BF DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') CODENAME=$(lsb_release -cs) -echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | tee /etc/apt/sources.list.d/mesosphere.list - -# add docker repo -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 -echo "deb http://get.docker.com/ubuntu docker main" > /etc/apt/sources.list.d/docker.list +echo "deb http://repos.mesosphere.io/${DISTRO} cosmic main" | tee /etc/apt/sources.list.d/mesosphere.list +REMOVED apt-get -qy update # install deps -apt-get install -qy vim zip mc curl wget openjdk-7-jre scala git python-setuptools python-dev +apt-get install -qy vim zip mc curl wget openjdk-11-jdk scala git python3-setuptools python3-dev libtool-bin libcppunit-dev python-is-python3 -install_mesos $mode +# install_mesos $mode if [ $mode == "master" ]; then - install_marathon + # install_marathon bazel_install - build_heron + # switch to non-root so bazel cache can be reused when SSHing in + # su --login vagrant /vagrant/scripts/travis/ci.sh fi - -install_docker - From 61df76df634cf864b2faf52d562ea09f5a7bee69 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Wed, 27 May 2020 00:37:46 +0100 Subject: [PATCH 13/22] Update Travis to Python3.7 + fix Vagrant on mac --- .travis.yml | 6 ++++-- vagrant/Vagrantfile | 6 +++--- vagrant/local-ci.sh | 31 +++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100755 vagrant/local-ci.sh diff --git a/.travis.yml b/.travis.yml index c6eb9fcd02a..2e5c798a4ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ addons: packages: - libtool-bin - libcppunit-dev - - python3 + - python3.7 - pkg-config - python-dev - python3-wheel @@ -37,9 +37,11 @@ before_install: install: - sudo apt-get install python3-pip python3-setuptools - - pip3 install travis-wait-improved + - pip3 install travis-wait-improved virtualenv script: + - python3 -m virtualenv --python=python3.7 /tmp/venv + - source /tmp/venv/bin/activate - which gcc - gcc --version - which g++ diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index df5cef9d30d..ad39ee48deb 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -29,8 +29,8 @@ File.open('.vagrant/hosts', 'w') do |file| end Vagrant.configure(2) do |config| - config.vm.box = "ubuntu/focal64" - # config.vm.box = "bento/ubuntu-20.04" + # config.vm.box = "ubuntu/focal64" + config.vm.box = "bento/ubuntu-20.04" config.vm.synced_folder "../", "/vagrant" config.vm.boot_timeout = 600 @@ -43,7 +43,7 @@ Vagrant.configure(2) do |config| if host =~ /darwin/ cpus = `sysctl -n hw.ncpu`.to_i # sysctl returns Bytes and we need to convert to MB - mem = `sysctl -n hw.memsize`.to_i / 1024^2 * mem_ratio + mem = `sysctl -n hw.memsize`.to_f / 1024**2 * mem_ratio elsif host =~ /linux/ cpus = `nproc`.to_i # meminfo shows KB and we need to convert to MB diff --git a/vagrant/local-ci.sh b/vagrant/local-ci.sh new file mode 100755 index 00000000000..482b6b27fb2 --- /dev/null +++ b/vagrant/local-ci.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +:<<'DOC' +This script is for running tests in a local VM, similar to the environment used in the CI pipeline. If the targent script fails, a shell will be opened up within the VM. + +To only run integration tests: + ./local-ci.sh test + +To run the full ci pipeline: + ./local-ci.sh ci + +The VM does not report the platform in python as expected, so PLATFORM=Ubuntu is needed to work around that for the CI script's platform discovery. + +DOC + +set -o errexit -o nounset -o pipefail +HERE="$(cd "$(dirname "$0")" && pwd -P)" + +cd "$HERE" + +state="$(vagrant status master --machine-readable | grep master,state, | cut -d, -f4)" +if [ "$state" != "running" ]; then + vagrant resume master +fi + + +# allows you to do `$0 test` to run only integration tests +script="${1-ci}" +env="PLATFORM=Ubuntu JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/" +# run the CI, if it fails drop into a shell +vagrant ssh --command "cd /vagrant && $env ./scripts/travis/$script.sh" \ + || vagrant ssh --command "cd /vagrant && $env exec bash" From 52f672dd3dd5f455dbc9f293469da79884deb514 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Thu, 28 May 2020 00:06:36 +0100 Subject: [PATCH 14/22] Reduce requirement to python3.6 + py3 fixes * use universal_newline in popen instead of text in Popen for py3.6 * fix bytes/str issues in deserialisation * fix file open modes * use set instead of sets.Set * fix __import__(level) default --- .travis.yml | 8 +++----- heron/common/src/python/pex_loader.py | 2 +- heron/executor/src/python/heron_executor.py | 7 ++++--- heron/shell/src/python/utils.py | 4 ++-- heron/statemgrs/src/python/filestatemanager.py | 16 ++++++++-------- heron/tools/admin/src/python/standalone.py | 14 +++++++------- heron/tools/cli/src/python/execute.py | 6 +++--- heron/tools/common/src/python/utils/config.py | 2 +- heron/tools/tracker/src/python/javaobj.py | 8 ++++---- heron/tools/tracker/src/python/utils.py | 2 +- heronpy/api/serializer.py | 8 ++++---- heronpy/streamlet/builder.py | 4 +--- .../python/local_test_runner/test_template.py | 6 +++--- integration_test/src/python/test_runner/main.py | 10 +++++----- .../src/python/topology_test_runner/main.py | 10 +++++----- vagrant/local-ci.sh | 4 ++-- 16 files changed, 54 insertions(+), 57 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2e5c798a4ca..caa404669b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,9 @@ addons: packages: - libtool-bin - libcppunit-dev - - python3.7 + - python3 - pkg-config - - python-dev + - python3-dev - python3-wheel - wget - zip @@ -37,11 +37,9 @@ before_install: install: - sudo apt-get install python3-pip python3-setuptools - - pip3 install travis-wait-improved virtualenv + - pip3 install travis-wait-improved script: - - python3 -m virtualenv --python=python3.7 /tmp/venv - - source /tmp/venv/bin/activate - which gcc - gcc --version - which g++ diff --git a/heron/common/src/python/pex_loader.py b/heron/common/src/python/pex_loader.py index bfcc0da2f72..565dbdf32ed 100644 --- a/heron/common/src/python/pex_loader.py +++ b/heron/common/src/python/pex_loader.py @@ -124,6 +124,6 @@ def import_and_get_class(path_to_pex, python_class_name): except: Log.error("Could not resolve class %s with special handling" % python_class_name) - mod = __import__(from_path, fromlist=[import_name], level=-1) + mod = __import__(from_path, fromlist=[import_name], level=0) Log.debug("Imported module: %s" % str(mod)) return getattr(mod, import_name) diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py index 2bf5d3f9b3e..cb063aab7f6 100755 --- a/heron/executor/src/python/heron_executor.py +++ b/heron/executor/src/python/heron_executor.py @@ -697,7 +697,8 @@ def _get_jvm_version(self): if not self.jvm_version: cmd = [os.path.join(self.heron_java_home, 'bin/java'), '-cp', self.instance_classpath, 'org.apache.heron.instance.util.JvmVersion'] - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) (process_stdout, process_stderr) = process.communicate() if process.returncode != 0: Log.error("Failed to determine JVM version. Exiting. Output of %s: %s", @@ -911,7 +912,7 @@ def _run_process(self, name, cmd): # stderr is redirected to stdout so that it can more easily be logged. stderr has a max buffer # size and can cause the child process to deadlock if it fills up process = subprocess.Popen(cmd.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - env=cmd.env, text=True, bufsize=1) + env=cmd.env, universal_newlines=True, bufsize=1) proc.async_stream_process_stdout(process, stdout_log_fn(name)) except Exception: Log.info("Exception running command %s", cmd) @@ -925,7 +926,7 @@ def _run_blocking_process(self, cmd, is_shell=False): # stderr is redirected to stdout so that it can more easily be logged. stderr has a max buffer # size and can cause the child process to deadlock if it fills up process = subprocess.Popen(cmd.cmd, shell=is_shell, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, text=True, env=cmd.env) + stderr=subprocess.STDOUT, universal_newlines=True, env=cmd.env) # wait for termination self._wait_process_std_out_err(cmd.cmd, process) diff --git a/heron/shell/src/python/utils.py b/heron/shell/src/python/utils.py index 60c55331389..e5e1cd8f5a1 100644 --- a/heron/shell/src/python/utils.py +++ b/heron/shell/src/python/utils.py @@ -170,7 +170,7 @@ def str_cmd(cmd, cwd, env): """ process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, - env=env, text=True) + env=env, universal_newlines=True) stdout_builder, stderr_builder = proc.async_stdout_stderr_builder(process) process.wait() stdout, stderr = stdout_builder.result(), stderr_builder.result() @@ -197,7 +197,7 @@ def get_container_id(instance_id): def get_asset(asset_name): ''' get assset ''' - return pkgutil.get_data("heron.shell", os.path.join("assets", asset_name)) + return pkgutil.get_data("heron.shell", os.path.join("assets", asset_name)).decode() def check_path(path): """ diff --git a/heron/statemgrs/src/python/filestatemanager.py b/heron/statemgrs/src/python/filestatemanager.py index 8ef1f82d51f..3ece9071a87 100644 --- a/heron/statemgrs/src/python/filestatemanager.py +++ b/heron/statemgrs/src/python/filestatemanager.py @@ -101,9 +101,9 @@ def trigger_watches_based_on_files(watchers, path, directory, ProtoClass): """ for topology, callbacks in list(watchers.items()): file_path = os.path.join(path, topology) - data = "" + data = b"" if os.path.exists(file_path): - with open(os.path.join(path, topology)) as f: + with open(os.path.join(path, topology), "rb") as f: data = f.read() if topology not in directory or data != directory[topology]: proto_object = ProtoClass() @@ -175,7 +175,7 @@ def get_topology(self, topologyName, callback=None): self.topology_watchers[topologyName].append(callback) else: topology_path = self.get_topology_path(topologyName) - with open(topology_path) as f: + with open(topology_path, "rb") as f: data = f.read() topology = Topology() topology.ParseFromString(data) @@ -198,7 +198,7 @@ def get_packing_plan(self, topologyName, callback=None): self.packing_plan_watchers[topologyName].append(callback) else: packing_plan_path = self.get_packing_plan_path(topologyName) - with open(packing_plan_path) as f: + with open(packing_plan_path, "rb") as f: data = f.read() packing_plan = PackingPlan() packing_plan.ParseFromString(data) @@ -211,7 +211,7 @@ def get_pplan(self, topologyName, callback=None): self.pplan_watchers[topologyName].append(callback) else: pplan_path = self.get_pplan_path(topologyName) - with open(pplan_path) as f: + with open(pplan_path, "rb") as f: data = f.read() pplan = PhysicalPlan() pplan.ParseFromString(data) @@ -236,7 +236,7 @@ def get_execution_state(self, topologyName, callback=None): self.execution_state_watchers[topologyName].append(callback) else: execution_state_path = self.get_execution_state_path(topologyName) - with open(execution_state_path) as f: + with open(execution_state_path, "rb") as f: data = f.read() executionState = ExecutionState() executionState.ParseFromString(data) @@ -261,7 +261,7 @@ def get_tmaster(self, topologyName, callback=None): self.tmaster_watchers[topologyName].append(callback) else: tmaster_path = self.get_tmaster_path(topologyName) - with open(tmaster_path) as f: + with open(tmaster_path, "rb") as f: data = f.read() tmaster = TMasterLocation() tmaster.ParseFromString(data) @@ -276,7 +276,7 @@ def get_scheduler_location(self, topologyName, callback=None): self.scheduler_location_watchers[topologyName].append(callback) else: scheduler_location_path = self.get_scheduler_location_path(topologyName) - with open(scheduler_location_path) as f: + with open(scheduler_location_path, "rb") as f: data = f.read() scheduler_location = SchedulerLocation() scheduler_location.ParseFromString(data) diff --git a/heron/tools/admin/src/python/standalone.py b/heron/tools/admin/src/python/standalone.py index 65523739d44..656f7205385 100644 --- a/heron/tools/admin/src/python/standalone.py +++ b/heron/tools/admin/src/python/standalone.py @@ -423,7 +423,7 @@ def stop_cluster(cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -438,7 +438,7 @@ def stop_cluster(cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -497,7 +497,7 @@ def start_api_server(masters, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -527,7 +527,7 @@ def start_heron_tools(masters, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -615,7 +615,7 @@ def scp_package(package_file, destinations, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pids.append({"pid": pid, "dest": dest}) @@ -657,7 +657,7 @@ def start_master_nodes(masters, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pids.append({"pid": pid, "dest": master}) @@ -692,7 +692,7 @@ def start_slave_nodes(slaves, cl_args): Log.debug(cmd) pid = subprocess.Popen(cmd, shell=True, - text=True, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pids.append({"pid": pid, "dest": slave}) diff --git a/heron/tools/cli/src/python/execute.py b/heron/tools/cli/src/python/execute.py index 86aeaa87f1e..bff33b5f19f 100644 --- a/heron/tools/cli/src/python/execute.py +++ b/heron/tools/cli/src/python/execute.py @@ -84,7 +84,7 @@ def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=N # invoke the command with subprocess and print error message, if any process = subprocess.Popen(all_args, env=heron_env, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, text=True, bufsize=1) + stderr=subprocess.PIPE, universal_newlines=True, bufsize=1) # stdout message has the information Java program sends back # stderr message has extra information, such as debugging message return ProcessResult(process) @@ -135,7 +135,7 @@ def heron_pex(topology_pex, topology_class_name, args=None): Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS'])) # invoke the command with subprocess and print error message, if any process = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, text=True, bufsize=1) + stderr=subprocess.PIPE, universal_newlines=True, bufsize=1) # pylint: disable=fixme # todo(rli): improve python topology submission workflow return ProcessResult(process) @@ -172,5 +172,5 @@ def heron_cpp(topology_binary, args=None): print('Heron options: {%s}' % str(heron_env['HERON_OPTIONS'])) # invoke the command with subprocess and print error message, if any proc = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, text=True, bufsize=1) + stderr=subprocess.PIPE, universal_newlines=True, bufsize=1) return ProcessResult(proc) diff --git a/heron/tools/common/src/python/utils/config.py b/heron/tools/common/src/python/utils/config.py index a6a0c547a15..577ee9930da 100644 --- a/heron/tools/common/src/python/utils/config.py +++ b/heron/tools/common/src/python/utils/config.py @@ -114,7 +114,7 @@ def cygpath(x): normalized class path on cygwin ''' command = ['cygpath', '-wp', x] - p = subprocess.Popen(command, stdout=subprocess.PIPE, text=True) + p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) result = p.communicate() output = result[0] lines = output.split("\n") diff --git a/heron/tools/tracker/src/python/javaobj.py b/heron/tools/tracker/src/python/javaobj.py index 1587d05d0a2..2bf1b230997 100644 --- a/heron/tools/tracker/src/python/javaobj.py +++ b/heron/tools/tracker/src/python/javaobj.py @@ -28,8 +28,8 @@ See: http://download.oracle.com/javase/6/docs/platform/serialization/spec/protocol.html """ +import io import struct -import six from heron.common.src.python.utils.log import Log @@ -54,12 +54,12 @@ def load(file_object): # pylint: disable=undefined-variable -def loads(value): +def loads(value: bytes): """ Deserializes Java objects and primitive data serialized by ObjectOutputStream from a string. """ - f = six.StringIO(value) + f = io.BytesIO(value) marshaller = JavaObjectUnmarshaller(f) marshaller.add_transformer(DefaultObjectTransformer()) return marshaller.readObject() @@ -558,7 +558,7 @@ def __init__(self, stream=None): # pylint: disable=attribute-defined-outside-init def dump(self, obj): self.object_obj = obj - self.object_stream = six.StringIO() + self.object_stream = io.BytesIO() self._writeStreamHeader() self.writeObject(obj) return self.object_stream.getvalue() diff --git a/heron/tools/tracker/src/python/utils.py b/heron/tools/tracker/src/python/utils.py index d2443f403b6..c78f1b4a0b6 100644 --- a/heron/tools/tracker/src/python/utils.py +++ b/heron/tools/tracker/src/python/utils.py @@ -118,7 +118,7 @@ def cygpath(x): :return: the path in windows """ command = ['cygpath', '-wp', x] - p = subprocess.Popen(command, stdout=subprocess.PIPE, text=True) + p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) output, _ = p.communicate() lines = output.split("\n") return lines[0] diff --git a/heronpy/api/serializer.py b/heronpy/api/serializer.py index 4f9759e567e..dc5545622d9 100644 --- a/heronpy/api/serializer.py +++ b/heronpy/api/serializer.py @@ -35,7 +35,7 @@ def initialize(self, config): """Initializes the serializer""" @abstractmethod - def serialize(self, obj): + def serialize(self, obj) -> bytes: """Serialize an object :param obj: The object to be serialized @@ -43,7 +43,7 @@ def serialize(self, obj): """ @abstractmethod - def deserialize(self, input_str): + def deserialize(self, input_str: bytes): """Deserialize an object :param input_str: Serialized object as byte string @@ -55,10 +55,10 @@ class PythonSerializer(IHeronSerializer): def initialize(self, config=None): pass - def serialize(self, obj): + def serialize(self, obj) -> bytes: return cloudpickle.dumps(obj) - def deserialize(self, input_str): + def deserialize(self, input_str: bytes): return pickle.loads(input_str) default_serializer = PythonSerializer() diff --git a/heronpy/streamlet/builder.py b/heronpy/streamlet/builder.py index 050bc06dee8..b46c0dd4223 100644 --- a/heronpy/streamlet/builder.py +++ b/heronpy/streamlet/builder.py @@ -20,8 +20,6 @@ '''builder.py: module for creating streamlets''' -import sets - from heronpy.streamlet.generator import Generator from heronpy.streamlet.impl.supplierspout import SupplierStreamlet from heronpy.streamlet.impl.generatorspout import GeneratorStreamlet @@ -50,7 +48,7 @@ def new_source(self, source): # pylint: disable=protected-access def build(self, bldr): """Builds the topology and returns the builder""" - stage_names = sets.Set() + stage_names = set() for source in self._sources: source._build(bldr, stage_names) for source in self._sources: diff --git a/integration_test/src/python/local_test_runner/test_template.py b/integration_test/src/python/local_test_runner/test_template.py index cc2e6c0c141..33219de77d8 100644 --- a/integration_test/src/python/local_test_runner/test_template.py +++ b/integration_test/src/python/local_test_runner/test_template.py @@ -215,7 +215,7 @@ def get_pid(self, process_name, heron_working_directory): try: with open(process_pid_file, 'r') as f: pid = f.readline() - return pid + return int(pid) except Exception: logging.error("Unable to open file %s", process_pid_file) return -1 @@ -230,7 +230,7 @@ def kill_process(self, process_number): logging.info("Killing process number %s", process_number) try: - os.kill(int(process_number), signal.SIGTERM) + os.kill(process_number, signal.SIGTERM) except OSError as ex: if "No such process" in str(ex): # killing a non-existing process condsidered as success logging.info(str(ex)) @@ -329,7 +329,7 @@ def _get_processes(): """ # pylint: disable=fixme # TODO: if the submit fails before we get here (e.g., Topology already exists), this hangs - processes = subprocess.check_output(['ps', '-o', 'pid,args'], text=True) + processes = subprocess.check_output(['ps', '-o', 'pid,args'], universal_newlines=True) processes = processes.split('\n') processes = processes[1:] # remove first line, which is name of columns process_list = [] diff --git a/integration_test/src/python/test_runner/main.py b/integration_test/src/python/test_runner/main.py index c348ced8cf4..fb44a10eeac 100644 --- a/integration_test/src/python/test_runner/main.py +++ b/integration_test/src/python/test_runner/main.py @@ -46,7 +46,7 @@ class FileBasedExpectedResultsHandler: def __init__(self, file_path): self.file_path = file_path - def fetch_results(self): + def fetch_results(self) -> str: # Read expected result from the expected result file try: if not os.path.exists(self.file_path): @@ -64,7 +64,7 @@ def __init__(self, server_host_port, topology_name, task_count): self.task_count = task_count # pylint: disable=unnecessary-lambda - def fetch_results(self): + def fetch_results(self) -> str: try: result = [] decoder = json.JSONDecoder(strict=False) @@ -92,7 +92,7 @@ def __init__(self, server_host_port, topology_name): self.server_host_port = server_host_port self.topology_name = topology_name - def fetch_results(self): + def fetch_results(self) -> str: try: return fetch_from_server(self.server_host_port, self.topology_name, 'results', '/results/%s' % self.topology_name) @@ -221,13 +221,13 @@ def update_state_server(http_server_host_port, topology_name, key, value): response = connection.getresponse() return response.status == 200 -def fetch_from_server(server_host_port, topology_name, data_name, path): +def fetch_from_server(server_host_port, topology_name, data_name, path) -> str: ''' Make a http get request to fetch actual results from http server ''' for i in range(0, RETRY_ATTEMPTS): logging.info("Fetching %s for topology %s, retry count: %d", data_name, topology_name, i) response = get_http_response(server_host_port, path) if response.status == 200: - return response.read() + return response.read().decode() elif i != RETRY_ATTEMPTS: logging.info("Fetching %s failed with status: %s; reason: %s; body: %s", data_name, response.status, response.reason, response.read()) diff --git a/integration_test/src/python/topology_test_runner/main.py b/integration_test/src/python/topology_test_runner/main.py index a83ed894560..12d86851767 100644 --- a/integration_test/src/python/topology_test_runner/main.py +++ b/integration_test/src/python/topology_test_runner/main.py @@ -231,7 +231,7 @@ class FileBasedExpectedResultsHandler: def __init__(self, file_path): self.file_path = file_path - def fetch_results(self): + def fetch_results(self) -> str: """ Read expected result from the expected result file """ @@ -240,7 +240,7 @@ def fetch_results(self): raise status.TestFailure("Expected results file %s does not exist" % self.file_path) else: with open(self.file_path, "r") as expected_result_file: - return expected_result_file.read().decode().rstrip() + return expected_result_file.read().rstrip() except Exception as e: raise status.TestFailure("Failed to read expected result file %s" % self.file_path, e) @@ -303,14 +303,14 @@ def __init__(self, server_host_port, topology_name): self.server_host_port = server_host_port self.topology_name = topology_name - def fetch_results(self): + def fetch_results(self) -> str: try: return self.fetch_from_server(self.server_host_port, self.topology_name, 'instance_state', '/stateResults/%s' % self.topology_name) except Exception as e: raise status.TestFailure("Fetching instance state failed for %s topology" % self.topology_name, e) - def fetch_from_server(self, server_host_port, topology_name, data_name, path): + def fetch_from_server(self, server_host_port, topology_name, data_name, path) -> str: ''' Make a http get request to fetch actual results from http server ''' for i in range(0, RETRY_ATTEMPTS): logging.info("Fetching %s for topology %s, retry count: %d", data_name, topology_name, i) @@ -632,7 +632,7 @@ def main(): log.configure(level=logging.DEBUG) conf_file = DEFAULT_TEST_CONF_FILE # Read the configuration file from package - conf_string = pkgutil.get_data(__name__, conf_file) + conf_string = pkgutil.get_data(__name__, conf_file).decode() decoder = json.JSONDecoder(strict=False) # Convert the conf file to a json format conf = decoder.decode(conf_string) diff --git a/vagrant/local-ci.sh b/vagrant/local-ci.sh index 482b6b27fb2..ef4a5e55888 100755 --- a/vagrant/local-ci.sh +++ b/vagrant/local-ci.sh @@ -27,5 +27,5 @@ fi script="${1-ci}" env="PLATFORM=Ubuntu JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/" # run the CI, if it fails drop into a shell -vagrant ssh --command "cd /vagrant && $env ./scripts/travis/$script.sh" \ - || vagrant ssh --command "cd /vagrant && $env exec bash" +vagrant ssh master --command "cd /vagrant && $env ./scripts/travis/$script.sh" \ + || vagrant ssh master --command "cd /vagrant && $env exec bash" From 1f80d5a20fbd146597de7dab3c94816a602239fd Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Thu, 28 May 2020 13:10:37 +0100 Subject: [PATCH 15/22] Update cloudpickle --- heronpy/api/cloudpickle.py | 1939 +++++++++++++++++++++++------------- 1 file changed, 1233 insertions(+), 706 deletions(-) diff --git a/heronpy/api/cloudpickle.py b/heronpy/api/cloudpickle.py index 88e94c5ab22..27a218036a0 100644 --- a/heronpy/api/cloudpickle.py +++ b/heronpy/api/cloudpickle.py @@ -6,22 +6,22 @@ -Deal with other non-serializable objects It does not include an unpickler, as standard python unpickling suffices. This module was extracted from the `cloud` package, developed by `PiCloud, Inc. -`_. +`_. Copyright (c) 2012, Regents of the University of California. -Copyright (c) 2009 `PiCloud, Inc. `_. +Copyright (c) 2009 `PiCloud, Inc. `_. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the University of California, Berkeley nor the - names of its contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California, Berkeley nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -34,785 +34,1312 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ +# pylint: skip-file from __future__ import print_function -import operator -import opcode -import os +import abc +import builtins +import dis import io +import itertools +import logging +import opcode +import operator import pickle +import platform import struct import sys import types -from functools import partial -import itertools -import dis -import traceback import weakref - -# pylint: disable-all - -if sys.version < '3': - from pickle import Pickler # pylint: disable=ungrouped-imports - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - PY3 = False -else: - types.ClassType = type - from pickle import _Pickler as Pickler # pylint: disable=ungrouped-imports - from io import BytesIO as StringIO # pylint: disable=ungrouped-imports - PY3 = True - -#relevant opcodes -STORE_GLOBAL = opcode.opmap['STORE_GLOBAL'] -DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL'] -LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL'] -GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL) -HAVE_ARGUMENT = dis.HAVE_ARGUMENT -EXTENDED_ARG = dis.EXTENDED_ARG - - -def islambda(func): - return getattr(func, '__name__') == '' - - -_BUILTIN_TYPE_NAMES = {} -for k1, v1 in list(types.__dict__.items()): - if type(v1) is type: # pylint: disable=unidiomatic-typecheck - _BUILTIN_TYPE_NAMES[v1] = k1 - - -def _builtin_type(name): - return getattr(types, name) - - -if sys.version_info < (3, 4): - def _walk_global_ops(code): - """ - Yield (opcode, argument number) tuples for all - global-referencing instructions in *code*. +import uuid +import threading +import typing +from enum import Enum + +from typing import Generic, Union, Tuple, Callable +from pickle import _Pickler as Pickler +from pickle import _getattribute +from io import BytesIO +from importlib._bootstrap import _find_spec + +try: # pragma: no branch + import typing_extensions as _typing_extensions + from typing_extensions import Literal, Final +except ImportError: + _typing_extensions = Literal = Final = None + +if sys.version_info >= (3, 5, 3): + from typing import ClassVar +else: # pragma: no cover + ClassVar = None + + +# cloudpickle is meant for inter process communication: we expect all +# communicating processes to run the same Python version hence we favor +# communication speed over compatibility: +DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL + +# Track the provenance of reconstructed dynamic classes to make it possible to +# recontruct instances from the matching singleton class definition when +# appropriate and preserve the usual "isinstance" semantics of Python objects. +_DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary() +_DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary() +_DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock() + +PYPY = platform.python_implementation() == "PyPy" + +builtin_code_type = None +if PYPY: + # builtin-code objects only exist in pypy + builtin_code_type = type(float.__new__.__code__) + +_extract_code_globals_cache = weakref.WeakKeyDictionary() + + +def _get_or_create_tracker_id(class_def): + with _DYNAMIC_CLASS_TRACKER_LOCK: + class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def) + if class_tracker_id is None: + class_tracker_id = uuid.uuid4().hex + _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id + _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def + return class_tracker_id + + +def _lookup_class_or_track(class_tracker_id, class_def): + if class_tracker_id is not None: + with _DYNAMIC_CLASS_TRACKER_LOCK: + class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault( + class_tracker_id, class_def) + _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id + return class_def + + +def _whichmodule(obj, name): + """Find the module an object belongs to. + This function differs from ``pickle.whichmodule`` in two ways: + - it does not mangle the cases where obj's module is __main__ and obj was + not found in any module. + - Errors arising during module introspection are ignored, as those errors + are considered unwanted side effects. """ - code = getattr(code, 'co_code', b'') - if not PY3: - code = list(map(ord, code)) - - n = len(code) - i = 0 - extended_arg = 0 - while i < n: - op = code[i] - i += 1 - if op >= HAVE_ARGUMENT: - oparg = code[i] + code[i + 1] * 256 + extended_arg - extended_arg = 0 - i += 2 - if op == EXTENDED_ARG: - extended_arg = oparg * 65536 - if op in GLOBAL_OPS: - yield op, oparg + if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa + # Workaround bug in old Python versions: prior to Python 3.7, + # T.__module__ would always be set to "typing" even when the TypeVar T + # would be defined in a different module. + # + # For such older Python versions, we ignore the __module__ attribute of + # TypeVar instances and instead exhaustively lookup those instances in + # all currently imported modules. + module_name = None + else: + module_name = getattr(obj, '__module__', None) + + if module_name is not None: + return module_name + # Protect the iteration by using a copy of sys.modules against dynamic + # modules that trigger imports of other modules upon calls to getattr or + # other threads importing at the same time. + for module_name, module in sys.modules.copy().items(): + # Some modules such as coverage can inject non-module objects inside + # sys.modules + if ( + module_name == '__main__' or + module is None or + not isinstance(module, types.ModuleType) + ): + continue + try: + if _getattribute(module, name)[0] is obj: + return module_name + except Exception: + pass + return None + + +def _is_importable_by_name(obj, name=None): + """Determine if obj can be pickled as attribute of a file-backed module""" + return _lookup_module_and_qualname(obj, name=name) is not None + + +def _lookup_module_and_qualname(obj, name=None): + if name is None: + name = getattr(obj, '__qualname__', None) + if name is None: # pragma: no cover + # This used to be needed for Python 2.7 support but is probably not + # needed anymore. However we keep the __name__ introspection in case + # users of cloudpickle rely on this old behavior for unknown reasons. + name = getattr(obj, '__name__', None) + + module_name = _whichmodule(obj, name) + + if module_name is None: + # In this case, obj.__module__ is None AND obj was not found in any + # imported module. obj is thus treated as dynamic. + return None + + if module_name == "__main__": + return None + + module = sys.modules.get(module_name, None) + if module is None: + # The main reason why obj's module would not be imported is that this + # module has been dynamically created, using for example + # types.ModuleType. The other possibility is that module was removed + # from sys.modules after obj was created/imported. But this case is not + # supported, as the standard pickle does not support it either. + return None + + # module has been added to sys.modules, but it can still be dynamic. + if _is_dynamic(module): + return None -else: - def _walk_global_ops(code): - """ - Yield (opcode, argument number) tuples for all - global-referencing instructions in *code*. - """ - for instr in dis.get_instructions(code): # pylint: disable=no-member - op = instr.opcode - if op in GLOBAL_OPS: - yield op, instr.arg - - -class CloudPickler(Pickler): # pylint: disable=too-many-public-methods - """ - CloudPickler class - """ - dispatch = Pickler.dispatch.copy() - - def __init__(self, filen, protocol=None): - Pickler.__init__(self, filen, protocol) - # set of modules to unpickle - self.modules = set() - # map ids to dictionary. used to ensure that functions can share global env - self.globals_ref = {} - - def dump(self, obj): - self.inject_addons() try: - return Pickler.dump(self, obj) - except RuntimeError as e: - if 'recursion' in e.args[0]: - msg = """Could not pickle object as excessively deep recursion required.""" - raise pickle.PicklingError(msg) - except pickle.PickleError: - raise - except Exception as e: - print_exec(sys.stderr) - raise pickle.PicklingError(str(e)) - - def save_memoryview(self, obj): - """Fallback to save_string""" - Pickler.save_string(self, str(obj)) - - def save_buffer(self, obj): - """Fallback to save_string""" - Pickler.save_string(self, str(obj)) - if PY3: - dispatch[memoryview] = save_memoryview - else: - dispatch[buffer] = save_buffer - - def save_unsupported(self, obj): # pylint: disable=no-self-use - raise pickle.PicklingError("Cannot pickle objects of type %s" % type(obj)) - dispatch[types.GeneratorType] = save_unsupported - - # itertools objects do not pickle! - for v in list(itertools.__dict__.values()): - if type(v) is type: # pylint: disable=unidiomatic-typecheck - dispatch[v] = save_unsupported - - def save_module(self, obj): - """ - Save a module as an import - """ - self.modules.add(obj) - self.save_reduce(subimport, (obj.__name__,), obj=obj) - dispatch[types.ModuleType] = save_module + obj2, parent = _getattribute(module, name) + except AttributeError: + # obj was not found inside the module it points to + return None + if obj2 is not obj: + return None + return module, name - def save_codeobject(self, obj): - """ - Save a code object - """ - if PY3: - args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) - else: - args = ( - obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, - obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars - ) - self.save_reduce(types.CodeType, args, obj=obj) - dispatch[types.CodeType] = save_codeobject - - def save_function(self, obj, name=None): - """ Registered with the dispatch to handle all function types. - Determines what kind of function obj is (e.g. lambda, defined at - interactive prompt, etc) and handles the pickling appropriately. - """ - write = self.write - if name is None: - name = obj.__name__ - try: - # whichmodule() could fail, see - # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling - modname = pickle.whichmodule(obj, name) - except Exception: - modname = None - # print('which gives %s %s %s' % (modname, obj, name)) - try: - themodule = sys.modules[modname] - except KeyError: - # eval'd items such as namedtuple give invalid items for their function __module__ - modname = '__main__' - - if modname == '__main__': - themodule = None - - if themodule: - self.modules.add(themodule) - if getattr(themodule, name, None) is obj: - return self.save_global(obj, name) - - # if func is lambda, def'ed at prompt, is in main, or is nested, then - # we'll pickle the actual function object rather than simply saving a - # reference (as is done in default pickler), via save_function_tuple. - if islambda(obj) or obj.__code__.co_filename == '' or themodule is None: - #print("save global", islambda(obj), obj.__code__.co_filename, modname, themodule) - self.save_function_tuple(obj) - return - else: - # func is nested - klass = getattr(themodule, name, None) - if klass is None or klass is not obj: - self.save_function_tuple(obj) - return - - if obj.__dict__: - # essentially save_reduce, but workaround needed to avoid recursion - self.save(_restore_attr) - write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') - self.memoize(obj) - self.save(obj.__dict__) - write(pickle.TUPLE + pickle.REDUCE) - else: - write(pickle.GLOBAL + modname + '\n' + name + '\n') - self.memoize(obj) - dispatch[types.FunctionType] = save_function - - def save_function_tuple(self, func): - """ Pickles an actual func object. - A func comprises: code, globals, defaults, closure, and dict. We - extract and save these, injecting reducing functions at certain points - to recreate the func object. Keep in mind that some of these pieces - can contain a ref to the func itself. Thus, a naive save on these - pieces could trigger an infinite loop of save's. To get around that, - we first create a skeleton func object using just the code (this is - safe, since this won't contain a ref to the func), and memoize it as - soon as it's created. The other stuff can then be filled in later. - """ - save = self.save - write = self.write - - code, f_globals, defaults, closure, dct, base_globals = self.extract_func_data(func) - - save(_fill_function) # skeleton function updater - write(pickle.MARK) # beginning of tuple that _fill_function expects - - # create a skeleton function object and memoize it - save(_make_skel_func) - save((code, closure, base_globals)) - write(pickle.REDUCE) - self.memoize(func) - - # save the rest of the func data needed by _fill_function - save(f_globals) - save(defaults) - save(dct) - save(func.__module__) - write(pickle.TUPLE) - write(pickle.REDUCE) # applies _fill_function on the tuple - - _extract_code_globals_cache = ( - weakref.WeakKeyDictionary() - if sys.version_info >= (2, 7) and not hasattr(sys, "pypy_version_info") - else {} - ) - - @classmethod - def extract_code_globals(cls, co): +def _extract_code_globals(co): """ Find all globals names read or written to by codeblock co """ - out_names = cls._extract_code_globals_cache.get(co) + out_names = _extract_code_globals_cache.get(co) if out_names is None: - try: names = co.co_names - except AttributeError: - # PyPy "builtin-code" object - out_names = set() - else: - out_names = set(names[oparg] - for op, oparg in _walk_global_ops(co)) - - # see if nested function have any global refs + out_names = {names[oparg] for _, oparg in _walk_global_ops(co)} + + # Declaring a function inside another one using the "def ..." + # syntax generates a constant code object corresonding to the one + # of the nested function's As the nested function may itself need + # global variables, we need to introspect its code, extract its + # globals, (look for code object in it's co_consts attribute..) and + # add the result to code_globals if co.co_consts: - for const in co.co_consts: - if type(const) is types.CodeType: # pylint: disable=unidiomatic-typecheck - out_names |= cls.extract_code_globals(const) + for const in co.co_consts: + if isinstance(const, types.CodeType): + out_names |= _extract_code_globals(const) - cls._extract_code_globals_cache[co] = out_names + _extract_code_globals_cache[co] = out_names return out_names - def extract_func_data(self, func): + +def _find_imported_submodules(code, top_level_dependencies): """ - Turn the function into a tuple of data necessary to recreate it: - code, globals, defaults, closure, dict + Find currently imported submodules used by a function. + Submodules used by a function need to be detected and referenced for the + function to work correctly at depickling time. Because submodules can be + referenced as attribute of their parent package (``package.submodule``), we + need a special introspection technique that does not rely on GLOBAL-related + opcodes to find references of them in a code object. + Example: + ``` + import concurrent.futures + import cloudpickle + def func(): + x = concurrent.futures.ThreadPoolExecutor + if __name__ == '__main__': + cloudpickle.dumps(func) + ``` + The globals extracted by cloudpickle in the function's state include the + concurrent package, but not its submodule (here, concurrent.futures), which + is the module used by func. Find_imported_submodules will detect the usage + of concurrent.futures. Saving this module alongside with func will ensure + that calling func once depickled does not fail due to concurrent.futures + not being imported """ - code = func.__code__ - - # extract all global ref's - func_global_refs = self.extract_code_globals(code) - # process all variables referenced by global environment - f_globals = {} - for var in func_global_refs: - if var in func.__globals__: - f_globals[var] = func.__globals__[var] - - # defaults requires no processing - defaults = func.__defaults__ - - # process closure - closure = [c.cell_contents for c in func.__closure__] if func.__closure__ else [] + subimports = [] + # check if any known dependency is an imported package + for x in top_level_dependencies: + if (isinstance(x, types.ModuleType) and + hasattr(x, '__package__') and x.__package__): + # check if the package has any currently loaded sub-imports + prefix = x.__name__ + '.' + # A concurrent thread could mutate sys.modules, + # make sure we iterate over a copy to avoid exceptions + for name in list(sys.modules): + # Older versions of pytest will add a "None" module to + # sys.modules. + if name is not None and name.startswith(prefix): + # check whether the function can address the sub-module + tokens = set(name[len(prefix):].split('.')) + if not tokens - set(code.co_names): + subimports.append(sys.modules[name]) + return subimports + + +def cell_set(cell, value): + """Set the value of a closure cell. + The point of this function is to set the cell_contents attribute of a cell + after its creation. This operation is necessary in case the cell contains a + reference to the function the cell belongs to, as when calling the + function's constructor + ``f = types.FunctionType(code, globals, name, argdefs, closure)``, + closure will not be able to contain the yet-to-be-created f. + In Python3.7, cell_contents is writeable, so setting the contents of a cell + can be done simply using + >>> cell.cell_contents = value + In earlier Python3 versions, the cell_contents attribute of a cell is read + only, but this limitation can be worked around by leveraging the Python 3 + ``nonlocal`` keyword. + In Python2 however, this attribute is read only, and there is no + ``nonlocal`` keyword. For this reason, we need to come up with more + complicated hacks to set this attribute. + The chosen approach is to create a function with a STORE_DEREF opcode, + which sets the content of a closure variable. Typically: + >>> def inner(value): + ... lambda: cell # the lambda makes cell a closure + ... cell = value # cell is a closure, so this triggers a STORE_DEREF + (Note that in Python2, A STORE_DEREF can never be triggered from an inner + function. The function g for example here + >>> def f(var): + ... def g(): + ... var += 1 + ... return g + will not modify the closure variable ``var```inplace, but instead try to + load a local variable var and increment it. As g does not assign the local + variable ``var`` any initial value, calling f(1)() will fail at runtime.) + Our objective is to set the value of a given cell ``cell``. So we need to + somewhat reference our ``cell`` object into the ``inner`` function so that + this object (and not the smoke cell of the lambda function) gets affected + by the STORE_DEREF operation. + In inner, ``cell`` is referenced as a cell variable (an enclosing variable + that is referenced by the inner function). If we create a new function + cell_set with the exact same code as ``inner``, but with ``cell`` marked as + a free variable instead, the STORE_DEREF will be applied on its closure - + ``cell``, which we can specify explicitly during construction! The new + cell_set variable thus actually sets the contents of a specified cell! + Note: we do not make use of the ``nonlocal`` keyword to set the contents of + a cell in early python3 versions to limit possible syntax errors in case + test and checker libraries decide to parse the whole file. + """ - # save the dict - dct = func.__dict__ + if sys.version_info[:2] >= (3, 7): # pragma: no branch + cell.cell_contents = value + else: + _cell_set = types.FunctionType( + _cell_set_template_code, {}, '_cell_set', (), (cell,),) + _cell_set(value) + + +def _make_cell_set_template_code(): + def _cell_set_factory(value): + lambda: cell + cell = value + + co = _cell_set_factory.__code__ + + _cell_set_template_code = types.CodeType( + co.co_argcount, + co.co_kwonlyargcount, # Python 3 only argument + co.co_nlocals, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + co.co_cellvars, # co_freevars is initialized with co_cellvars + (), # co_cellvars is made empty + ) + return _cell_set_template_code + + +if sys.version_info[:2] < (3, 7): + _cell_set_template_code = _make_cell_set_template_code() + +# relevant opcodes +STORE_GLOBAL = opcode.opmap['STORE_GLOBAL'] +DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL'] +LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL'] +GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL) +HAVE_ARGUMENT = dis.HAVE_ARGUMENT +EXTENDED_ARG = dis.EXTENDED_ARG - base_globals = self.globals_ref.get(id(func.__globals__), {}) - self.globals_ref[id(func.__globals__)] = base_globals - return (code, f_globals, defaults, closure, dct, base_globals) +_BUILTIN_TYPE_NAMES = {} +for k, v in types.__dict__.items(): + if type(v) is type: + _BUILTIN_TYPE_NAMES[v] = k - def save_builtin_function(self, obj): - if obj.__module__ is "__builtin__": - return self.save_global(obj) - return self.save_function(obj) - dispatch[types.BuiltinFunctionType] = save_builtin_function - def save_global(self, obj, name=None, pack=struct.pack): # pylint: disable=too-many-branches - if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": - if obj in _BUILTIN_TYPE_NAMES: - return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) +def _builtin_type(name): + if name == "ClassType": # pragma: no cover + # Backward compat to load pickle files generated with cloudpickle + # < 1.3 even if loading pickle files from older versions is not + # officially supported. + return type + return getattr(types, name) - if name is None: - name = obj.__name__ - - modname = getattr(obj, "__module__", None) - if modname is None: - try: - # whichmodule() could fail, see - # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling - modname = pickle.whichmodule(obj, name) - except Exception: - modname = '__main__' - - if modname == '__main__': - themodule = None - else: - __import__(modname) - themodule = sys.modules[modname] - self.modules.add(themodule) - - if hasattr(themodule, name) and getattr(themodule, name) is obj: - return Pickler.save_global(self, obj, name) - - typ = type(obj) - if typ is not obj and isinstance(obj, (type, types.ClassType)): - d = dict(obj.__dict__) # copy dict proxy to a dict - if not isinstance(d.get('__dict__', None), property): - # don't extract dict that are properties - d.pop('__dict__', None) - d.pop('__weakref__', None) - - # hack as __new__ is stored differently in the __dict__ - new_override = d.get('__new__', None) - if new_override: - d['__new__'] = obj.__new__ - - # workaround for namedtuple (hijacked by PySpark) - if getattr(obj, '_is_namedtuple_', False): - self.save_reduce(_load_namedtuple, (obj.__name__, obj._fields)) - return - - self.save(_load_class) - self.save_reduce(typ, (obj.__name__, obj.__bases__, {"__doc__": obj.__doc__}), obj=obj) - d.pop('__doc__', None) - # handle property and staticmethod - dd = {} - for k, v in list(d.items()): - if isinstance(v, property): - k = ('property', k) - v = (v.fget, v.fset, v.fdel, v.__doc__) - elif isinstance(v, staticmethod) and hasattr(v, '__func__'): - k = ('staticmethod', k) - v = v.__func__ - elif isinstance(v, classmethod) and hasattr(v, '__func__'): - k = ('classmethod', k) - v = v.__func__ - dd[k] = v - self.save(dd) - self.write(pickle.TUPLE2) - self.write(pickle.REDUCE) - else: - raise pickle.PicklingError("Can't pickle %r" % obj) +def _walk_global_ops(code): + """ + Yield (opcode, argument number) tuples for all + global-referencing instructions in *code*. + """ + for instr in dis.get_instructions(code): + op = instr.opcode + if op in GLOBAL_OPS: + yield op, instr.arg - dispatch[type] = save_global - dispatch[types.ClassType] = save_global - def save_instancemethod(self, obj): - # Memoization rarely is ever useful due to python bounding - if PY3: - self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj) - else: - self.save_reduce( - types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__), - obj=obj) - dispatch[types.MethodType] = save_instancemethod - - def save_inst(self, obj): - """Inner logic to save instance. Based off pickle.save_inst - Supports __transient__""" - cls = obj.__class__ - - memo = self.memo - write = self.write - save = self.save - - if hasattr(obj, '__getinitargs__'): - args = obj.__getinitargs__() - len(args) # assert it's a sequence - pickle._keep_alive(args, memo) # pylint: disable=protected-access +def _extract_class_dict(cls): + """Retrieve a copy of the dict of a class without the inherited methods""" + clsdict = dict(cls.__dict__) # copy dict proxy to a dict + if len(cls.__bases__) == 1: + inherited_dict = cls.__bases__[0].__dict__ else: - args = () + inherited_dict = {} + for base in reversed(cls.__bases__): + inherited_dict.update(base.__dict__) + to_remove = [] + for name, value in clsdict.items(): + try: + base_value = inherited_dict[name] + if value is base_value: + to_remove.append(name) + except KeyError: + pass + for name in to_remove: + clsdict.pop(name) + return clsdict + + +if sys.version_info[:2] < (3, 7): # pragma: no branch + def _is_parametrized_type_hint(obj): + # This is very cheap but might generate false positives. + # general typing Constructs + is_typing = getattr(obj, '__origin__', None) is not None + + # typing_extensions.Literal + is_litteral = getattr(obj, '__values__', None) is not None + + # typing_extensions.Final + is_final = getattr(obj, '__type__', None) is not None + + # typing.Union/Tuple for old Python 3.5 + is_union = getattr(obj, '__union_params__', None) is not None + is_tuple = getattr(obj, '__tuple_params__', None) is not None + is_callable = ( + getattr(obj, '__result__', None) is not None and + getattr(obj, '__args__', None) is not None + ) + return any((is_typing, is_litteral, is_final, is_union, is_tuple, + is_callable)) + + def _create_parametrized_type_hint(origin, args): + return origin[args] + + +class CloudPickler(Pickler): + + dispatch = Pickler.dispatch.copy() + + def __init__(self, file, protocol=None): + if protocol is None: + protocol = DEFAULT_PROTOCOL + Pickler.__init__(self, file, protocol=protocol) + # map ids to dictionary. used to ensure that functions can share global env + self.globals_ref = {} + + def dump(self, obj): + self.inject_addons() + try: + return Pickler.dump(self, obj) + except RuntimeError as e: + if 'recursion' in e.args[0]: + msg = """Could not pickle object as excessively deep recursion required.""" + raise pickle.PicklingError(msg) + else: + raise + + def save_typevar(self, obj): + self.save_reduce(*_typevar_reduce(obj), obj=obj) + + dispatch[typing.TypeVar] = save_typevar + + def save_memoryview(self, obj): + self.save(obj.tobytes()) - write(pickle.MARK) + dispatch[memoryview] = save_memoryview - if self.bin: - save(cls) - for arg in args: - save(arg) - write(pickle.OBJ) - else: - for arg in args: - save(arg) - write(pickle.INST + cls.__module__ + '\n' + cls.__name__ + '\n') + def save_module(self, obj): + """ + Save a module as an import + """ + if _is_dynamic(obj): + obj.__dict__.pop('__builtins__', None) + self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)), + obj=obj) + else: + self.save_reduce(subimport, (obj.__name__,), obj=obj) + + dispatch[types.ModuleType] = save_module + + def save_codeobject(self, obj): + """ + Save a code object + """ + if hasattr(obj, "co_posonlyargcount"): # pragma: no branch + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) + else: + args = ( + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars + ) + self.save_reduce(types.CodeType, args, obj=obj) + + dispatch[types.CodeType] = save_codeobject + + def save_function(self, obj, name=None): + """ Registered with the dispatch to handle all function types. + Determines what kind of function obj is (e.g. lambda, defined at + interactive prompt, etc) and handles the pickling appropriately. + """ + if _is_importable_by_name(obj, name=name): + return Pickler.save_global(self, obj, name=name) + elif PYPY and isinstance(obj.__code__, builtin_code_type): + return self.save_pypy_builtin_func(obj) + else: + return self.save_function_tuple(obj) + + dispatch[types.FunctionType] = save_function + + def save_pypy_builtin_func(self, obj): + """Save pypy equivalent of builtin functions. + PyPy does not have the concept of builtin-functions. Instead, + builtin-functions are simple function instances, but with a + builtin-code attribute. + Most of the time, builtin functions should be pickled by attribute. But + PyPy has flaky support for __qualname__, so some builtin functions such + as float.__new__ will be classified as dynamic. For this reason only, + we created this special routine. Because builtin-functions are not + expected to have closure or globals, there is no additional hack + (compared the one already implemented in pickle) to protect ourselves + from reference cycles. A simple (reconstructor, newargs, obj.__dict__) + tuple is save_reduced. + Note also that PyPy improved their support for __qualname__ in v3.6, so + this routing should be removed when cloudpickle supports only PyPy 3.6 + and later. + """ + rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, + obj.__defaults__, obj.__closure__), + obj.__dict__) + self.save_reduce(*rv, obj=obj) + + def _save_dynamic_enum(self, obj, clsdict): + """Special handling for dynamic Enum subclasses + Use a dedicated Enum constructor (inspired by EnumMeta.__call__) as the + EnumMeta metaclass has complex initialization that makes the Enum + subclasses hold references to their own instances. + """ + members = dict((e.name, e.value) for e in obj) + + self.save_reduce( + _make_skeleton_enum, + (obj.__bases__, obj.__name__, obj.__qualname__, + members, obj.__module__, _get_or_create_tracker_id(obj), None), + obj=obj + ) + + # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: + # Those attributes are already handled by the metaclass. + for attrname in ["_generate_next_value_", "_member_names_", + "_member_map_", "_member_type_", + "_value2member_map_"]: + clsdict.pop(attrname, None) + for member in members: + clsdict.pop(member) + + def save_dynamic_class(self, obj): + """Save a class that can't be stored as module global. + This method is used to serialize classes that are defined inside + functions, or that otherwise can't be serialized as attribute lookups + from global modules. + """ + clsdict = _extract_class_dict(obj) + clsdict.pop('__weakref__', None) + + if issubclass(type(obj), abc.ABCMeta): + # If obj is an instance of an ABCMeta subclass, dont pickle the + # cache/negative caches populated during isinstance/issubclass + # checks, but pickle the list of registered subclasses of obj. + clsdict.pop('_abc_cache', None) + clsdict.pop('_abc_negative_cache', None) + clsdict.pop('_abc_negative_cache_version', None) + registry = clsdict.pop('_abc_registry', None) + if registry is None: + # in Python3.7+, the abc caches and registered subclasses of a + # class are bundled into the single _abc_impl attribute + clsdict.pop('_abc_impl', None) + (registry, _, _, _) = abc._get_dump(obj) + + clsdict["_abc_impl"] = [subclass_weakref() + for subclass_weakref in registry] + else: + # In the above if clause, registry is a set of weakrefs -- in + # this case, registry is a WeakSet + clsdict["_abc_impl"] = [type_ for type_ in registry] + + # On PyPy, __doc__ is a readonly attribute, so we need to include it in + # the initial skeleton class. This is safe because we know that the + # doc can't participate in a cycle with the original class. + type_kwargs = {'__doc__': clsdict.pop('__doc__', None)} + + if "__slots__" in clsdict: + type_kwargs['__slots__'] = obj.__slots__ + # pickle string length optimization: member descriptors of obj are + # created automatically from obj's __slots__ attribute, no need to + # save them in obj's state + if isinstance(obj.__slots__, str): + clsdict.pop(obj.__slots__) + else: + for k in obj.__slots__: + clsdict.pop(k, None) + + # If type overrides __dict__ as a property, include it in the type + # kwargs. In Python 2, we can't set this attribute after construction. + # XXX: can this ever happen in Python 3? If so add a test. + __dict__ = clsdict.pop('__dict__', None) + if isinstance(__dict__, property): + type_kwargs['__dict__'] = __dict__ + + save = self.save + write = self.write + + # We write pickle instructions explicitly here to handle the + # possibility that the type object participates in a cycle with its own + # __dict__. We first write an empty "skeleton" version of the class and + # memoize it before writing the class' __dict__ itself. We then write + # instructions to "rehydrate" the skeleton class by restoring the + # attributes from the __dict__. + # + # A type can appear in a cycle with its __dict__ if an instance of the + # type appears in the type's __dict__ (which happens for the stdlib + # Enum class), or if the type defines methods that close over the name + # of the type, (which is common for Python 2-style super() calls). + + # Push the rehydration function. + save(_rehydrate_skeleton_class) + + # Mark the start of the args tuple for the rehydration function. + write(pickle.MARK) + + # Create and memoize an skeleton class with obj's name and bases. + if Enum is not None and issubclass(obj, Enum): + # Special handling of Enum subclasses + self._save_dynamic_enum(obj, clsdict) + else: + # "Regular" class definition: + tp = type(obj) + self.save_reduce(_make_skeleton_class, + (tp, obj.__name__, _get_bases(obj), type_kwargs, + _get_or_create_tracker_id(obj), None), + obj=obj) + + # Now save the rest of obj's __dict__. Any references to obj + # encountered while saving will point to the skeleton class. + save(clsdict) + + # Write a tuple of (skeleton_class, clsdict). + write(pickle.TUPLE) + + # Call _rehydrate_skeleton_class(skeleton_class, clsdict) + write(pickle.REDUCE) + + def save_function_tuple(self, func): + """ Pickles an actual func object. + A func comprises: code, globals, defaults, closure, and dict. We + extract and save these, injecting reducing functions at certain points + to recreate the func object. Keep in mind that some of these pieces + can contain a ref to the func itself. Thus, a naive save on these + pieces could trigger an infinite loop of save's. To get around that, + we first create a skeleton func object using just the code (this is + safe, since this won't contain a ref to the func), and memoize it as + soon as it's created. The other stuff can then be filled in later. + """ + if is_tornado_coroutine(func): + self.save_reduce(_rebuild_tornado_coroutine, (func.__wrapped__,), + obj=func) + return + + save = self.save + write = self.write + + code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func) + + save(_fill_function) # skeleton function updater + write(pickle.MARK) # beginning of tuple that _fill_function expects + + # Extract currently-imported submodules used by func. Storing these + # modules in a smoke _cloudpickle_subimports attribute of the object's + # state will trigger the side effect of importing these modules at + # unpickling time (which is necessary for func to work correctly once + # depickled) + submodules = _find_imported_submodules( + code, + itertools.chain(f_globals.values(), closure_values or ()), + ) + + # create a skeleton function object and memoize it + save(_make_skel_func) + save(( + code, + len(closure_values) if closure_values is not None else -1, + base_globals, + )) + write(pickle.REDUCE) + self.memoize(func) + + # save the rest of the func data needed by _fill_function + state = { + 'globals': f_globals, + 'defaults': defaults, + 'dict': dct, + 'closure_values': closure_values, + 'module': func.__module__, + 'name': func.__name__, + 'doc': func.__doc__, + '_cloudpickle_submodules': submodules + } + if hasattr(func, '__annotations__'): + state['annotations'] = func.__annotations__ + if hasattr(func, '__qualname__'): + state['qualname'] = func.__qualname__ + if hasattr(func, '__kwdefaults__'): + state['kwdefaults'] = func.__kwdefaults__ + save(state) + write(pickle.TUPLE) + write(pickle.REDUCE) # applies _fill_function on the tuple + + def extract_func_data(self, func): + """ + Turn the function into a tuple of data necessary to recreate it: + code, globals, defaults, closure_values, dict + """ + code = func.__code__ + + # extract all global ref's + func_global_refs = _extract_code_globals(code) + + # process all variables referenced by global environment + f_globals = {} + for var in func_global_refs: + if var in func.__globals__: + f_globals[var] = func.__globals__[var] + + # defaults requires no processing + defaults = func.__defaults__ + + # process closure + closure = ( + list(map(_get_cell_contents, func.__closure__)) + if func.__closure__ is not None + else None + ) + + # save the dict + dct = func.__dict__ + + # base_globals represents the future global namespace of func at + # unpickling time. Looking it up and storing it in globals_ref allow + # functions sharing the same globals at pickling time to also + # share them once unpickled, at one condition: since globals_ref is + # an attribute of a Cloudpickler instance, and that a new CloudPickler is + # created each time pickle.dump or pickle.dumps is called, functions + # also need to be saved within the same invokation of + # cloudpickle.dump/cloudpickle.dumps (for example: cloudpickle.dumps([f1, f2])). There + # is no such limitation when using Cloudpickler.dump, as long as the + # multiple invokations are bound to the same Cloudpickler. + base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) + + if base_globals == {}: + # Add module attributes used to resolve relative imports + # instructions inside func. + for k in ["__package__", "__name__", "__path__", "__file__"]: + # Some built-in functions/methods such as object.__new__ have + # their __globals__ set to None in PyPy + if func.__globals__ is not None and k in func.__globals__: + base_globals[k] = func.__globals__[k] + + return (code, f_globals, defaults, closure, dct, base_globals) + + def save_getset_descriptor(self, obj): + return self.save_reduce(getattr, (obj.__objclass__, obj.__name__)) + + dispatch[types.GetSetDescriptorType] = save_getset_descriptor + + def save_global(self, obj, name=None, pack=struct.pack): + """ + Save a "global". + The name of this method is somewhat misleading: all types get + dispatched here. + """ + if obj is type(None): + return self.save_reduce(type, (None,), obj=obj) + elif obj is type(Ellipsis): + return self.save_reduce(type, (Ellipsis,), obj=obj) + elif obj is type(NotImplemented): + return self.save_reduce(type, (NotImplemented,), obj=obj) + elif obj in _BUILTIN_TYPE_NAMES: + return self.save_reduce( + _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) + + if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch + # Parametrized typing constructs in Python < 3.7 are not compatible + # with type checks and ``isinstance`` semantics. For this reason, + # it is easier to detect them using a duck-typing-based check + # (``_is_parametrized_type_hint``) than to populate the Pickler's + # dispatch with type-specific savers. + self._save_parametrized_type_hint(obj) + elif name is not None: + Pickler.save_global(self, obj, name=name) + elif not _is_importable_by_name(obj, name=name): + self.save_dynamic_class(obj) + else: + Pickler.save_global(self, obj, name=name) - self.memoize(obj) + dispatch[type] = save_global - try: - getstate = obj.__getstate__ - except AttributeError: - stuff = obj.__dict__ - #remove items if transient - if hasattr(obj, '__transient__'): - transient = obj.__transient__ - stuff = stuff.copy() - for k in list(stuff.keys()): - if k in transient: - del stuff[k] - else: - stuff = getstate() - pickle._keep_alive(stuff, memo) # pylint: disable=protected-access - save(stuff) - write(pickle.BUILD) - - if not PY3: - dispatch[types.InstanceType] = save_inst - - def save_property(self, obj): - # properties not correctly saved in python - self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj) - dispatch[property] = save_property - - def save_itemgetter(self, obj): - """itemgetter serializer (needed for namedtuple support)""" - class Dummy: # pylint: disable=old-style-class - def __init__(self): - pass - def __getitem__(self, item): - return item - items = obj(Dummy()) - if not isinstance(items, tuple): - items = (items, ) - return self.save_reduce(operator.itemgetter, items) - - if type(operator.itemgetter) is type: # pylint: disable=unidiomatic-typecheck - dispatch[operator.itemgetter] = save_itemgetter - - def save_attrgetter(self, obj): - """attrgetter serializer""" - class Dummy: - def __init__(self, attrs, index=None): - self.attrs = attrs - self.index = index - def __getattribute__(self, item): - attrs = object.__getattribute__(self, "attrs") - index = object.__getattribute__(self, "index") - if index is None: - index = len(attrs) - attrs.append(item) + def save_instancemethod(self, obj): + # Memoization rarely is ever useful due to python bounding + if obj.__self__ is None: + self.save_reduce(getattr, (obj.im_class, obj.__name__)) else: - attrs[index] = ".".join([attrs[index], item]) - return type(self)(attrs, index) - attrs = [] - obj(Dummy(attrs)) - return self.save_reduce(operator.attrgetter, tuple(attrs)) - - if type(operator.attrgetter) is type: # pylint: disable=unidiomatic-typecheck - dispatch[operator.attrgetter] = save_attrgetter - - def save_reduce(self, func, args, state=None, # pylint: disable=too-many-branches - listitems=None, dictitems=None, obj=None): - """Modified to support __transient__ on new objects - Change only affects protocol level 2 (which is always used by PiCloud""" - # Assert that args is a tuple or None - if not isinstance(args, tuple): - raise pickle.PicklingError("args from reduce() should be a tuple") - - # Assert that func is callable - if not hasattr(func, '__call__'): - raise pickle.PicklingError("func from reduce should be callable") - - save = self.save - write = self.write - - # Protocol 2 special case: if func's name is __newobj__, use NEWOBJ - if self.proto >= 2 and getattr(func, "__name__", "") == "__newobj__": - #Added fix to allow transient - cls = args[0] - if not hasattr(cls, "__new__"): - raise pickle.PicklingError( - "args[0] from __newobj__ args has no __new__") - if obj is not None and cls is not obj.__class__: - raise pickle.PicklingError( - "args[0] from __newobj__ args has the wrong class") - args = args[1:] - save(cls) - - #Don't pickle transient entries - if hasattr(obj, '__transient__'): - transient = obj.__transient__ - state = state.copy() - - for k in list(state.keys()): - if k in transient: - del state[k] - - save(args) - write(pickle.NEWOBJ) - else: - save(func) - save(args) - write(pickle.REDUCE) + self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj) + + dispatch[types.MethodType] = save_instancemethod + + def save_property(self, obj): + # properties not correctly saved in python + self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), + obj=obj) + + dispatch[property] = save_property + + def save_classmethod(self, obj): + orig_func = obj.__func__ + self.save_reduce(type(obj), (orig_func,), obj=obj) + + dispatch[classmethod] = save_classmethod + dispatch[staticmethod] = save_classmethod + + def save_itemgetter(self, obj): + """itemgetter serializer (needed for namedtuple support)""" + class Dummy: + def __getitem__(self, item): + return item + items = obj(Dummy()) + if not isinstance(items, tuple): + items = (items,) + return self.save_reduce(operator.itemgetter, items) + + if type(operator.itemgetter) is type: + dispatch[operator.itemgetter] = save_itemgetter + + def save_attrgetter(self, obj): + """attrgetter serializer""" + class Dummy(object): + def __init__(self, attrs, index=None): + self.attrs = attrs + self.index = index + def __getattribute__(self, item): + attrs = object.__getattribute__(self, "attrs") + index = object.__getattribute__(self, "index") + if index is None: + index = len(attrs) + attrs.append(item) + else: + attrs[index] = ".".join([attrs[index], item]) + return type(self)(attrs, index) + attrs = [] + obj(Dummy(attrs)) + return self.save_reduce(operator.attrgetter, tuple(attrs)) + + if type(operator.attrgetter) is type: + dispatch[operator.attrgetter] = save_attrgetter + + def save_file(self, obj): + """Save a file""" + + if not hasattr(obj, 'name') or not hasattr(obj, 'mode'): + raise pickle.PicklingError("Cannot pickle files that do not map to an actual file") + if obj is sys.stdout: + return self.save_reduce(getattr, (sys, 'stdout'), obj=obj) + if obj is sys.stderr: + return self.save_reduce(getattr, (sys, 'stderr'), obj=obj) + if obj is sys.stdin: + raise pickle.PicklingError("Cannot pickle standard input") + if obj.closed: + raise pickle.PicklingError("Cannot pickle closed files") + if hasattr(obj, 'isatty') and obj.isatty(): + raise pickle.PicklingError("Cannot pickle files that map to tty objects") + if 'r' not in obj.mode and '+' not in obj.mode: + raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode) + + name = obj.name + + # TODO: also support binary mode files with io.BytesIO + retval = io.StringIO() + + try: + # Read the whole file + curloc = obj.tell() + obj.seek(0) + contents = obj.read() + obj.seek(curloc) + except IOError: + raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name) + retval.write(contents) + retval.seek(curloc) + + retval.name = name + self.save(retval) + self.memoize(obj) + + def save_ellipsis(self, obj): + self.save_reduce(_gen_ellipsis, ()) + + def save_not_implemented(self, obj): + self.save_reduce(_gen_not_implemented, ()) - if obj is not None: - self.memoize(obj) + dispatch[io.TextIOWrapper] = save_file + dispatch[type(Ellipsis)] = save_ellipsis + dispatch[type(NotImplemented)] = save_not_implemented - # More new special cases (that work with older protocols as - # well): when __reduce__ returns a tuple with 4 or 5 items, - # the 4th and 5th item should be iterators that provide list - # items and dict items (as (key, value) tuples), or None. + def save_weakset(self, obj): + self.save_reduce(weakref.WeakSet, (list(obj),)) - if listitems is not None: - self._batch_appends(listitems) + dispatch[weakref.WeakSet] = save_weakset - if dictitems is not None: - self._batch_setitems(dictitems) + def save_logger(self, obj): + self.save_reduce(logging.getLogger, (obj.name,), obj=obj) - if state is not None: - save(state) - write(pickle.BUILD) + dispatch[logging.Logger] = save_logger - def save_partial(self, obj): - """Partial objects do not serialize correctly in python2.x -- this fixes the bugs""" - self.save_reduce(_genpartial, (obj.func, obj.args, obj.keywords)) + def save_root_logger(self, obj): + self.save_reduce(logging.getLogger, (), obj=obj) - if sys.version_info < (2, 7): # 2.7 supports partial pickling - dispatch[partial] = save_partial + dispatch[logging.RootLogger] = save_root_logger + if hasattr(types, "MappingProxyType"): # pragma: no branch + def save_mappingproxy(self, obj): + self.save_reduce(types.MappingProxyType, (dict(obj),), obj=obj) - def save_file(self, obj): # pylint: disable=too-many-branches - """Save a file""" - try: - import StringIO as pystringIO #we can't use cStringIO as it lacks the name attribute - except ImportError: - import io as pystringIO # pylint: disable=reimported - - if not hasattr(obj, 'name') or not hasattr(obj, 'mode'): - raise pickle.PicklingError("Cannot pickle files that do not map to an actual file") - if obj is sys.stdout: - return self.save_reduce(getattr, (sys, 'stdout'), obj=obj) - if obj is sys.stderr: - return self.save_reduce(getattr, (sys, 'stderr'), obj=obj) - if obj is sys.stdin: - raise pickle.PicklingError("Cannot pickle standard input") - if hasattr(obj, 'isatty') and obj.isatty(): - raise pickle.PicklingError("Cannot pickle files that map to tty objects") - if 'r' not in obj.mode: - raise pickle.PicklingError("Cannot pickle files that are not opened for reading") - name = obj.name - try: - fsize = os.stat(name).st_size - except OSError: - raise pickle.PicklingError("Cannot pickle file %s as it cannot be stat" % name) - - if obj.closed: - #create an empty closed string io - retval = pystringIO.StringIO("") - retval.close() - elif not fsize: #empty file - retval = pystringIO.StringIO("") - try: - tmpfile = file(name) - tst = tmpfile.read(1) - except IOError: - raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name) - tmpfile.close() - if tst != '': - raise pickle.PicklingError( - "Cannot pickle file %s as it does not appear to map to a physical, real file" % name) - else: - try: - tmpfile = file(name) - contents = tmpfile.read() - tmpfile.close() - except IOError: - raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name) - retval = pystringIO.StringIO(contents) - curloc = obj.tell() - retval.seek(curloc) - - retval.name = name - self.save(retval) - self.memoize(obj) - - if PY3: - dispatch[io.TextIOWrapper] = save_file - else: - dispatch[file] = save_file + dispatch[types.MappingProxyType] = save_mappingproxy - # Special functions for Add-on libraries + """Special functions for Add-on libraries""" + def inject_addons(self): + """Plug in system. Register additional pickling functions if modules already loaded""" + pass - def inject_numpy(self): - numpy = sys.modules.get('numpy') - if not numpy or not hasattr(numpy, 'ufunc'): - return - self.dispatch[numpy.ufunc] = self.__class__.save_ufunc + if sys.version_info < (3, 7): # pragma: no branch + def _save_parametrized_type_hint(self, obj): + # The distorted type check sematic for typing construct becomes: + # ``type(obj) is type(TypeHint)``, which means "obj is a + # parametrized TypeHint" + if type(obj) is type(Literal): # pragma: no branch + initargs = (Literal, obj.__values__) + elif type(obj) is type(Final): # pragma: no branch + initargs = (Final, obj.__type__) + elif type(obj) is type(ClassVar): + initargs = (ClassVar, obj.__type__) + elif type(obj) is type(Generic): + parameters = obj.__parameters__ + if len(obj.__parameters__) > 0: + # in early Python 3.5, __parameters__ was sometimes + # preferred to __args__ + initargs = (obj.__origin__, parameters) + else: + initargs = (obj.__origin__, obj.__args__) + elif type(obj) is type(Union): + if sys.version_info < (3, 5, 3): # pragma: no cover + initargs = (Union, obj.__union_params__) + else: + initargs = (Union, obj.__args__) + elif type(obj) is type(Tuple): + if sys.version_info < (3, 5, 3): # pragma: no cover + initargs = (Tuple, obj.__tuple_params__) + else: + initargs = (Tuple, obj.__args__) + elif type(obj) is type(Callable): + if sys.version_info < (3, 5, 3): # pragma: no cover + args = obj.__args__ + result = obj.__result__ + if args != Ellipsis: + if isinstance(args, tuple): + args = list(args) + else: + args = [args] + else: + (*args, result) = obj.__args__ + if len(args) == 1 and args[0] is Ellipsis: + args = Ellipsis + else: + args = list(args) + initargs = (Callable, (args, result)) + else: # pragma: no cover + raise pickle.PicklingError( + "Cloudpickle Error: Unknown type {}".format(type(obj)) + ) + self.save_reduce(_create_parametrized_type_hint, initargs, obj=obj) + + +# Tornado support + +def is_tornado_coroutine(func): + """ + Return whether *func* is a Tornado coroutine function. + Running coroutines are not supported. + """ + if 'tornado.gen' not in sys.modules: + return False + gen = sys.modules['tornado.gen'] + if not hasattr(gen, "is_coroutine_function"): + # Tornado version is too old + return False + return gen.is_coroutine_function(func) - def save_ufunc(self, obj): - """Hack function for saving numpy ufunc objects""" - name = obj.__name__ - numpy_tst_mods = ['numpy', 'scipy.special'] - for tst_mod_name in numpy_tst_mods: - tst_mod = sys.modules.get(tst_mod_name, None) - if tst_mod and name in tst_mod.__dict__: - return self.save_reduce(_getobject, (tst_mod_name, name)) - raise pickle.PicklingError( - 'cannot save %s. Cannot resolve what module it is defined in' % str(obj)) - def inject_addons(self): - """Plug in system. Register additional pickling functions if modules already loaded""" - self.inject_numpy() +def _rebuild_tornado_coroutine(func): + from tornado import gen + return gen.coroutine(func) # Shorthands for legacy support -def dump(obj, filen, protocol=2): - CloudPickler(filen, protocol).dump(obj) +def dump(obj, file, protocol=None): + """Serialize obj as bytes streamed into file + protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to + pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed + between processes running the same Python version. + Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure + compatibility with older versions of Python. + """ + CloudPickler(file, protocol=protocol).dump(obj) -def dumps(obj, protocol=2): - filen = StringIO() +def dumps(obj, protocol=None): + """Serialize obj as a string of bytes allocated in memory + protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to + pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed + between processes running the same Python version. + Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure + compatibility with older versions of Python. + """ + file = BytesIO() + try: + cp = CloudPickler(file, protocol=protocol) + cp.dump(obj) + return file.getvalue() + finally: + file.close() - cp = CloudPickler(filen, protocol) - cp.dump(obj) - return filen.getvalue() +# including pickles unloading functions in this namespace +load = pickle.load +loads = pickle.loads -#hack for __import__ not working as desired +# hack for __import__ not working as desired def subimport(name): - __import__(name) - return sys.modules[name] - + __import__(name) + return sys.modules[name] -# restores function attributes -def _restore_attr(obj, attr): - for key, val in list(attr.items()): - setattr(obj, key, val) - return obj +def dynamic_subimport(name, vars): + mod = types.ModuleType(name) + mod.__dict__.update(vars) + mod.__dict__['__builtins__'] = builtins.__dict__ + return mod -def _get_module_builtins(): - return pickle.__builtins__ # pylint: disable=no-member +def _gen_ellipsis(): + return Ellipsis -def print_exec(stream): - ei = sys.exc_info() - traceback.print_exception(ei[0], ei[1], ei[2], None, stream) +def _gen_not_implemented(): + return NotImplemented -def _modules_to_main(modList): - """Force every module in modList to be placed into main""" - if not modList: - return - - main = sys.modules['__main__'] - for modname in modList: - if isinstance(modname, str): - try: - mod = __import__(modname) - except Exception: - sys.stderr.write( - 'warning: could not import %s\n. ' - 'Your function may unexpectedly error due to this import failing;' - 'A version mismatch is likely. Specific error was:\n' % modname) - print_exec(sys.stderr) - else: - setattr(main, mod.__name__, mod) - - -#object generators: -def _genpartial(func, args, kwds): - if not args: - args = () - if not kwds: - kwds = {} - return partial(func, *args, **kwds) - - -def _fill_function(func, globalsn, defaults, dictn, module): - """ Fills in the rest of function data into the skeleton function object - that were created via _make_skel_func(). - """ - func.__globals__.update(globalsn) - func.__defaults__ = defaults - func.__dict__ = dictn - func.__module__ = module - - return func +def _get_cell_contents(cell): + try: + return cell.cell_contents + except ValueError: + # sentinel used by ``_fill_function`` which will leave the cell empty + return _empty_cell_value + + +def instance(cls): + """Create a new instance of a class. + Parameters + ---------- + cls : type + The class to create an instance of. + Returns + ------- + instance : cls + A new instance of ``cls``. + """ + return cls() -def _make_cell(value): - return (lambda: value).__closure__[0] +@instance +class _empty_cell_value(object): + """sentinel for empty closures + """ + @classmethod + def __reduce__(cls): + return cls.__name__ -def _reconstruct_closure(values): - return tuple([_make_cell(v) for v in values]) +def _fill_function(*args): + """Fills in the rest of function data into the skeleton function object + The skeleton itself is create by _make_skel_func(). + """ + if len(args) == 2: + func = args[0] + state = args[1] + elif len(args) == 5: + # Backwards compat for cloudpickle v0.4.0, after which the `module` + # argument was introduced + func = args[0] + keys = ['globals', 'defaults', 'dict', 'closure_values'] + state = dict(zip(keys, args[1:])) + elif len(args) == 6: + # Backwards compat for cloudpickle v0.4.1, after which the function + # state was passed as a dict to the _fill_function it-self. + func = args[0] + keys = ['globals', 'defaults', 'dict', 'module', 'closure_values'] + state = dict(zip(keys, args[1:])) + else: + raise ValueError('Unexpected _fill_value arguments: %r' % (args,)) + + # - At pickling time, any dynamic global variable used by func is + # serialized by value (in state['globals']). + # - At unpickling time, func's __globals__ attribute is initialized by + # first retrieving an empty isolated namespace that will be shared + # with other functions pickled from the same original module + # by the same CloudPickler instance and then updated with the + # content of state['globals'] to populate the shared isolated + # namespace with all the global variables that are specifically + # referenced for this function. + func.__globals__.update(state['globals']) + + func.__defaults__ = state['defaults'] + func.__dict__ = state['dict'] + if 'annotations' in state: + func.__annotations__ = state['annotations'] + if 'doc' in state: + func.__doc__ = state['doc'] + if 'name' in state: + func.__name__ = state['name'] + if 'module' in state: + func.__module__ = state['module'] + if 'qualname' in state: + func.__qualname__ = state['qualname'] + if 'kwdefaults' in state: + func.__kwdefaults__ = state['kwdefaults'] + # _cloudpickle_subimports is a set of submodules that must be loaded for + # the pickled function to work correctly at unpickling time. Now that these + # submodules are depickled (hence imported), they can be removed from the + # object's state (the object state only served as a reference holder to + # these submodules) + if '_cloudpickle_submodules' in state: + state.pop('_cloudpickle_submodules') + + cells = func.__closure__ + if cells is not None: + for cell, value in zip(cells, state['closure_values']): + if value is not _empty_cell_value: + cell_set(cell, value) + + return func + + +def _make_empty_cell(): + if False: + # trick the compiler into creating an empty cell in our lambda + cell = None + raise AssertionError('this route should not be executed') + + return (lambda: cell).__closure__[0] + + +def _make_skel_func(code, cell_count, base_globals=None): + """ Creates a skeleton function object that contains just the provided + code and the correct number of cells in func_closure. All other + func attributes (e.g. func_globals) are empty. + """ + # This is backward-compatibility code: for cloudpickle versions between + # 0.5.4 and 0.7, base_globals could be a string or None. base_globals + # should now always be a dictionary. + if base_globals is None or isinstance(base_globals, str): + base_globals = {} + + base_globals['__builtins__'] = __builtins__ + + closure = ( + tuple(_make_empty_cell() for _ in range(cell_count)) + if cell_count >= 0 else + None + ) + return types.FunctionType(code, base_globals, None, None, closure) + + +def _make_skeleton_class(type_constructor, name, bases, type_kwargs, + class_tracker_id, extra): + """Build dynamic class with an empty __dict__ to be filled once memoized + If class_tracker_id is not None, try to lookup an existing class definition + matching that id. If none is found, track a newly reconstructed class + definition under that id so that other instances stemming from the same + class id will also reuse this class definition. + The "extra" variable is meant to be a dict (or None) that can be used for + forward compatibility shall the need arise. + """ + skeleton_class = types.new_class( + name, bases, {'metaclass': type_constructor}, + lambda ns: ns.update(type_kwargs) + ) + return _lookup_class_or_track(class_tracker_id, skeleton_class) -def _make_skel_func(code, closures, base_globals=None): - """ Creates a skeleton function object that contains just the provided - code and the correct number of cells in func_closure. All other - func attributes (e.g. func_globals) are empty. - """ - closure = _reconstruct_closure(closures) if closures else None - if base_globals is None: - base_globals = {} - base_globals['__builtins__'] = __builtins__ +def _rehydrate_skeleton_class(skeleton_class, class_dict): + """Put attributes from `class_dict` back on `skeleton_class`. + See CloudPickler.save_dynamic_class for more info. + """ + registry = None + for attrname, attr in class_dict.items(): + if attrname == "_abc_impl": + registry = attr + else: + setattr(skeleton_class, attrname, attr) + if registry is not None: + for subclass in registry: + skeleton_class.register(subclass) + + return skeleton_class + + +def _make_skeleton_enum(bases, name, qualname, members, module, + class_tracker_id, extra): + """Build dynamic enum with an empty __dict__ to be filled once memoized + The creation of the enum class is inspired by the code of + EnumMeta._create_. + If class_tracker_id is not None, try to lookup an existing enum definition + matching that id. If none is found, track a newly reconstructed enum + definition under that id so that other instances stemming from the same + class id will also reuse this enum definition. + The "extra" variable is meant to be a dict (or None) that can be used for + forward compatibility shall the need arise. + """ + # enums always inherit from their base Enum class at the last position in + # the list of base classes: + enum_base = bases[-1] + metacls = enum_base.__class__ + classdict = metacls.__prepare__(name, bases) - return types.FunctionType(code, base_globals, None, None, closure) + for member_name, member_value in members.items(): + classdict[member_name] = member_value + enum_class = metacls.__new__(metacls, name, bases, classdict) + enum_class.__module__ = module + enum_class.__qualname__ = qualname + return _lookup_class_or_track(class_tracker_id, enum_class) -def _load_class(cls, d): - """ - Loads additional properties into class `cls`. - """ - for k, v in list(d.items()): - if isinstance(k, tuple): - typ, k = k - if typ == 'property': - v = property(*v) - elif typ == 'staticmethod': - v = staticmethod(v) - elif typ == 'classmethod': - v = classmethod(v) - setattr(cls, k, v) - return cls +def _is_dynamic(module): + """ + Return True if the module is special module that cannot be imported by its + name. + """ + # Quick check: module that have __file__ attribute are not dynamic modules. + if hasattr(module, '__file__'): + return False + + if module.__spec__ is not None: + return False + + # In PyPy, Some built-in modules such as _codecs can have their + # __spec__ attribute set to None despite being imported. For such + # modules, the ``_find_spec`` utility of the standard library is used. + parent_name = module.__name__.rpartition('.')[0] + if parent_name: # pragma: no cover + # This code handles the case where an imported package (and not + # module) remains with __spec__ set to None. It is however untested + # as no package in the PyPy stdlib has __spec__ set to None after + # it is imported. + try: + parent = sys.modules[parent_name] + except KeyError: + msg = "parent {!r} not in sys.modules" + raise ImportError(msg.format(parent_name)) + else: + pkgpath = parent.__path__ + else: + pkgpath = None + return _find_spec(module.__name__, pkgpath, module) is None -def _load_namedtuple(name, fields): - """ - Loads a class generated by namedtuple - """ - from collections import namedtuple - return namedtuple(name, fields) +def _make_typevar(name, bound, constraints, covariant, contravariant, + class_tracker_id): + tv = typing.TypeVar( + name, *constraints, bound=bound, + covariant=covariant, contravariant=contravariant + ) + if class_tracker_id is not None: + return _lookup_class_or_track(class_tracker_id, tv) + else: # pragma: nocover + # Only for Python 3.5.3 compat. + return tv -# Constructors for 3rd party libraries -# Note: These can never be renamed due to client compatibility issues -def _getobject(modname, attribute): - mod = __import__(modname, fromlist=[attribute]) - return mod.__dict__[attribute] +def _decompose_typevar(obj): + try: + class_tracker_id = _get_or_create_tracker_id(obj) + except TypeError: # pragma: nocover + # TypeVar instances are not weakref-able in Python 3.5.3 + class_tracker_id = None + return ( + obj.__name__, obj.__bound__, obj.__constraints__, + obj.__covariant__, obj.__contravariant__, + class_tracker_id, + ) + + +def _typevar_reduce(obj): + # TypeVar instances have no __qualname__ hence we pass the name explicitly. + module_and_name = _lookup_module_and_qualname(obj, name=obj.__name__) + if module_and_name is None: + return (_make_typevar, _decompose_typevar(obj)) + return (getattr, module_and_name) + + +def _get_bases(typ): + if hasattr(typ, '__orig_bases__'): + # For generic types (see PEP 560) + bases_attr = '__orig_bases__' + else: + # For regular class objects + bases_attr = '__bases__' + return getattr(typ, bases_attr) From 5efa3444e781720000252883f87dd08d3c715427 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sun, 31 May 2020 14:42:03 +0100 Subject: [PATCH 16/22] Fix python addressing of release.yaml --- heron/tools/common/src/python/utils/config.py | 36 ++++--------------- heron/tools/ui/src/python/consts.py | 2 +- heron/tools/ui/src/python/main.py | 2 +- 3 files changed, 8 insertions(+), 32 deletions(-) diff --git a/heron/tools/common/src/python/utils/config.py b/heron/tools/common/src/python/utils/config.py index 577ee9930da..9747021c1a5 100644 --- a/heron/tools/common/src/python/utils/config.py +++ b/heron/tools/common/src/python/utils/config.py @@ -46,7 +46,6 @@ LIB_DIR = "lib" CLI_DIR = ".heron" RELEASE_YAML = "release.yaml" -ZIPPED_RELEASE_YAML = "scripts/packages/release.yaml" OVERRIDE_YAML = "override.yaml" # mode of deployment @@ -217,17 +216,6 @@ def get_heron_release_file(): return os.path.join(get_heron_dir(), RELEASE_YAML) -def get_zipped_heron_release_file(): - """ - This will provide the path to heron release.yaml file. - To be used for .pex file built with `zip_safe = False` flag. - For example, `heron-ui'. - - :return: absolute path of heron release.yaml file - """ - return os.path.join(get_zipped_heron_dir(), ZIPPED_RELEASE_YAML) - - def get_heron_cluster_conf_dir(cluster, default_config_path): """ This will provide heron cluster config directory, if config path is default @@ -448,15 +436,9 @@ def check_release_file_exists(): return True -def print_build_info(zipped_pex=False): - """Print build_info from release.yaml - - :param zipped_pex: True if the PEX file is built with flag `zip_safe=False'. - """ - if zipped_pex: - release_file = get_zipped_heron_release_file() - else: - release_file = get_heron_release_file() +def print_build_info(): + """Print build_info from release.yaml""" + release_file = get_heron_release_file() with open(release_file) as release_info: release_map = yaml.load(release_info) @@ -464,15 +446,9 @@ def print_build_info(zipped_pex=False): for key, value in release_items: print("%s : %s" % (key, value)) -def get_version_number(zipped_pex=False): - """Print version from release.yaml - - :param zipped_pex: True if the PEX file is built with flag `zip_safe=False'. - """ - if zipped_pex: - release_file = get_zipped_heron_release_file() - else: - release_file = get_heron_release_file() +def get_version_number(): + """Print version from release.yaml""" + release_file = get_heron_release_file() with open(release_file) as release_info: for line in release_info: trunks = line[:-1].split(' ') diff --git a/heron/tools/ui/src/python/consts.py b/heron/tools/ui/src/python/consts.py index 63e7adff9c7..7116a53044f 100644 --- a/heron/tools/ui/src/python/consts.py +++ b/heron/tools/ui/src/python/consts.py @@ -32,4 +32,4 @@ DEFAULT_BASE_URL = "" -VERSION = common_config.get_version_number(zipped_pex=True) +VERSION = common_config.get_version_number() diff --git a/heron/tools/ui/src/python/main.py b/heron/tools/ui/src/python/main.py index 3ca49476e2a..224937a1940 100644 --- a/heron/tools/ui/src/python/main.py +++ b/heron/tools/ui/src/python/main.py @@ -163,7 +163,7 @@ def main(): r = child_parser.parse_args(args=remaining, namespace=parsed_args) namespace = vars(r) if 'version' in namespace: - common_config.print_build_info(zipped_pex=True) + common_config.print_build_info() else: parser.print_help() parser.exit() From 6eeb7d244604b8715c5c9241b13e6f124b9ca4aa Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 30 May 2020 18:15:41 +0100 Subject: [PATCH 17/22] Additions to get docker image builds working and tested * use new external pkg_* rules * add python to compile docker images until pkg_* * add --host_force_python=PY3 to other bazel.rc files --- WORKSPACE | 8 ++++++++ docker/compile/Dockerfile.centos7 | 1 + docker/compile/Dockerfile.debian10 | 1 + docker/compile/Dockerfile.debian9 | 1 + docker/compile/Dockerfile.ubuntu16.04 | 1 + docker/compile/Dockerfile.ubuntu18.04 | 1 + docker/compile/Dockerfile.ubuntu20.04 | 1 + scripts/packages/BUILD | 3 +-- third_party/python/cpplint/BUILD | 5 +---- tools/docker/bazel.rc | 3 ++- tools/travis/bazel.rc | 2 ++ 11 files changed, 20 insertions(+), 7 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 72c1b1f73a5..279524d3903 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -402,6 +402,14 @@ container_pull( # end docker image building +http_archive( + name = "rules_pkg", + url = "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.5/rules_pkg-0.2.5.tar.gz", + sha256 = "352c090cc3d3f9a6b4e676cf42a6047c16824959b438895a76c2989c6d7c246a", +) +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") +rules_pkg_dependencies() + # for nomad repear http_archive( name = "nomad_mac", diff --git a/docker/compile/Dockerfile.centos7 b/docker/compile/Dockerfile.centos7 index 6d1bb0cad40..5f39bde33f2 100644 --- a/docker/compile/Dockerfile.centos7 +++ b/docker/compile/Dockerfile.centos7 @@ -35,6 +35,7 @@ RUN yum -y install \ libtool \ make \ patch \ + python \ python3-devel \ zip \ unzip \ diff --git a/docker/compile/Dockerfile.debian10 b/docker/compile/Dockerfile.debian10 index fab07045a6c..c6918315f12 100644 --- a/docker/compile/Dockerfile.debian10 +++ b/docker/compile/Dockerfile.debian10 @@ -32,6 +32,7 @@ RUN apt-get update && apt-get -y install \ libtool-bin \ libcppunit-dev \ pkg-config \ + python \ python3 \ python3-dev \ software-properties-common \ diff --git a/docker/compile/Dockerfile.debian9 b/docker/compile/Dockerfile.debian9 index e37da48fe24..ffe96bcf726 100644 --- a/docker/compile/Dockerfile.debian9 +++ b/docker/compile/Dockerfile.debian9 @@ -33,6 +33,7 @@ RUN apt-get update && apt-get -y install \ libcppunit-dev \ pkg-config \ software-properties-common \ + python \ python3-dev \ python3-setuptools \ tree \ diff --git a/docker/compile/Dockerfile.ubuntu16.04 b/docker/compile/Dockerfile.ubuntu16.04 index f3654b858eb..f3f6fb5dbd0 100644 --- a/docker/compile/Dockerfile.ubuntu16.04 +++ b/docker/compile/Dockerfile.ubuntu16.04 @@ -34,6 +34,7 @@ RUN apt-get update && apt-get -y install \ libssl-dev \ libtool-bin \ libunwind-setjmp0-dev \ + python \ python3-dev \ pkg-config \ libcppunit-dev \ diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04 index e276c7b461a..b0c71ea890b 100644 --- a/docker/compile/Dockerfile.ubuntu18.04 +++ b/docker/compile/Dockerfile.ubuntu18.04 @@ -29,6 +29,7 @@ RUN apt-get update && apt-get -y install \ libunwind8 \ libcppunit-dev \ patch \ + python \ python3-dev \ pkg-config \ wget \ diff --git a/docker/compile/Dockerfile.ubuntu20.04 b/docker/compile/Dockerfile.ubuntu20.04 index eab62993d24..a028c4e6125 100644 --- a/docker/compile/Dockerfile.ubuntu20.04 +++ b/docker/compile/Dockerfile.ubuntu20.04 @@ -32,6 +32,7 @@ RUN apt-get update && apt-get -y install \ libcppunit-dev \ patch \ python3-dev \ + python \ pkg-config \ wget \ zip \ diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD index ac98b87908e..f3b7e1c5908 100644 --- a/scripts/packages/BUILD +++ b/scripts/packages/BUILD @@ -1,5 +1,4 @@ -# load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_deb", "pkg_tar") -load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") +load("@rules_pkg//:pkg.bzl", "pkg_tar") load("//scripts/packages:self_extract_binary.bzl", "self_extract_binary") package(default_visibility = ["//visibility:public"]) diff --git a/third_party/python/cpplint/BUILD b/third_party/python/cpplint/BUILD index b1116787ce6..6a67bcce270 100644 --- a/third_party/python/cpplint/BUILD +++ b/third_party/python/cpplint/BUILD @@ -1,12 +1,9 @@ -load("@rules_python//python:defs.bzl", "py_binary") - licenses(["notice"]) package(default_visibility = ["//visibility:public"]) -py_binary( +pex_binary( name = "cpplint", srcs = ["cpplint.py"], main = "cpplint.py", - stamp = 1, ) diff --git a/tools/docker/bazel.rc b/tools/docker/bazel.rc index 26d04457fb4..05a67d78cde 100644 --- a/tools/docker/bazel.rc +++ b/tools/docker/bazel.rc @@ -17,6 +17,7 @@ # This is so we understand failures better build --verbose_failures +build --host_force_python=PY3 # This is so we don't use sandboxed execution. Sandboxed execution # runs stuff in a container, and since Travis already runs its script @@ -32,4 +33,4 @@ build --local_ram_resources=4096 build --local_cpu_resources=2 # Echo all the configuration settings and their source - build --announce_rc \ No newline at end of file + build --announce_rc diff --git a/tools/travis/bazel.rc b/tools/travis/bazel.rc index a96e92f72fa..518f2d1d0bf 100644 --- a/tools/travis/bazel.rc +++ b/tools/travis/bazel.rc @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +build --host_force_python=PY3 + # This is from Bazel's former travis setup, to avoid blowing up the RAM usage. startup --host_jvm_args=-Xmx2500m startup --host_jvm_args=-Xms2500m From 9c855d7177f33ddb58e783f9068056521a7e155f Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 30 May 2020 21:11:09 +0100 Subject: [PATCH 18/22] WIP: Add CI for docker images/releases * use kind to create ephemeral clusters * start consolidating scripts with python --- scripts/release/docker-images | 145 ++++++++++++++++++++++++++++++++ scripts/travis/k8s.sh | 92 ++++++++++++++++++++ scripts/travis/k8s.sh.kind.yaml | 4 + vagrant/local-ci.sh | 2 +- 4 files changed, 242 insertions(+), 1 deletion(-) create mode 100755 scripts/release/docker-images create mode 100755 scripts/travis/k8s.sh create mode 100644 scripts/travis/k8s.sh.kind.yaml diff --git a/scripts/release/docker-images b/scripts/release/docker-images new file mode 100755 index 00000000000..7316abcf049 --- /dev/null +++ b/scripts/release/docker-images @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +""" +This script cooridnates other scripts to put together a release. + +Generated images will have a label in the form heron/heron: and will be placed in the //distro/ directory. + +## Examples + +List available target distrobutions on separate stdout lines: + ./docker-images + +Build and tag a single distorbution image then print where the archive's path: + ./docker-images build 0.1.0-debian10 debian10 + +Build and tag all distrobution images then print each archive's path: + ./docker-images build "$(git describe --tags)" --all + +""" +from pathlib import Path + +import logging +import re +import shutil +import subprocess +import sys +import tempfile +import typing + +ROOT = Path(__file__).resolve().parent.parent.parent +BUILD_ARTIFACTS = ROOT / "docker/scripts/build-artifacts.sh" +BUILD_IMAGE = ROOT / "docker/scripts/build-docker.sh" + + +class BuildFailure(Exception): + """Raised to indicate a failure buliding.""" + + +class BadDistrobutionName(BuildFailure): + """Raised when a bad distrobution name is provided.""" + + +def configure_logging(debug: bool): + """Use standard logging config and write to stdout and a logfile.""" + logging.basicConfig( + format="[%(asctime)s] %(levelname)s: %(message)s", + level=(logging.DEBUG if debug else logging.INFO), + ) + + +def log_run(args: typing.List[str], log: typing.IO[str]) -> subprocess.CompletedProcess: + """Run an executable and direct its output to the given log file.""" + return subprocess.run( + args, stdout=log, stderr=log, universal_newlines=True, check=True + ) + + +def build_dockerfile( + scratch: Path, dist: str, tag: str, out_dir: Path, log: typing.IO[str] +) -> Path: + """ + Raises CalledProcessError if either of the external scripts fail. + """ + logging.info("building package for %s", dist) + log_run([str(BUILD_ARTIFACTS), dist, tag, scratch], log) + logging.info("building docker image for %s", dist) + log_run([str(BUILD_IMAGE), dist, tag, scratch], log) + tar = Path(scratch) / f"heron-docker-{tag}-{dist}.tar.gz" + tar_out = out_dir / tar.name + tar.replace(tar_out) + logging.info("docker image complete: %s", tar_out) + return tar_out + + +def available_distrobutions() -> typing.List[str]: + """Return a list of available target distrobutions.""" + compile_files = (ROOT / "docker/compile").glob("Dockerfile.*") + dist_files = (ROOT / "docker/dist").glob("Dockerfile.dist.*") + compile_distros = {re.sub(r"^Dockerfile\.", "", f.name) for f in compile_files} + dist_distros = {re.sub(r"^Dockerfile\.dist\.", "", f.name) for f in dist_files} + distros = compile_distros & dist_distros + mismatch = (compile_distros | dist_distros) ^ distros + if mismatch: + logging.warning( + "docker distros found without both compile+dist files: %s", mismatch + ) + + return sorted(distros) + + +def build_target(tag: str, target: str) -> typing.List[Path]: + """Build docker images for the given target distrobutions.""" + debug = True + + distros = available_distrobutions() + logging.debug("available distro targets: %s", distros) + if target == "--all": + targets = distros + elif target not in distros: + raise BadDistrobutionName(f"distrobution {target!r} does not exist") + else: + targets = [target] + + out_dir = ROOT / "dist" + out_dir.mkdir(exist_ok=True) + + for target in targets: + scratch = Path(tempfile.mkdtemp(prefix=f"build-{target}-")) + log_path = scratch / "log.txt" + log = log_path.open("w") + logging.debug("building %s", target) + + try: + tar = build_dockerfile(scratch, target, tag, out_dir, log) + except Exception as e: + logging.error( + "an error occurred building %s. See log in %s", target, log_path + ) + if isinstance(e, subprocess.CalledProcessError): + raise BuildFailure("failure in underlying build scripts") from e + raise + + if not debug: + shutil.rmtree(scratch) + yield tar + + +def cli(args=sys.argv): + operation = sys.argv[1] + if operation == "list": + print("\n".join(available_distrobutions())) + elif operation == "build": + tag, target = sys.argv[2:] + try: + for archive in build_target(tag=tag, target=target): + print(archive) + except BuildFailure as e: + logging.error(e) + pass + else: + logging.error("unknown operation %r", operation) + + +if __name__ == "__main__": + configure_logging(debug=True) + cli() diff --git a/scripts/travis/k8s.sh b/scripts/travis/k8s.sh new file mode 100755 index 00000000000..9cb5af76e12 --- /dev/null +++ b/scripts/travis/k8s.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +:<<'DOC' +set NO_CACHE=1 to always rebuild images. + +DOC +set -o errexit -o nounset -o pipefail +TAG=test +HERE="$(cd "$(dirname "$0")"; pwd -P)" +ROOT="$(cd "$HERE/../.."; pwd -P)" + +function bazel_file { + # bazel_file VAR_NAME //some/build:target + # this will set VAR_NAME to the path of the build artefact + local var="${1:?}" + local ref="${2:?}" + local path="$(bazel info bazel-genfiles)/$(echo "${ref##//}" | tr ':' '/')" + bazel build "$ref" + eval "$var=$path" +} + +function kind_images { + # list all images in the kind registry + docker exec -it kind-control-plane crictl images +} + +function install_helm3 { + pushd /tmp + curl --location https://get.helm.sh/helm-v3.2.1-linux-amd64.tar.gz --output helm.tar.gz + tar --extract --file=helm.tar.gz --strip-components=1 linux-amd64/helm + mv helm ~/.local/bin/ + popd +} + +function action { + ( + tput setaf 4; + echo "[$(date --rfc-3339=seconds)] $*"; + tput sgr0 + ) > /dev/stderr +} + + +function create_cluster { + # trap "kind delete cluster" EXIT + if [ -z "$(kind get clusters)" ]; then + action "Creating kind cluster" + kind create cluster --config="$0.kind.yaml" + fi +} + +function get_image { + # cannot use `bazel_file heron_archive //scripts/images:heron.tar` as not distro image + local tag="$TAG" + local distro="${1:?}" + local out + local expected="$ROOT/dist/heron-docker-$tag-$distro.tar" + if [ -f "$expected" ] && [ -z "${NO_CACHE-}" ]; then + action "Using pre-existing heron image" + out="$expected" + else + action "Creating heron image" + local gz="$(scripts/release/docker-images build test debian10)" + # XXX: must un .gz https://github.com/kubernetes-sigs/kind/issues/1636 + gzip --decompress "$gz" + out="${gz%%.gz}" + fi + archive="$out" +} + +create_cluster + +get_image debian10 +heron_archive="$archive" +action "Loading heron docker image" +kind load image-archive "$heron_archive" +#image_heron="docker.io/bazel/scripts/images:heron" +#image_heron="$heron_image" +image_heron="heron/heron:$TAG" + +action "Loading bookkeeper image" +image_bookkeeper="docker.io/apache/bookkeeper:4.7.3" +docker pull "$image_bookkeeper" +kind load docker-image "$image_bookkeeper" + +action "Deploying heron with helm" +# install heron in kind using helm +bazel_file helm_yaml //scripts/packages:index.yaml +helm install heron "$(dirname "$helm_yaml")/heron-0.0.0.tgz" \ + --set image="$image_heron" \ + --set imagePullPolicy=IfNotPresent \ + --set bookieReplicas=1 \ + --set zkReplicas=1 diff --git a/scripts/travis/k8s.sh.kind.yaml b/scripts/travis/k8s.sh.kind.yaml new file mode 100644 index 00000000000..f76d19f36a2 --- /dev/null +++ b/scripts/travis/k8s.sh.kind.yaml @@ -0,0 +1,4 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane diff --git a/vagrant/local-ci.sh b/vagrant/local-ci.sh index ef4a5e55888..63617bdf4a5 100755 --- a/vagrant/local-ci.sh +++ b/vagrant/local-ci.sh @@ -19,7 +19,7 @@ cd "$HERE" state="$(vagrant status master --machine-readable | grep master,state, | cut -d, -f4)" if [ "$state" != "running" ]; then - vagrant resume master + vagrant up master fi From 8cad7772ebea7d4f65c41d435e246d4c5748986d Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sun, 31 May 2020 19:35:08 +0100 Subject: [PATCH 19/22] Fix helm chart --- deploy/kubernetes/helm/templates/tools.yaml | 3 ++- deploy/kubernetes/helm/values.yaml.template | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/helm/templates/tools.yaml b/deploy/kubernetes/helm/templates/tools.yaml index 64d57b7323b..2d3b8d31802 100644 --- a/deploy/kubernetes/helm/templates/tools.yaml +++ b/deploy/kubernetes/helm/templates/tools.yaml @@ -17,6 +17,7 @@ {{- $platform := .Values.platform -}} {{- $jobReplicas := .Values.jobReplicas -}} +{{- $defaultUrl := (printf "/api/v1/namespaces/%s/services/%s-ui:8889/proxy" .Release.Namespace .Release.Name) -}} {{- $apiServerMemory := .Values.apiServerMemory }} apiVersion: v1 @@ -115,7 +116,7 @@ spec: - >- heron-ui --port=8889 - --base_url={{ .Values.heron.url }} + --base_url={{ $defaultUrl }} - name: heron-apiserver image: {{ .Values.image }} imagePullPolicy: {{ .Values.imagePullPolicy }} diff --git a/deploy/kubernetes/helm/values.yaml.template b/deploy/kubernetes/helm/values.yaml.template index ae310beeb88..313211b1b25 100644 --- a/deploy/kubernetes/helm/values.yaml.template +++ b/deploy/kubernetes/helm/values.yaml.template @@ -39,7 +39,7 @@ jobReplicas: 1 # amount of memory to provide for API server apiServerMemory: 512M heron: - url: /api/v1/namespaces/{{ .Release.Namespace }}/services/{{ .Release.Name }}-ui:8889/proxy + url: ~ # Topologies uploader uploader: class: dlog # s3 From ede14374b01a8b77fbb766f9135a9da9246c3b88 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 6 Jun 2020 13:22:55 +0100 Subject: [PATCH 20/22] bytes vs str fix --- heron/tools/cli/src/python/cliconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heron/tools/cli/src/python/cliconfig.py b/heron/tools/cli/src/python/cliconfig.py index 2daf6f0d309..f88fd70ebc6 100644 --- a/heron/tools/cli/src/python/cliconfig.py +++ b/heron/tools/cli/src/python/cliconfig.py @@ -79,7 +79,7 @@ def _save_or_remove(config, cluster): config_directory = get_config_directory(cluster) if not os.path.isdir(config_directory): os.makedirs(config_directory) - with open(cluster_config_file, 'wb') as cf: + with open(cluster_config_file, 'w') as cf: yaml.dump(config, cf, default_flow_style=False) else: if os.path.isfile(cluster_config_file): From 525aedfeeac845d28066b28ec7952465ce676776 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 4 Jul 2020 22:21:53 +0100 Subject: [PATCH 21/22] Mention Python 3.6 requirement in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 84a2f251706..77f949d8ee3 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Confluence: https://cwiki.apache.org/confluence/display/HERON #### Heron Requirements: * Java 11 - * Python 3 + * Python 3.6 * Bazel 3.0.0 ## Contact From d280a75e8c0cae1bdb03d1d4a82e9c84977d598b Mon Sep 17 00:00:00 2001 From: huijunwu Date: Sun, 5 Jul 2020 06:49:28 -0700 Subject: [PATCH 22/22] updatedockerfile --- deploy/kubernetes/minikube/apiserver.yaml | 6 +++--- deploy/kubernetes/minikube/bookkeeper.yaml | 7 +++++-- docker/dist/Dockerfile.dist.ubuntu14.04 | 1 + docker/dist/Dockerfile.dist.ubuntu16.04 | 1 + docker/dist/Dockerfile.dist.ubuntu18.04 | 1 + docker/dist/Dockerfile.dist.ubuntu20.04 | 1 + 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/deploy/kubernetes/minikube/apiserver.yaml b/deploy/kubernetes/minikube/apiserver.yaml index 502f746ab2c..b6e8df7aca0 100644 --- a/deploy/kubernetes/minikube/apiserver.yaml +++ b/deploy/kubernetes/minikube/apiserver.yaml @@ -64,7 +64,7 @@ spec: initContainers: - name: init-heron-apiserver image: apache/bookkeeper:4.7.3 - command: ['sh', '-c', '/opt/bookkeeper/bin/dlog admin bind -l /ledgers -s zookeeper:2181 -c distributedlog://zookeeper:2181/heron'] + command: ['sh', '-c', '/opt/bookkeeper/bin/dlog admin bind -l /ledgers -s zookeeper:2181 -c distributedlog://zookeeper:2181/heronbkdl'] containers: - name: heron-apiserver image: heron/heron:latest @@ -79,9 +79,9 @@ spec: -D heron.executor.docker.image=heron/heron:latest -D heron.class.uploader=org.apache.heron.uploader.dlog.DLUploader -D heron.uploader.dlog.topologies.num.replicas=1 - -D heron.uploader.dlog.topologies.namespace.uri=distributedlog://zookeeper:2181/heron + -D heron.uploader.dlog.topologies.namespace.uri=distributedlog://zookeeper:2181/heronbkdl -D heron.statefulstorage.classname=org.apache.heron.statefulstorage.dlog.DlogStorage - -D heron.statefulstorage.dlog.namespace.uri=distributedlog://zookeeper:2181/heron + -D heron.statefulstorage.dlog.namespace.uri=distributedlog://zookeeper:2181/heronbkdl --- apiVersion: v1 diff --git a/deploy/kubernetes/minikube/bookkeeper.yaml b/deploy/kubernetes/minikube/bookkeeper.yaml index f5778c3c303..9e1d80fdc5f 100644 --- a/deploy/kubernetes/minikube/bookkeeper.yaml +++ b/deploy/kubernetes/minikube/bookkeeper.yaml @@ -65,6 +65,11 @@ spec: envFrom: - configMapRef: name: bookie-config + volumeMounts: + - name: journal-disk + mountPath: /bookkeeper/data/journal + - name: ledgers-disk + mountPath: /bookkeeper/data/ledgers containers: - name: bookie image: apache/bookkeeper:4.7.3 @@ -91,13 +96,11 @@ spec: valueFrom: fieldRef: fieldPath: status.hostIP - volumeMounts: - name: journal-disk mountPath: /bookkeeper/data/journal - name: ledgers-disk mountPath: /bookkeeper/data/ledgers - volumes: # Mount local disks - name: journal-disk diff --git a/docker/dist/Dockerfile.dist.ubuntu14.04 b/docker/dist/Dockerfile.dist.ubuntu14.04 index 259a818504f..4bc93e01354 100644 --- a/docker/dist/Dockerfile.dist.ubuntu14.04 +++ b/docker/dist/Dockerfile.dist.ubuntu14.04 @@ -22,6 +22,7 @@ RUN apt-get -y update \ curl \ netcat-openbsd \ python3 \ + python3-distutils \ software-properties-common \ supervisor \ unzip \ diff --git a/docker/dist/Dockerfile.dist.ubuntu16.04 b/docker/dist/Dockerfile.dist.ubuntu16.04 index 9c8e5659030..deb97d8a4fd 100644 --- a/docker/dist/Dockerfile.dist.ubuntu16.04 +++ b/docker/dist/Dockerfile.dist.ubuntu16.04 @@ -22,6 +22,7 @@ RUN apt-get -y update \ curl \ netcat-openbsd \ python3 \ + python3-distutils \ software-properties-common \ supervisor \ unzip \ diff --git a/docker/dist/Dockerfile.dist.ubuntu18.04 b/docker/dist/Dockerfile.dist.ubuntu18.04 index 8c4e2c0f45d..7eaa7f8f06f 100644 --- a/docker/dist/Dockerfile.dist.ubuntu18.04 +++ b/docker/dist/Dockerfile.dist.ubuntu18.04 @@ -23,6 +23,7 @@ RUN apt-get -y update \ netcat-openbsd \ openjdk-11-jre-headless \ python3 \ + python3-distutils \ supervisor \ unzip \ && apt-get clean diff --git a/docker/dist/Dockerfile.dist.ubuntu20.04 b/docker/dist/Dockerfile.dist.ubuntu20.04 index 7a1b4d0cde6..8dc6224beb3 100644 --- a/docker/dist/Dockerfile.dist.ubuntu20.04 +++ b/docker/dist/Dockerfile.dist.ubuntu20.04 @@ -25,6 +25,7 @@ RUN apt-get -y update \ openjdk-11-jre-headless \ netcat-openbsd \ python3 \ + python3-distutils \ supervisor \ unzip \ && apt-get clean