diff --git a/deploy-agent/deployd/__init__.py b/deploy-agent/deployd/__init__.py index 68023a79ca..a5ec14b2eb 100644 --- a/deploy-agent/deployd/__init__.py +++ b/deploy-agent/deployd/__init__.py @@ -27,4 +27,4 @@ # 2: puppet applied successfully with changes PUPPET_SUCCESS_EXIT_CODES = [0, 2] -__version__ = '1.2.63' +__version__ = '1.2.64' diff --git a/deploy-agent/deployd/client/base_client.py b/deploy-agent/deployd/client/base_client.py index 2bb5130c0e..4bac47f22b 100644 --- a/deploy-agent/deployd/client/base_client.py +++ b/deploy-agent/deployd/client/base_client.py @@ -3,30 +3,29 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABCMeta, abstractmethod -from future.utils import with_metaclass +from abc import ABC, abstractmethod -class BaseClient(with_metaclass(ABCMeta, object)): +class BaseClient(ABC): """This class plays a role as an interface defining methods for agent to communicate with teletraan service. """ - + @abstractmethod def send_reports(self, env_reports=None): """Args: env_reports: a dict with env name as key and DeployStatus as value. Returns: - PingResponse describing next action for deploy agent. + PingResponse describing next action for deploy agent. """ pass diff --git a/deploy-agent/deployd/common/caller.py b/deploy-agent/deployd/common/caller.py index cbc008022b..dae2d19a0b 100644 --- a/deploy-agent/deployd/common/caller.py +++ b/deploy-agent/deployd/common/caller.py @@ -3,9 +3,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,11 +15,9 @@ import subprocess import traceback import logging -import time +import time from typing import Optional, Tuple -from future.utils import PY3 - log = logging.getLogger(__name__) @@ -32,12 +30,8 @@ def call_and_log(cmd, **kwargs) -> Tuple[Optional[str], str, Optional[int]]: output = "" start = time.time() try: - if PY3: - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, encoding='utf-8', **kwargs) - else: - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, **kwargs) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, encoding='utf-8', **kwargs) while process.poll() is None: line = process.stdout.readline() if line: diff --git a/deploy-agent/deployd/common/config.py b/deploy-agent/deployd/common/config.py index 4e6f22659c..3d94a0131e 100644 --- a/deploy-agent/deployd/common/config.py +++ b/deploy-agent/deployd/common/config.py @@ -1,4 +1,3 @@ -from __future__ import print_function # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/deploy-agent/deployd/common/executor.py b/deploy-agent/deployd/common/executor.py index 90b0736b8a..13aad17d98 100644 --- a/deploy-agent/deployd/common/executor.py +++ b/deploy-agent/deployd/common/executor.py @@ -3,9 +3,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,8 +22,6 @@ import traceback from typing import Tuple -from future.utils import PY3 - from deployd.common.types import DeployReport, PingStatus, PRE_STAGE_STEPS, AgentStatus log = logging.getLogger(__name__) @@ -111,14 +109,11 @@ def run_cmd(self, cmd, **kw) -> DeployReport: # sleep some seconds before next poll sleep_time = self._get_sleep_interval(start, self.PROCESS_POLL_INTERVAL) - if PY3: - # Wait up to sleep_time for the process to terminate (new in Python 3.3) - try: - process.wait(sleep_time) - except subprocess.TimeoutExpired: - pass - else: - time.sleep(sleep_time) + # Wait up to sleep_time for the process to terminate (new in Python 3.3) + try: + process.wait(sleep_time) + except subprocess.TimeoutExpired: + pass # finish executing sub process deploy_report.error_code = process.returncode @@ -202,14 +197,7 @@ def _graceful_shutdown(self, process) -> None: try: log.info('Gracefully shutdown currently running process with timeout {}'.format(self.TERMINATE_TIMEOUT)) os.killpg(process.pid, signal.SIGTERM) - if PY3: - process.wait(self.TERMINATE_TIMEOUT) - else: - start_time = datetime.datetime.now() - while process.poll() is None: - if (datetime.datetime.now() - start_time).seconds > self.TERMINATE_TIMEOUT: - raise Exception('Timed out while waiting for the process to shutdown') - time.sleep(min(self.PROCESS_POLL_INTERVAL, self.TERMINATE_TIMEOUT)) + process.wait(self.TERMINATE_TIMEOUT) except Exception as e: log.debug('Failed to gracefully shutdown: {}'.format(e)) Executor._kill_process(process) diff --git a/deploy-agent/deployd/common/single_instance.py b/deploy-agent/deployd/common/single_instance.py index 33713f8939..9a345dd8ae 100644 --- a/deploy-agent/deployd/common/single_instance.py +++ b/deploy-agent/deployd/common/single_instance.py @@ -1,5 +1,3 @@ -from __future__ import print_function -from __future__ import absolute_import # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,10 +15,8 @@ import logging import os import stat -import errno import fcntl from . import utils -from future.utils import PY3 import tempfile log = logging.getLogger(__name__) LOCKFILE_DIR = '/var/lock' @@ -32,12 +28,12 @@ def __init__(self) -> None: appname = 'deploy-agent' lockfile_name = '.{}.lock'.format(appname) self._create_lock_dir() - # Backward compatibility as old deploy agent versions use lock file in /tmp. - # Use the old lock file if it exists + # Backward compatibility as old deploy agent versions use lock file in /tmp. + # Use the old lock file if it exists tmp_lockfile_path = os.path.join(tempfile.gettempdir(), lockfile_name) if os.path.exists(tmp_lockfile_path): lockfile_path = tmp_lockfile_path - else: + else: lockfile_path = os.path.join(LOCKFILE_DIR, lockfile_name) lockfile_flags = os.O_WRONLY | os.O_CREAT # This is 0o222, i.e. 146, --w--w--w- @@ -61,12 +57,4 @@ def __init__(self) -> None: utils.exit_abruptly(1) def _create_lock_dir(self) -> None: - if PY3: - os.makedirs(LOCKFILE_DIR, exist_ok=True) - else: - # Need to handle the case when lock dir exists in py2 - try: - os.makedirs(LOCKFILE_DIR) # py2 - except OSError as e: - if e.errno != errno.EEXIST: - raise + os.makedirs(LOCKFILE_DIR, exist_ok=True) diff --git a/deploy-agent/deployd/common/utils.py b/deploy-agent/deployd/common/utils.py index b6b63e9294..192feadfe9 100644 --- a/deploy-agent/deployd/common/utils.py +++ b/deploy-agent/deployd/common/utils.py @@ -1,4 +1,3 @@ -from __future__ import print_function # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/deploy-agent/deployd/download/download_helper_factory.py b/deploy-agent/deployd/download/download_helper_factory.py index 9db07046f2..78ee85a262 100644 --- a/deploy-agent/deployd/download/download_helper_factory.py +++ b/deploy-agent/deployd/download/download_helper_factory.py @@ -16,7 +16,7 @@ from typing import Optional import boto3 -from future.moves.urllib.parse import urlparse +from urllib.parse import urlparse from deployd.download.download_helper import DownloadHelper from deployd.download.s3_download_helper import S3DownloadHelper diff --git a/deploy-agent/deployd/download/http_download_helper.py b/deploy-agent/deployd/download/http_download_helper.py index 5f0dd3d18e..96a9cdc33e 100644 --- a/deploy-agent/deployd/download/http_download_helper.py +++ b/deploy-agent/deployd/download/http_download_helper.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -98,4 +97,4 @@ def validate_source(self) -> bool: return True else: log.error(f"{domain} is not in the allow list: {allow_list}.") - return False \ No newline at end of file + return False diff --git a/deploy-agent/deployd/staging/stager.py b/deploy-agent/deployd/staging/stager.py index 935ff76576..73c0456185 100644 --- a/deploy-agent/deployd/staging/stager.py +++ b/deploy-agent/deployd/staging/stager.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/deploy-agent/setup.py b/deploy-agent/setup.py index 060191e834..fd2937ce52 100644 --- a/deploy-agent/setup.py +++ b/deploy-agent/setup.py @@ -30,8 +30,7 @@ "gevent==24.2.1; python_version >= '3.12'", "lockfile==0.10.2", "boto3==1.34.134", - "python-daemon==2.0.6", - "future==0.18.2" + "python-daemon==2.0.6" ] setup( diff --git a/deploy-agent/tests/unit/deploy/client/test_base_client.py b/deploy-agent/tests/unit/deploy/client/test_base_client.py index c4aada1cdc..74c8fd5e48 100644 --- a/deploy-agent/tests/unit/deploy/client/test_base_client.py +++ b/deploy-agent/tests/unit/deploy/client/test_base_client.py @@ -13,8 +13,7 @@ # limitations under the License. import unittest -from abc import ABCMeta, abstractmethod -from future.utils import with_metaclass +from abc import ABC, abstractmethod from tests import TestCase from deployd.client.base_client import BaseClient @@ -34,7 +33,7 @@ def test_abc_equivalent_to_old(self): """ Make sure that new changes to base client extend the original class """ - class OldBaseClient(with_metaclass(ABCMeta, object)): + class OldBaseClient(ABC): @abstractmethod def send_reports(self, env_reports=None): pass diff --git a/deploy-board/deploy_board/templates/environs/env_tabs.tmpl b/deploy-board/deploy_board/templates/environs/env_tabs.tmpl index 108bda0420..50d738e8ea 100644 --- a/deploy-board/deploy_board/templates/environs/env_tabs.tmpl +++ b/deploy-board/deploy_board/templates/environs/env_tabs.tmpl @@ -1,4 +1,3 @@ - {% if envs|length < 11 %}
-{% else %} -This method can only provide an approximation of the start time. Therefore it is not + * suitable for high precision use cases. You shouldn't use a {@link LongTaskTimer} for tracking + * high precision durations anyways. + * + *
If for any reason the start time cannot be set, the current time will be used.
+ *
+ * @param startTime start time
+ * @return the sample with specified start time
+ */
+ public Sample start(Instant startTime) {
+ Sample sample = start();
+ try {
+ long timeLapsed = clock.wallTime() - startTime.toEpochMilli();
+ long monotonicStartTime = clock.monotonicTime() - timeLapsed * 1000000;
+ // The class `SampleImpl` is not visible, so we have to use reflection to set
+ // the start time.
+ Class> sampleImplClass = sample.getClass();
+ Field field = sampleImplClass.getDeclaredField("startTime");
+ field.setAccessible(true);
+ field.set(sample, monotonicStartTime);
+ } catch (Exception e) {
+ LOG.error("Failed to set start time, use current time instead", e);
+ }
+ return sample;
}
- return sample;
- }
}
diff --git a/deploy-service/universal/src/test/java/com/pinterest/teletraan/universal/metrics/micrometer/PinStatsLongTaskTimerTest.java b/deploy-service/universal/src/test/java/com/pinterest/teletraan/universal/metrics/micrometer/PinStatsLongTaskTimerTest.java
index 4949bf38ba..7d790b19fb 100644
--- a/deploy-service/universal/src/test/java/com/pinterest/teletraan/universal/metrics/micrometer/PinStatsLongTaskTimerTest.java
+++ b/deploy-service/universal/src/test/java/com/pinterest/teletraan/universal/metrics/micrometer/PinStatsLongTaskTimerTest.java
@@ -25,6 +25,7 @@
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
import io.micrometer.core.instrument.simple.SimpleConfig;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.BeforeEach;
@@ -47,7 +48,11 @@ protected LongTaskTimer newLongTaskTimer(
id, clock, getBaseTimeUnit(), distributionStatisticConfig);
}
};
- sut = (PinStatsLongTaskTimer) LongTaskTimer.builder("my.ltt").register(registry);
+ sut =
+ (PinStatsLongTaskTimer)
+ LongTaskTimer.builder("my.ltt")
+ .serviceLevelObjectives(Duration.ofSeconds(10))
+ .register(registry);
}
@Test
@@ -71,4 +76,26 @@ void withInput_start_inputAsStartTime() {
clock.addSeconds(5);
assertEquals(10 * 1000, sample.duration(TimeUnit.MILLISECONDS), 0.1);
}
+
+ @Test
+ void testGaugeHistogram() {
+ Sample sample = sut.start();
+ sut.start();
+ clock.addSeconds(1);
+
+ // both active and within SLO
+ assertEquals(2, sut.activeTasks());
+ assertEquals(2, sut.takeSnapshot().histogramCounts()[0].count());
+
+ // 1 remains active
+ sample.stop();
+ assertEquals(1, sut.activeTasks());
+ assertEquals(1, sut.takeSnapshot().histogramCounts()[0].count());
+
+ // remaining exceeds SLO
+ clock.addSeconds(10);
+
+ assertEquals(1, sut.activeTasks());
+ assertEquals(0, sut.takeSnapshot().histogramCounts()[0].count());
+ }
}
diff --git a/deploy-service/universal/src/test/java/com/pinterest/teletraan/universal/metrics/micrometer/PinStatsPublisherTest.java b/deploy-service/universal/src/test/java/com/pinterest/teletraan/universal/metrics/micrometer/PinStatsPublisherTest.java
index 192b0a0665..e048911bef 100644
--- a/deploy-service/universal/src/test/java/com/pinterest/teletraan/universal/metrics/micrometer/PinStatsPublisherTest.java
+++ b/deploy-service/universal/src/test/java/com/pinterest/teletraan/universal/metrics/micrometer/PinStatsPublisherTest.java
@@ -23,6 +23,7 @@
import io.micrometer.core.instrument.FunctionCounter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.LongTaskTimer;
+import io.micrometer.core.instrument.LongTaskTimer.Sample;
import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.Meter.Type;
import io.micrometer.core.instrument.MockClock;
@@ -32,7 +33,9 @@
import java.time.Duration;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,17 +101,16 @@ void histogramBucketsHaveCorrectBaseUnit() {
clock.add(config.step());
publisher.writeTimer(timer).forEach(LOG::debug);
+ Supplier