Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: reduce allowed migration time to 15 minutes and stop uploading #10232

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

ci: reduce allowed migration time to 15 minutes and stop uploading #10232

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 2 additions & 88 deletions .circleci/scripts/wait_for_perf_migration_upload_results.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,7 @@
import subprocess
import time

import psycopg2
import requests
from psycopg2 import extensions, sql

from determined.common import api, util
from determined.common.api import authentication, bindings, certs
Expand All @@ -18,11 +16,11 @@ def _wait_for_master() -> None:
cert = certs.Cert(noverify=True)
sess = api.UnauthSession("http://127.0.0.1:8080", cert).with_retry(0)

# 2 hours is the most a migration can take, with this setup.
# 15 minutes is the most a migration can take, with this setup.
# If a migration takes longer than that we have hit an issue a customer will likely hit too.
time_start = time.time()
time_last_report = time_start
while time_start - time.time() < 2 * 60 * 60:
while time_start - time.time() < 15 * 60:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This line may also be relevant:

determined/.circleci/real_config.yml

Line 2149 in 40a70cf

time: "120" # Wait two hours at most. Adjust this over time.

try:
r = sess.get("info")
if r.status_code == requests.codes.ok:
Expand All @@ -37,89 +35,5 @@ def _wait_for_master() -> None:
raise ConnectionError("Timed out connecting to Master")


def _upload_migration_length(conn: extensions.connection) -> None:
cert = certs.Cert(noverify=True)
sess = authentication.login("http://127.0.0.1:8080", "admin", "", cert)

migration_start_log = None
migration_end_log = None
no_migration_to_apply_log = None
for log in bindings.get_MasterLogs(sess):
if "running DB migrations from" in log.logEntry.message:
migration_start_log = log.logEntry
continue

match = re.search(r"migrated from (\d+) to (\d+)", log.logEntry.message)
if match:
from_version = int(match.group(1))
to_version = int(match.group(2))
migration_end_log = log.logEntry
continue

if "no migrations to apply" in log.logEntry.message:
no_migration_to_apply_log = log.logEntry

assert migration_start_log is not None
if no_migration_to_apply_log is not None:
print(
"got no migration to apply message (nothing to record) "
+ f"'{no_migration_to_apply_log.message}'"
)

indicate_file = "/tmp/no-migrations-needed"
print(f"creating file at {indicate_file} to indicate this")
pathlib.Path(indicate_file).touch()

assert migration_end_log is None
return

assert migration_end_log is not None
print(
f"migration start message '{migration_start_log.message}' "
+ f"at {migration_start_log.timestamp}"
)
print(f"migration end message '{migration_end_log.message}' at {migration_end_log.timestamp}")

start_ts = util.parse_protobuf_timestamp(migration_start_log.timestamp)
end_ts = util.parse_protobuf_timestamp(migration_end_log.timestamp)
duration = (end_ts - start_ts) / datetime.timedelta(microseconds=1)
print(f"migrating {from_version} to {to_version} took {duration}ms")

try:
commit = subprocess.check_output(
["git", "log", "-1", "--pretty=format:%H"], universal_newlines=True
).strip()
except subprocess.CalledProcessError:
commit = "unknown"

run_sql_query = sql.SQL(
"""
INSERT INTO migration_runs (commit, branch, duration_ms, from_version, to_version)
VALUES ({commit}, {branch}, {duration_ms}, {from_version}, {to_version})
RETURNING id;
"""
).format(
commit=sql.Literal(commit),
branch=sql.Literal(os.environ.get("CIRCLE_BRANCH", "unknown")),
duration_ms=sql.Literal(duration),
from_version=sql.Literal(from_version),
to_version=sql.Literal(to_version),
)

cursor = conn.cursor()
cursor.execute(run_sql_query)
conn.commit()


if __name__ == "__main__":
db_params = {
"host": os.environ["PERF_RESULT_DB_HOST"],
"user": os.environ["PERF_RESULT_DB_USER"],
"password": os.environ["PERF_RESULT_DB_PASS"],
"dbname": "postgres",
}
connection = psycopg2.connect(**db_params)

_wait_for_master()

_upload_migration_length(connection)
Loading