Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sql alchemy #992

Open
wants to merge 29 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ jobs:

test_no_extras:
needs: [pre-commit, pretest]
runs-on: ubuntu-latest
runs-on: [ubuntu-latest]
steps:
- uses: actions/checkout@v1
- name: Set up Python 3.9
Expand Down
8 changes: 7 additions & 1 deletion docs/src/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,13 @@

# -- Autodoc configuration -----------------------------------------------

autodoc_mock_imports = ["_version", "utils._appdirs", "nevergrad", "torch"]
autodoc_mock_imports = [
"_version",
"utils._appdirs",
"nevergrad",
"torch",
"sqlalchemy",
]

# -- Gallery configuration -----------------------------------------------

Expand Down
2 changes: 2 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
],
"dask": ["dask[complete]"],
"track": ["track @ git+https://github.com/Delaunay/track@master#egg=track"],
"sqlalchemy": ["sqlalchemy"],
"profet": ["emukit", "GPy", "torch", "pybnn"],
"configspace": ["ConfigSpace"],
"ax": [
Expand Down Expand Up @@ -137,6 +138,7 @@
"BaseStorageProtocol": [
"track = orion.storage.track:Track",
"legacy = orion.storage.legacy:Legacy",
"sqlalchemy = orion.storage.sql:SQLAlchemy",
],
"BaseExecutor": [
"singleexecutor = orion.executor.single_backend:SingleExecutor",
Expand Down
8 changes: 8 additions & 0 deletions src/orion/core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,14 @@ def define_storage_config(config):
"type", option_type=str, default="legacy", env_var="ORION_STORAGE_TYPE"
)

storage_config.add_option(
"uri", option_type=str, default="", env_var="ORION_STORAGE_URI"
)

storage_config.add_option(
"token", option_type=str, default="", env_var="ORION_STORAGE_TOKEN"
)

config.storage = storage_config

define_database_config(config.storage)
Expand Down
3 changes: 3 additions & 0 deletions src/orion/core/io/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,9 @@ def to_dict(self):
def from_dict(self, config):
"""Set the configuration from a dictionary"""

if config is None:
return

logger.debug("Setting config to %s", config)
logger.debug("Config was %s", repr(self))

Expand Down
3 changes: 2 additions & 1 deletion src/orion/core/io/database/pickleddb.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from filelock import FileLock, SoftFileLock, Timeout

import orion.core
import orion.core.utils.compat
from orion.core.io.database import Database, DatabaseTimeout
from orion.core.io.database.ephemeraldb import EphemeralDB

Expand Down Expand Up @@ -227,7 +228,7 @@ def _dump_database(self, database):
log.error("because (value %s) in (field: %s) is not pickable", value, key)
raise

os.rename(tmp_file, self.host)
orion.core.utils.compat.replace(tmp_file, self.host)

@contextmanager
def locked_database(self, write=True):
Expand Down
8 changes: 5 additions & 3 deletions src/orion/core/io/experiment_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,14 +77,14 @@

import copy
import datetime
import getpass
import logging
import pprint
import sys
import typing
from typing import Any, TypeVar

import orion.core
import orion.core.utils.compat
from orion.algo.base import BaseAlgorithm, algo_factory
from orion.algo.space import Space
from orion.core.evc.adapters import BaseAdapter
Expand Down Expand Up @@ -864,7 +864,7 @@ def _branch_experiment(self, experiment, conflicts, version, branching_arguments

return self.create_experiment(mode="x", **config)

# pylint: disable=too-many-arguments
# pylint: disable=too-many-arguments,too-many-locals
def create_experiment(
self,
name: str,
Expand Down Expand Up @@ -934,9 +934,11 @@ def _default(v: T | None, default: V) -> T | V:
knowledge_base=knowledge_base,
)

username = orion.core.utils.compat.getuser()

max_broken = _default(max_broken, orion.core.config.experiment.max_broken)
working_dir = _default(working_dir, orion.core.config.experiment.working_dir)
metadata = _default(metadata, {"user": _default(user, getpass.getuser())})
metadata = _default(metadata, {"user": _default(user, username)})
refers = _default(refers, dict(parent_id=None, root_id=None, adapter=[]))
refers["adapter"] = _instantiate_adapters(refers.get("adapter", [])) # type: ignore

Expand Down
2 changes: 1 addition & 1 deletion src/orion/core/io/interactive_commands/branching_prompt.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@
import functools
import io
import os
import readline
import shlex
import traceback

from orion.algo.space import Dimension
from orion.core.evc import adapters, conflicts
from orion.core.utils.compat import readline
from orion.core.utils.diff import green, red

readline.set_completer_delims(" ")
Expand Down
8 changes: 5 additions & 3 deletions src/orion/core/io/resolve_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

"""
import copy
import getpass
import hashlib
import logging
import os
Expand All @@ -15,6 +14,7 @@

import orion
import orion.core
import orion.core.utils.compat
from orion.core.io.orion_cmdline_parser import OrionCmdlineParser
from orion.core.utils.flatten import unflatten

Expand Down Expand Up @@ -267,7 +267,8 @@ def fetch_env_vars():

def fetch_metadata(user=None, user_args=None, user_script_config=None):
"""Infer rest information about the process + versioning"""
metadata = {"user": user if user else getpass.getuser()}
username = orion.core.utils.compat.getuser()
metadata = {"user": user if user else username}

metadata["orion_version"] = orion.core.__version__

Expand Down Expand Up @@ -300,7 +301,8 @@ def fetch_metadata(user=None, user_args=None, user_script_config=None):

def update_metadata(metadata):
"""Update information about the process + versioning"""
metadata.setdefault("user", getpass.getuser())
username = orion.core.utils.compat.getuser()
metadata.setdefault("user", username)
metadata["orion_version"] = orion.core.__version__

if not metadata.get("user_args"):
Expand Down
60 changes: 60 additions & 0 deletions src/orion/core/utils/compat.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
"""Windows compatibility utilities"""
import os
import time


def getuser():
"""getpass use pwd which is UNIX only"""

if os.name == "nt":
return os.getlogin()

import getpass

return getpass.getuser()


# pylint: disable=too-few-public-methods
class _readline:
# pylint: disable=no-method-argument
def set_completer_delims(*args, **kwargs):
"""Fake method for windows"""


def get_readline():
"""Fake readline interface, readline is UNIX only"""
if os.name == "nt":
return _readline

import readline

return readline


readline = get_readline()


def replace(old, new, tries=3, sleep=0.01):
"""Windows file replacing is more strict than linux"""
if os.name != "nt":
# Rename on UNIX is practically atomic
# so we use that
os.rename(old, new)
return

# Rename raise an exception on windows if the file exists
# so we have to use replace
#
# if the file is open already windows will raise permission error
# even if the lock was free, waiting a bit usually fix the issue
exception = None
for _ in range(tries):
try:
os.replace(old, new)
return
except PermissionError as exc:
time.sleep(sleep)
exception = exc

if exception:
raise exception
2 changes: 1 addition & 1 deletion src/orion/storage/legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ class Legacy(BaseStorageProtocol):

"""

def __init__(self, database=None, setup=True):
def __init__(self, database=None, setup=True, **kwargs):
self._db = setup_database(database)

if setup:
Expand Down
19 changes: 19 additions & 0 deletions src/orion/storage/sql.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
IMPORT_ERROR = None
try:
from orion.storage.sql_impl import SQLAlchemy as SQLAlchemyImpl

HAS_SQLALCHEMY = True
except ModuleNotFoundError as err:
IMPORT_ERROR = err
HAS_SQLALCHEMY = False


if IMPORT_ERROR is not None:
from orion.storage.base import BaseStorageProtocol

class SQLAlchemy(BaseStorageProtocol):
def __init__(self, uri, token=None, **kwargs):
raise IMPORT_ERROR

else:
SQLAlchemy = SQLAlchemyImpl
Loading