Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

adds condition to check XDG_DATA_HOME to set global_dir value #2361

Open
wants to merge 1 commit into
base: devel
Choose a base branch
from
Open

adds condition to check XDG_DATA_HOME to set global_dir value #2361

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions dlt/common/runtime/run_context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import tempfile
import warnings
from types import ModuleType
from typing import Any, ClassVar, Dict, List, Optional

Expand Down Expand Up @@ -137,9 +138,10 @@ def plug_run_context_impl(
def global_dir() -> str:
"""Gets default directory where pipelines' data (working directories) will be stored
1. if DLT_DATA_DIR is set in env then it is used
2. in user home directory: ~/.dlt/
3. if current user is root: in /var/dlt/
4. if current user does not have a home directory: in /tmp/dlt/
2. if XDG_DATA_DIR is set in env then it is used
3. in user home directory: ~/.dlt/
4. if current user is root: in /var/dlt/
5. if current user does not have a home directory: in /tmp/dlt/
"""
if known_env.DLT_DATA_DIR in os.environ:
return os.environ[known_env.DLT_DATA_DIR]
Expand All @@ -154,7 +156,17 @@ def global_dir() -> str:
# no home dir - use temp
return os.path.join(tempfile.gettempdir(), "dlt")
else:
# if home directory is available use ~/.dlt/pipelines
# use XDG_DATA_HOME only if ~/.dlt doesn't exist
if os.environ["XDG_DATA_HOME"] is not None:
if not os.path.isdir(os.path.join(home, DOT_DLT)):
return os.path.join(os.environ["XDG_DATA_HOME"], "dlt")
else:
warnings.warn(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we use logger.warning in our codebase, maybe use that instead

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you mean dlt.common.logger, when I import it into run_context.py the conftest fails:

ImportError while loading conftest 'tests/common/runtime/conftest.py'.
tests/common/runtime/conftest.py:1: in <module>
    from tests.utils import preserve_environ
tests/utils.py:43: in <module>
    ALL_DESTINATIONS = dlt.config.get("ALL_DESTINATIONS", list) or [
dlt/common/configuration/accessors.py:39: in get
    value, _ = self._get_value(field, expected_type)
dlt/common/configuration/accessors.py:74: in _get_value
    for provider in self.config_providers:
dlt/common/configuration/accessors.py:104: in config_providers
    return [p for p in self._get_providers_from_context()]
dlt/common/configuration/accessors.py:64: in _get_providers_from_context
    return Container()[PluggableRunContext].providers.providers
dlt/common/configuration/container.py:67: in __getitem__
    item = spec()
dlt/common/configuration/specs/pluggable_run_context.py:90: in __init__
    self._plug(run_dir=None)
dlt/common/configuration/specs/pluggable_run_context.py:161: in _plug
    m = plugins.manager()
dlt/common/configuration/plugins.py:47: in manager
    return Container()[PluginContext].manager
dlt/common/configuration/container.py:67: in __getitem__
    item = spec()
dlt/common/configuration/plugins.py:31: in __init__
    self.manager.add_hookspecs(run_context)
E   AttributeError: 'function' object has no attribute 'get'

Without importing it, the tests run normally.

f"XDG_DATA_HOME is set to {os.environ['XDG_DATA_HOME']} but ~/.dlt already"
" exists. Using ~/.dlt"
)

# else default to ~/.dlt
return os.path.join(home, DOT_DLT)


Expand Down
Loading