Sync branches 251011-0050: rightlib to main

build/conf/custom_lint.conf

@@ -14,10 +14,8 @@ macro _ADD_JSON_EXPLICIT_LINTER_CHECK(Args...) {
 # tag:lint tag:test
 ### @usage: STYLE_JSON([DIRS dirs] [DIRS_RECURSE dirs_recurse])
 macro STYLE_JSON(DIRS[], DIRS_RECURSE[]) {
-    # XXX: _MAKEFILE_INCLUDE_LIKE_DEPS is extended to get linter wrapper exported to opensource
-    SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/tools/clang_format_json_linter/wrapper.py)
     SET(_VAR_JSON_LINT_FILES_SALT __DIRS__ ${DIRS} __DIRS_RECURSE__ ${DIRS_RECURSE})
     SET(_CUSTOM_JSON_LINT_FILES_GLOB uniq_json_lint_files_${hash:_VAR_JSON_LINT_FILES_SALT})
     _CUSTOM_LINT_FILES(${_CUSTOM_JSON_LINT_FILES_GLOB} json DIRS .${pre=/:DIRS} DIRS_RECURSE $DIRS_RECURSE)
     _ADD_JSON_EXPLICIT_LINTER_CHECK(NAME clang_format_json WRAPPER_SCRIPT tools/clang_format_json_linter/wrapper.py CONFIGS $JSON_LINTER_DEFAULT_CONFIGS FILES ${${_CUSTOM_JSON_LINT_FILES_GLOB}})
-}
+}

build/conf/python.conf

@@ -309,8 +309,6 @@ macro _ADD_PY_LINTER_CHECK(Args...) {
 ###
 ### Not an actual linter, used for dummy linter demonstration
 macro STYLE_DUMMY() {
-    # XXX: _MAKEFILE_INCLUDE_LIKE_DEPS is extended to get linter wrapper exported to opensource
-    SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/tools/dummy_linter/__main__.py)
     _ADD_PY_LINTER_CHECK(NAME dummy_linter WRAPPER_SCRIPT tools/dummy_linter/__main__.py CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS)
 }
 
@@ -320,8 +318,6 @@ macro STYLE_DUMMY() {
 ### Check python3 sources for style issues using black.
 macro STYLE_PYTHON(CONFIG_TYPE="") {
     .ALLOWED_IN_LINTERS_MAKE=yes
-    # XXX: _MAKEFILE_INCLUDE_LIKE_DEPS is extended to get linter wrapper exported to opensource
-    SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/tools/black_linter/wrapper.py)
     _ADD_PY_LINTER_CHECK(NAME black WRAPPER_SCRIPT tools/black_linter/wrapper.py FILE_PROCESSING_TIME $BLACK_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS CONFIG_TYPE $CONFIG_TYPE)
 }
 
@@ -331,8 +327,6 @@ macro STYLE_PYTHON(CONFIG_TYPE="") {
 ### Check python3 sources for style issues using ruff. `CHECK_FORMAT` enables `ruff format` check.
 macro STYLE_RUFF(CONFIG_TYPE="", CHECK_FORMAT?"yes":"no") {
     .ALLOWED_IN_LINTERS_MAKE=yes
-    # XXX: _MAKEFILE_INCLUDE_LIKE_DEPS is extended to get linter wrapper exported to opensource
-    SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/tools/ruff_linter/wrapper.py)
     _ADD_PY_LINTER_CHECK(NAME ruff WRAPPER_SCRIPT tools/ruff_linter/wrapper.py FILE_PROCESSING_TIME $RUFF_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS CONFIG_TYPE $CONFIG_TYPE EXTRA_PARAMS check_format=${CHECK_FORMAT})
 }
 
@@ -341,8 +335,6 @@ macro STYLE_RUFF(CONFIG_TYPE="", CHECK_FORMAT?"yes":"no") {
 ###
 ### Check python3 sources for style issues using flake8.
 macro STYLE_FLAKE8() {
-    # XXX: _MAKEFILE_INCLUDE_LIKE_DEPS is extended to get linter wrapper exported to opensource
-    SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/tools/flake8_linter/__main__.py)
     _ADD_PY_LINTER_CHECK(NAME flake8 WRAPPER_SCRIPT tools/flake8_linter/__main__.py FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS FLAKE_MIGRATIONS_CONFIG $FLAKE_MIGRATIONS_CONFIG EXTRA_PARAMS $FLAKE_EXTRA_PARAMS)
 }
 
@@ -351,8 +343,6 @@ macro STYLE_FLAKE8() {
 ###
 ### Check python3 sources for style issues using flake8.
 macro STYLE_PY2_FLAKE8() {
-    # XXX: _MAKEFILE_INCLUDE_LIKE_DEPS is extended to get linter wrapper exported to opensource
-    SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/tools/flake8_linter/__main__.py)
     _ADD_PY_LINTER_CHECK(NAME py2_flake8 WRAPPER_SCRIPT tools/flake8_linter/__main__.py FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS FLAKE_MIGRATIONS_CONFIG $FLAKE_MIGRATIONS_CONFIG EXTRA_PARAMS $FLAKE_EXTRA_PARAMS)
 }
 

build/conf/ts/ts_test.conf

@@ -8,15 +8,9 @@ _TS_TEST_DEPENDS_ON_BUILD=
 
 _TS_TEST_COPY_NODEJS_CMD=
 
-# We have to rename node_modules.tar to workspace_node_modules.tar,
-# so TS_TEST_JEST module has it's own unique output.
 # TS_TEST_JEST_FOR module has to output all files required for test run.
 TS_TEST_JEST_CMD=$TOUCH_UNIT \
     && $_TS_TEST_COPY_NODEJS_CMD \
-    && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS --nm-bundle yes create-node-modules \
-    --moddir $TS_TEST_FOR_PATH \
-    $_NODE_MODULES_INOUTS ${hide:PEERS} \
-    && ${cwd:BINDIR} $MOVE_FILE $TS_TEST_NM ${output:"workspace_node_modules.tar"} \
     ${hide;kv:"p TS_JST"} ${hide;kv:"pc magenta"}
 
 ### @usage: TS_TEST_JEST_FOR(Path)
@@ -43,9 +37,8 @@ module TS_TEST_JEST_FOR: _TS_TEST_BASE {
     # nots.py will decide if we need to depend on the testing target module output
     _DEPENDS_ON_MOD()
     _PEERDIR_TS_RESOURCE(nodejs jest)
-    _TS_TEST_FOR_CONFIGURE(jest jest.config.js workspace_node_modules.tar)
-
     _TS_ADD_NODE_MODULES_FOR_BUILDER()
+    _TS_TEST_FOR_CONFIGURE(jest jest.config.js workspace_node_modules.tar)
 }
 
 TS_TEST_HERMIONE_CMD=$TOUCH_UNIT ${hide:PEERS} \
@@ -190,8 +183,8 @@ macro _SETUP_EXTRACT_OUTPUT_TARS_RECIPE(FOR_PATH) {
     USE_RECIPE(devtools/frontend_build_platform/nots/recipes/extract_output_tars/recipe $FOR_PATH)
 }
 
-macro _SETUP_INSTALL_NODE_MODULES_RECIPE() {
-    USE_RECIPE(devtools/frontend_build_platform/nots/recipes/install_node_modules/recipe $NOTS_TOOL_BASE_ARGS --nm-bundle no)
+macro _SETUP_INSTALL_NODE_MODULES_RECIPE(FOR_PATH) {
+    USE_RECIPE(devtools/frontend_build_platform/nots/recipes/install_node_modules/recipe $NOTS_TOOL_BASE_ARGS --nm-bundle no --for-path $FOR_PATH)
 }
 
 

build/external_resources/gdb/a.yaml

@@ -76,23 +76,6 @@ ci:
       - service: ya_make
 
   releases:
-    release-gdb:
-      title: "Release gdb14"
-      flow: build-and-release-gdb
-      flow-vars:
-        ix_package: gdb/14
-        toolchain_name: gdb14
-        mapping_path: build/external_resources/gdb/resources.json
-      flow-vars-ui:
-        schema:
-          <<: *ui-vars
-      filters:
-        - discovery: dir
-          abs-paths:
-            - devtools/gdb/**
-            - toolchains/ynd/bin/gdb/**
-            - toolchains/ynd/gdb/**
-
     release-gdb-16:
       title: "Release gdb16"
       flow: build-and-release-gdb

build/mapping.conf.json

@@ -625,6 +625,7 @@
         "9909336016": "{registry_endpoint}/9909336016",
         "9920199096": "{registry_endpoint}/9920199096",
         "9943449381": "{registry_endpoint}/9943449381",
+        "9954964563": "{registry_endpoint}/9954964563",
         "5486731632": "{registry_endpoint}/5486731632",
         "5514350352": "{registry_endpoint}/5514350352",
         "5514360398": "{registry_endpoint}/5514360398",
@@ -2340,6 +2341,7 @@
         "9909336016": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "9920199096": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "9943449381": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
+        "9954964563": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
         "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
         "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",

build/platform/test_tool/host.ya.make.inc

@@ -1,12 +1,12 @@
 IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943473694)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9955009970)
 ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943471270)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9955006215)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943479211)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9955016778)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943468701)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9955002556)
 ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943476759)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9955013369)
 
 ENDIF()

build/platform/test_tool/host_os.ya.make.inc

@@ -1,12 +1,12 @@
 IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943445461)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9954958551)
 ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943442708)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9954955639)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943449381)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9954964563)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943440788)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9954952667)
 ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9943447475)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:9954961674)
 
 ENDIF()

build/plugins/_dart_fields.py

@@ -608,7 +608,9 @@ class LintWrapperScript:
     @classmethod
     def value(cls, unit, flat_args, spec_args):
         if spec_args.get('WRAPPER_SCRIPT'):
-            return spec_args['WRAPPER_SCRIPT'][0]
+            path = spec_args['WRAPPER_SCRIPT'][0]
+            unit.on_data_files(path)
+            return path
 
 
 class LintConfigs:
@@ -640,6 +642,7 @@ def _from_config_type(unit, spec_args):
     def python_configs(cls, unit, flat_args, spec_args):
         if config := cls._from_config_type(unit, spec_args):
             # specified by config type, autoincludes scheme
+            unit.on_data_files(config)
             return serialize_list([config])
 
         # default config
@@ -654,12 +657,14 @@ def python_configs(cls, unit, flat_args, spec_args):
         configs = [config]
         if linter_name in ('flake8', 'py2_flake8'):
             configs.extend(spec_args.get('FLAKE_MIGRATIONS_CONFIG', []))
+        unit.on_data_files(configs)
         return serialize_list(configs)
 
     @classmethod
     def cpp_configs(cls, unit, flat_args, spec_args):
         if config := cls._from_config_type(unit, spec_args):
             # specified by config type, autoincludes scheme
+            unit.on_data_files(config)
             return serialize_list([config])
 
         # default config
@@ -671,6 +676,7 @@ def cpp_configs(cls, unit, flat_args, spec_args):
             message = f"Default config in {default_configs_path} can't be found for a linter {linter_name}"
             raise DartValueError(message)
         assert_file_exists(unit, config)
+        unit.on_data_files(config)
         return serialize_list([config])
 
     @classmethod
@@ -851,7 +857,12 @@ def pytest(cls, unit, flat_args, spec_args):
 
     @classmethod
     def junit(cls, unit, flat_args, spec_args):
-        return 'junit5.test' if unit.get('MODULE_TYPE') == 'JUNIT5' else 'junit.test'
+        if unit.get('MODULE_TYPE') == 'JUNIT5':
+            return 'junit5.test'
+        elif unit.get('MODULE_TYPE') == 'JUNIT6':
+            return 'junit6.test'
+        else:
+            return 'junit.test'
 
 
 class Size:

build/plugins/nots.py

@@ -130,7 +130,7 @@ def on_do_ts_yndexing(self) -> None:
         Turn on code navigation indexing
         """
 
-    def on_setup_install_node_modules_recipe(self) -> None:
+    def on_setup_install_node_modules_recipe(self, args: UnitType.PluginArgs) -> None:
         """
         Setup test recipe to install node_modules before running tests
         """
@@ -603,7 +603,9 @@ def _setup_eslint(unit: NotsUnitType) -> None:
 
     unit.on_peerdir_ts_resource("eslint")
     user_recipes = unit.get_subst("TEST_RECIPES_VALUE")
-    unit.on_setup_install_node_modules_recipe()
+    pm = _create_pm(unit)
+
+    unit.on_setup_install_node_modules_recipe(pm.module_path)
 
     test_type = TsTestType.ESLINT
 
@@ -673,7 +675,9 @@ def _setup_tsc_typecheck(unit: NotsUnitType) -> None:
 
     unit.on_peerdir_ts_resource("typescript")
     user_recipes = unit.get_subst("TEST_RECIPES_VALUE")
-    unit.on_setup_install_node_modules_recipe()
+    pm = _create_pm(unit)
+
+    unit.on_setup_install_node_modules_recipe(pm.module_path)
 
     test_type = TsTestType.TSC_TYPECHECK
 
@@ -723,11 +727,13 @@ def _setup_stylelint(unit: NotsUnitType) -> None:
     from lib.nots.package_manager import constants
 
     recipes_value = unit.get_subst("TEST_RECIPES_VALUE")
-    unit.on_setup_install_node_modules_recipe()
+    pm = _create_pm(unit)
+
+    unit.on_setup_install_node_modules_recipe(pm.module_path)
 
     test_type = TsTestType.TS_STYLELINT
 
-    peers = _create_pm(unit).get_local_peers_from_package_json()
+    peers = pm.get_local_peers_from_package_json()
 
     deps = df.CustomDependencies.nots_with_recipies(unit, (peers,), {}).split()
     if deps:
@@ -929,8 +935,14 @@ def on_ts_test_for_configure(
     # user-defined recipes should be in the end
     user_recipes = unit.get_subst("TEST_RECIPES_VALUE").strip()
     unit.set(["TEST_RECIPES_VALUE", ""])
-    unit.on_setup_extract_node_modules_recipe([for_mod_path])
+
+    if test_runner in [TsTestType.JEST]:
+        unit.on_setup_install_node_modules_recipe([for_mod_path])
+    else:
+        unit.on_setup_extract_node_modules_recipe([for_mod_path])
+
     unit.on_setup_extract_output_tars_recipe([for_mod_path])
+
     __set_append(unit, "TEST_RECIPES_VALUE", user_recipes)
 
     build_root = "$B" if test_runner in [TsTestType.HERMIONE, TsTestType.PLAYWRIGHT_LARGE] else "$(BUILD_ROOT)"

build/plugins/yql_python_udf.py

@@ -27,7 +27,7 @@ def onregister_yql_python_udf(unit, *args):
     if not yql_python_dir:
         yql_python_dir = 'yql/essentials/udfs/common/python'
 
-    unit.onyql_abi_version(['2', '27', '0'])
+    unit.onyql_abi_version(['2', '44', '0'])
     unit.onpeerdir(['yql/essentials/udfs/common/python/python_udf'])
     unit.onpeerdir(['yql/essentials/public/udf'])
 

build/scripts/configure_file.py

@@ -9,39 +9,39 @@
 cmakeDef = "#cmakedefine"
 
 
-def replaceLine(l, varDict, define):
-    words = l.split()
+def replaceLine(line, varDict, define):
+    words = line.split()
     if words:
         if words[0] == cmakeDef:
-            sPos = l.find(cmakeDef)
+            sPos = line.find(cmakeDef)
             ePos = sPos + len(cmakeDef)
-            l = l[:sPos] + define + l[ePos:] + '\n'
+            line = line[:sPos] + define + line[ePos:] + '\n'
         if words[0] == cmakeDef01:
             var = words[1]
             cmakeValue = varDict.get(var)
             if cmakeValue == 'yes':
                 val = '1'
             else:
                 val = '0'
-            sPos = l.find(cmakeDef01)
-            ePos = l.find(var) + len(var)
-            l = l[:sPos] + define + ' ' + var + ' ' + val + l[ePos + 1 :] + '\n'
+            sPos = line.find(cmakeDef01)
+            ePos = line.find(var) + len(var)
+            line = line[:sPos] + define + ' ' + var + ' ' + val + line[ePos + 1 :] + '\n'
 
     finder = re.compile(".*?(@[a-zA-Z0-9_]+@).*")
     while True:
-        re_result = finder.match(l)
+        re_result = finder.match(line)
         if not re_result:
-            return l
+            return line
         key = re_result.group(1)[1:-1]
-        l = l[: re_result.start(1)] + varDict.get(key, '') + l[re_result.end(1) :]
+        line = line[: re_result.start(1)] + varDict.get(key, '') + line[re_result.end(1) :]
 
 
 def main(inputPath, outputPath, varDict):
     define = '#define' if os.path.splitext(outputPath)[1] != '.asm' else '%define'
     with open(outputPath, 'w') as output:
         with open(inputPath, 'r') as input:
-            for l in input:
-                output.write(replaceLine(l, varDict, define))
+            for line in input:
+                output.write(replaceLine(line, varDict, define))
 
 
 def usage():

build/scripts/coverage-info.py

@@ -98,15 +98,15 @@ def print_stat(da, fnda, teamcity_stat_output):
             tc_file.write("##teamcity[blockClosed name='Code Coverage Summary']\n")
 
 
-def chunks(l, n):
+def chunks(items, n):
     """
     >>> list(chunks(range(10), 3))
     [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
     >>> list(chunks(range(10), 5))
     [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
     """
-    for i in xrange(0, len(l), n):
-        yield l[i : i + n]
+    for i in xrange(0, len(items), n):
+        yield items[i : i + n]
 
 
 def combine_info_files(lcov, files, out_file):

build/scripts/extract_asrc.py

@@ -23,6 +23,5 @@ def main():
                 tar.extractall(path=args.output)
 
 
-
 if __name__ == '__main__':
     main()

build/scripts/extract_docs.py

@@ -6,7 +6,7 @@
 # Explicitly enable local imports
 # Don't forget to add imported scripts to inputs of the calling command!
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-import process_command_files as pcf
+import process_command_files as pcf  # noqa: E402
 
 
 def parse_args():

build/scripts/fetch_from_archive.py

@@ -7,7 +7,7 @@
 # Explicitly enable local imports
 # Don't forget to add imported scripts to inputs of the calling command!
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-import fetch_from
+import fetch_from  # noqa: E402
 
 
 def parse_args():