Improve Dependency. Not all nodes have ref yet.

src/downloader/download.py

@@ -122,35 +122,30 @@ def download_action_or_reusable_workflow(uses_string: str, repo: str) -> None:
     """
     with RedisConnection(Config.redis_sets_db) as sets_db:
         uses_string_obj = UsesString.analyze(uses_string=uses_string)
-        absolute_path = uses_string_obj.get_absolute_path(repo)
 
         if uses_string_obj.type == UsesStringType.REUSABLE_WORKFLOW:
             # If already scanned workflow - Have to check workflow db because only it contains the full workflow path.
             with RedisConnection(Config.redis_workflows_db) as workflows_db:
                 if (
                     workflows_db.get_value_from_hash(
-                        absolute_path, Config.redis_data_hash_field_name
+                        uses_string_obj.absolute_path_with_ref,
+                        Config.redis_data_hash_field_name,
                     )
                     is not None
                 ):
                     return
-            workflow_path, tag = (
-                absolute_path.split("@")
-                if uses_string_obj.ref
-                else (absolute_path, None)
-            )
-            url = get_repository_reusable_workflow(workflow_path, tag)
+            workflow_path, ref = uses_string_obj.absolute_path, uses_string_obj.ref
+            url = get_repository_reusable_workflow(workflow_path, ref)
         elif uses_string_obj.type == UsesStringType.ACTION:
             # If already scanned action
-            if sets_db.exists_in_set(Config.action_download_history_set, absolute_path):
+            if sets_db.exists_in_set(
+                Config.action_download_history_set,
+                uses_string_obj.absolute_path_with_ref,
+            ):
                 return
 
-            action_path, tag = (
-                absolute_path.split("@")
-                if uses_string_obj.ref
-                else (absolute_path, None)
-            )
-            url = get_repository_composite_action(action_path, tag)
+            action_path, ref = uses_string_obj.absolute_path, uses_string_obj.ref
+            url = get_repository_composite_action(action_path, ref)
         else:
             # Can happen with docker references.
             return
@@ -182,39 +177,49 @@ def download_action_or_reusable_workflow(uses_string: str, repo: str) -> None:
 
         # We look for dependant external actions.
         uses_strings = find_uses_strings(resp.text)
-        new_repo = get_repo_name_from_path(absolute_path)
+        new_repo = get_repo_name_from_path(uses_string_obj.absolute_path)
 
         for new_uses_string in uses_strings:
             # Some infinite loop I met in several repositories
-            new_full_path = UsesString.analyze(new_uses_string).get_absolute_path(
-                new_repo
-            )
-            if new_full_path == absolute_path:
+            if (
+                UsesString.analyze(new_uses_string).absolute_path_with_ref
+                == uses_string_obj.absolute_path
+            ):
                 continue
 
             download_action_or_reusable_workflow(
                 uses_string=new_uses_string, repo=new_repo
             )
 
         if uses_string_obj.type == UsesStringType.REUSABLE_WORKFLOW:
-            sets_db.insert_to_set(Config.workflow_download_history_set, absolute_path)
+            sets_db.insert_to_set(
+                Config.workflow_download_history_set,
+                uses_string_obj.absolute_path_with_ref,
+            )
             with RedisConnection(Config.redis_workflows_db) as workflows_db:
                 workflows_db.insert_to_hash(
-                    absolute_path, Config.redis_data_hash_field_name, resp.text
+                    uses_string_obj.absolute_path_with_ref,
+                    Config.redis_data_hash_field_name,
+                    resp.text,
                 )
                 workflows_db.insert_to_hash(
-                    absolute_path,
+                    uses_string_obj.absolute_path_with_ref,
                     Config.redis_url_hash_field_name,
                     convert_raw_github_url_to_github_com_url(url),
                 )
         else:  # UsesStringType.ACTION
-            sets_db.insert_to_set(Config.action_download_history_set, absolute_path)
+            sets_db.insert_to_set(
+                Config.action_download_history_set,
+                uses_string_obj.absolute_path_with_ref,
+            )
             with RedisConnection(Config.redis_actions_db) as actions_db:
                 actions_db.insert_to_hash(
-                    absolute_path, Config.redis_data_hash_field_name, resp.text
+                    uses_string_obj.absolute_path_with_ref,
+                    Config.redis_data_hash_field_name,
+                    resp.text,
                 )
                 actions_db.insert_to_hash(
-                    absolute_path,
+                    uses_string_obj.absolute_path_with_ref,
                     Config.redis_url_hash_field_name,
                     convert_raw_github_url_to_github_com_url(url),
                 )

src/downloader/gh_api.py

@@ -215,14 +215,14 @@ def get_ref_of_tag(repo: str, tag: str) -> Optional[str]:
 def get_download_url(
     path: str,
     file_suffixes: Optional[List[str]] = None,
-    tag: Optional[str] = None,
+    ref: Optional[str] = None,
 ) -> Optional[str]:
     """
     Retrieves the downloadable URL for a GitHub resource located at the given path.
 
     Parameters:
     - path (str): The repository path containing the resource, formatted as "owner/repo/relative_path_to_resource".
-    - tag (Optional[str]): The version tag of the resource. If None, the latest version is used.
+    - ref (Optional[str]): The ref or the tag of the resource. If None, the latest version is used.
     - file_suffixes (List[str]): List of possible file suffixes that the resource could have (e.g., ["action.yml", "action.yaml"]).
 
     Returns:
@@ -237,7 +237,7 @@ def get_download_url(
     headers["Authorization"] = f"Token {Config.github_token}"
 
     # Get ref of commit if tag is provided
-    ref = get_ref_of_tag(repo, tag) if tag else None
+    ref = get_ref_of_tag(repo, ref) if ref else None
 
     files_to_try = (
         [os.path.join(relative_path, fs) for fs in file_suffixes]
@@ -250,7 +250,7 @@ def get_download_url(
         # Otherwise, we can use the normal contents API
         action_download_url = (
             CONTENTS_BY_REF_URL.format(repo_path=repo, file_path=file_path, ref=ref)
-            if tag
+            if ref
             else CONTENTS_URL.format(repo_path=repo, file_path=file_path)
         )
 
@@ -265,15 +265,15 @@ def get_download_url(
         return r.json()["download_url"]
 
 
-def get_repository_composite_action(path: str, tag: Optional[str]) -> str:
+def get_repository_composite_action(path: str, ref: Optional[str]) -> str:
     """
     Retrieves the downloadable URL for a specific composite action located at the given path.
     """
-    return get_download_url(path, tag=tag, file_suffixes=ACTION_SUFFIXES)
+    return get_download_url(path, ref=ref, file_suffixes=ACTION_SUFFIXES)
 
 
-def get_repository_reusable_workflow(path: str, tag: str) -> str:
+def get_repository_reusable_workflow(path: str, ref: str) -> str:
     """
     Retrieves the downloadable URL for a specific reusable workflow located at the given path.
     """
-    return get_download_url(path, tag=tag, file_suffixes=[])
+    return get_download_url(path, ref=ref, file_suffixes=[])

src/indexer/index.py

@@ -2,7 +2,7 @@
 
 import yaml
 from yaml.constructor import Constructor
-
+from src.workflow_components.dependency import UsesString
 from src.storage.redis_connection import RedisConnection
 from src.config.config import Config
 from src.workflow_components.workflow import Workflow
@@ -83,7 +83,7 @@ def index_action_file(action: str) -> None:
                 return
 
             obj["path"] = action
-            obj["tag"] = action.split("@")[-1] if "@" in action else None
+            obj["ref"] = UsesString.get_ref_from_path_string(action)
             obj["url"] = url
 
             Config.graph.push_object(CompositeAction.from_dict(obj))
@@ -132,7 +132,7 @@ def index_workflow_file(workflow: str) -> None:
                 return
 
             obj["path"] = workflow
-            obj["tag"] = workflow.split("@")[-1] if "@" in workflow else None
+            obj["ref"] = UsesString.get_ref_from_path_string(workflow)
             obj["url"] = url
 
             Config.graph.push_object(Workflow.from_dict(obj))

src/workflow_components/composite_action.py

@@ -1,4 +1,3 @@
-from __future__ import annotations
 from typing import Optional
 
 from hashlib import md5
@@ -13,7 +12,7 @@
 from src.workflow_components.dependency import UsesString, UsesStringType
 
 
-def get_or_create_composite_action(path: str) -> CompositeAction:
+def get_or_create_composite_action(path: str) -> "CompositeAction":
     """Used when need to create relations with another action.
     If action wasn't indexed yet, we create a stub node,
     that will be enriched eventually.
@@ -39,7 +38,8 @@ class CompositeActionStep(GraphObject):
     ref = Property()
     shell = Property()
     with_prop = Property("with")
-    tag = Property()
+    url = Property()
+    ref = Property()
 
     action = RelatedTo("CompositeAction")
     reusable_workflow = RelatedTo(workflow.Workflow)
@@ -54,8 +54,8 @@ def from_dict(obj_dict) -> "CompositeActionStep":
         s = CompositeActionStep(_id=obj_dict["_id"], path=obj_dict["path"])
         s.url = obj_dict["url"]
 
-        if "tag" in obj_dict:
-            s.tag = obj_dict["tag"]
+        if "ref" in obj_dict:
+            s.ref = obj_dict["ref"]
         if "id" in obj_dict:
             s.name = obj_dict["id"]
         if "run" in obj_dict:
@@ -76,15 +76,14 @@ def from_dict(obj_dict) -> "CompositeActionStep":
             uses_string_obj = UsesString.analyze(uses_string=s.uses)
             if uses_string_obj.type == UsesStringType.ACTION:
                 obj = get_or_create_composite_action(
-                    uses_string_obj.get_absolute_path(s.path)
+                    uses_string_obj.absolute_path_with_ref
                 )
                 s.action.add(obj)
 
             if "with" in obj_dict:
                 s.with_prop = convert_dict_to_list(obj_dict["with"])
 
-            if len(s.uses.split("@")) > 1:
-                s.ref = s.uses.split("@")[1]
+            s.ref = uses_string_obj.ref
 
         return s
 
@@ -99,7 +98,7 @@ class CompositeAction(GraphObject):
     using = Property()
     image = Property()
     url = Property()
-    tag = Property()
+    ref = Property()
 
     steps = RelatedTo(CompositeActionStep)
 
@@ -115,8 +114,8 @@ def from_dict(obj_dict) -> "CompositeAction":
         ca.url = obj_dict["url"]
 
         # Optional properties
-        if "tag" in obj_dict:
-            ca.tag = obj_dict["tag"]
+        if "ref" in obj_dict:
+            ca.ref = obj_dict["ref"]
 
         if "inputs" in obj_dict:
             ca.inputs = list(obj_dict["inputs"].keys())
@@ -135,7 +134,7 @@ def from_dict(obj_dict) -> "CompositeAction":
                     step["_id"] = md5(f"{ca._id}_{i}".encode()).hexdigest()
                     step["path"] = ca.path
                     step["url"] = ca.url
-                    step["tag"] = ca.tag
+                    step["ref"] = ca.ref
                     ca.steps.add(CompositeActionStep.from_dict(step))
 
         return ca