Feat/complex operators

frontend/src/context/workspaces/repositories.tsx

@@ -16,12 +16,14 @@ import { toast } from "react-toastify";
 import { createCustomContext } from "utils";
 
 import { useWorkspaces } from "./workspaces";
+import _default from "vite-tsconfig-paths";
 
 export interface IPiecesContext {
   repositories: Repository[];
   defaultRepositories: Repository[];
   repositoryPieces: PiecesRepository;
   repositoriesLoading: boolean;
+  defaultRepositoriesIsLoading: boolean;
 
   selectedRepositoryId?: number;
   setSelectedRepositoryId: React.Dispatch<
@@ -58,7 +60,8 @@ const PiecesProvider: React.FC<{ children: React.ReactNode }> = ({
 
   const queryClient = useQueryClient();
 
-  const { data: defaultRepositories } = useRepositories({
+  const { data: defaultRepositories, isLoading: defaultRepositoriesIsLoading } = useRepositories({
+    workspaceId: workspace?.id,
     source: "default",
   });
 
@@ -104,11 +107,11 @@ const PiecesProvider: React.FC<{ children: React.ReactNode }> = ({
     const repositoryPiecesAux: PiecesRepository = {};
     const foragePieces: PieceForageSchema = {};
 
-    if (!pieces?.length) {
+    if (!pieces?.length && !_defaultPieces?.length) {
       localStorage.setItem("pieces", foragePieces);
       return repositoryPiecesAux;
     } else {
-      for (const piece of pieces) {
+      for (const piece of [...pieces, ..._defaultPieces]) {
         if (repositoryPiecesAux[piece.repository_id]?.length) {
           repositoryPiecesAux[piece.repository_id].push(piece);
         } else {
@@ -121,7 +124,7 @@ const PiecesProvider: React.FC<{ children: React.ReactNode }> = ({
       localStorage.setItem("pieces", foragePieces);
       return repositoryPiecesAux;
     }
-  }, [pieces]);
+  }, [pieces, _defaultPieces]);
 
   const fetchForagePieceById = useCallback((id: number) => {
     const pieces = localStorage.getItem<PieceForageSchema>("pieces");
@@ -135,6 +138,7 @@ const PiecesProvider: React.FC<{ children: React.ReactNode }> = ({
     defaultRepositories: defaultRepositories?.data ?? [],
     repositoryPieces,
     repositoriesLoading,
+    defaultRepositoriesIsLoading,
 
     selectedRepositoryId,
     setSelectedRepositoryId,

frontend/src/features/workflowEditor/components/Drawers/PiecesDrawer/sidebarAddNode.tsx

@@ -23,7 +23,20 @@ interface Props {
 }
 
 const SidebarAddNode: FC<Props> = ({ setOrientation, orientation }) => {
-  const { repositories, repositoriesLoading, repositoryPieces } = usesPieces();
+  const {
+    repositories,
+    repositoriesLoading,
+    repositoryPieces,
+    defaultRepositories,
+    defaultRepositoriesIsLoading
+  } = usesPieces();
+
+  const controlRepository = useMemo(() => {
+    return defaultRepositories.find((repository) => {
+      return repository.name.includes("control");
+    });
+  }, [defaultRepositories]);
+
 
   const [filter, setFilter] = useState("");
   const [expanded, setExpanded] = useState<Record<string, boolean>>({});
@@ -71,12 +84,18 @@ const SidebarAddNode: FC<Props> = ({ setOrientation, orientation }) => {
     }
   }, [filter]);
 
+  const isLoading = useMemo(() =>{
+    return repositoriesLoading || defaultRepositoriesIsLoading;
+  }, [repositoriesLoading, defaultRepositoriesIsLoading])
+
+  const allRepos = controlRepository ? [...repositories, controlRepository] : repositories;
+
   return (
     <Box sx={{ padding: 2 }}>
-      {repositoriesLoading && (
+      {isLoading && (
         <Alert severity="info">Loading repositories...</Alert>
       )}
-      {!repositoriesLoading && (
+      {!isLoading && (
         <ToggleButtonGroup
           sx={{ display: "flex" }}
           value={orientation}
@@ -103,8 +122,8 @@ const SidebarAddNode: FC<Props> = ({ setOrientation, orientation }) => {
         variant="filled"
         label="search"
       />
-      {!repositoriesLoading &&
-        repositories.map((repo) => {
+      {!isLoading &&
+        allRepos.map((repo) => {
           if (!filteredRepositoryPieces[repo.id]?.length) {
             return null;
           }

frontend/src/features/workflowEditor/components/Panel/WorkflowPanel/BatchNode/index.tsx

@@ -0,0 +1,111 @@
+import { Paper, Typography, useTheme } from "@mui/material";
+import { type CSSProperties, memo, useMemo } from "react";
+import { Handle, Position } from "reactflow";
+import { Icon } from "@iconify/react";
+
+import { type DefaultNodeProps } from "../types";
+
+export const BatchNode = memo<DefaultNodeProps>(({ id, data, selected }) => {
+    const theme = useTheme();
+
+    const handleStyle = useMemo<CSSProperties>(
+        () => ({
+            border: 0,
+            borderRadius: "16px",
+            backgroundColor: theme.palette.info.main,
+            transition: "ease 100",
+            zIndex: 2,
+            width: "12px",
+            height: "12px",
+        }),
+        [theme.palette.info.main]
+    );
+
+    const nodeStyle = useMemo<CSSProperties>(
+        () => ({
+            display: "flex",
+            flexDirection: "column",
+            alignItems: "center",
+            justifyContent: "center",
+            position: "relative",
+            padding: 1,
+            textAlign: "center",
+            width: "100px", // Adjusted width to make it a square
+            height: "100px", // Adjusted height to make it a square
+            lineHeight: "60px",
+            border: selected ? `2px solid ${theme.palette.info.dark}` : "2px solid transparent", // Border color change based on selection
+            color: theme.palette.getContrastText(theme.palette.background.paper),
+            backgroundColor: theme.palette.background.paper,
+            borderRadius: "3px",
+        }),
+        [selected, theme.palette]
+    );
+
+    const iconStyle = useMemo<CSSProperties>(
+        () => ({
+            width: "50px", // Adjusted width of the icon
+            height: "50px", // Adjusted height of the icon
+            marginBottom: "10px", // Added margin bottom to space out from text
+        }),
+        []
+    );
+
+    const labelStyle = {
+        fontSize: "8px",
+        position: "absolute",
+        textAlign: "left",
+        justifyContent: "left",
+        right: "-25px",
+        top: "10px"
+    };
+
+    return (
+        <>
+            <Handle
+                type="target"
+                position={Position.Left}
+                id={`target-${id}`}
+                style={{
+                    ...handleStyle,
+                }}
+                isConnectable
+                isValidConnection={(connection) => connection.sourceHandle !== `target-${id}`}
+            />
+            <Handle
+                type="source"
+                position={Position.Right}
+                id={`source-done-${id}`}
+                style={{
+                    ...handleStyle,
+                    transform: "translate(0%, -30px)", // Adjusted handle position up by 30px
+                }}
+                isConnectable
+            >
+                <Typography variant="caption" sx={{ color: theme.palette.text.secondary, ...labelStyle }}>DONE</Typography>
+            </Handle>
+            <Handle
+                type="source"
+                position={Position.Right}
+                id={`source-batch-${id}`}
+                style={{
+                    ...handleStyle,
+                    transform: "translate(0%, 25px)", // Adjusted handle position down by 30px
+                    backgroundColor: theme.palette.success.main,
+                }}
+                isConnectable
+            >
+                <Typography variant="caption" sx={{ color: theme.palette.text.secondary, ...labelStyle }}>BATCH</Typography>
+            </Handle>
+            <Paper elevation={selected ? 12 : 3} sx={nodeStyle}>
+                <Typography component="div" variant="body2" fontWeight={500}>
+                    Batch Piece
+                </Typography>
+                <Icon icon="ic:baseline-loop" style={iconStyle} />
+            </Paper>
+        </>
+    );
+});
+
+BatchNode.displayName = "BatchNode";
+
+export default BatchNode;

frontend/src/features/workflowEditor/components/Panel/WorkflowPanel/WorkflowPanel.tsx

@@ -47,6 +47,7 @@ import {
 import { CustomConnectionLine } from "./ConnectionLine";
 import DefaultEdge from "./DefaultEdge";
 import { CustomNode } from "./DefaultNode";
+import BatchNode from "./BatchNode"; // Import the BatchNode
 import { type DefaultNode } from "./types";
 
 const getId = (module_name: string) => {
@@ -55,6 +56,7 @@ const getId = (module_name: string) => {
 
 const DEFAULT_NODE_TYPES: NodeTypes = {
   CustomNode,
+  BatchNode,
 };
 
 const EDGE_TYPES: EdgeTypes = {
@@ -106,7 +108,8 @@ const WorkflowPanel = forwardRef<WorkflowPanelRef, Props>(
         setInstance(instance);
         const edges = getWorkflowEdges();
         const nodes = getWorkflowNodes();
-        setNodes(nodes);
+
+        setNodes([...nodes, defaultBatchNode]);
         setEdges(edges);
         window.requestAnimationFrame(() => instance.fitView());
       },
@@ -126,9 +129,14 @@ const WorkflowPanel = forwardRef<WorkflowPanelRef, Props>(
           orientation: data?.orientation ?? "horizontal",
         };
 
+        let nodeType = "CustomNode";
+        if (data.repository_url === 'domino-default/default_control_repository') {
+          nodeType = "BatchNode";
+        }
+
         const newNode = {
           id: getId(data.id),
-          type: "CustomNode",
+          type: nodeType,
           position,
           data: newNodeData,
         };

rest/auth/permission_authorizer.py

@@ -1,8 +1,7 @@
 from fastapi import HTTPException, Security
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
-import jwt
 from schemas.errors.base import ForbiddenError, ResourceNotFoundError
-from schemas.context.auth_context import AuthorizationContextData, WorkspaceAuthorizerData
+from schemas.context.auth_context import WorkspaceAuthorizerData
 from database.models.enums import Permission, UserWorkspaceStatus
 from typing import Optional, Dict
 from auth.base_authorizer import BaseAuthorizer

rest/constants/default_pieces/control/__init__.py

@@ -0,0 +1,5 @@
+from .batch import BatchPiece
+
+DEFAULT_CONTROL_PIECES = [
+    BatchPiece
+]

rest/constants/default_pieces/control/batch.py

@@ -0,0 +1,35 @@
+from pydantic import BaseModel, Field, PositiveInt
+from typing import List, Union, Optional
+from datetime import datetime
+
+
+
+class InputModel(BaseModel):
+    batch_over: List[Union[str, int, float, bool, dict, datetime]] = Field(
+        title='Batch Over',
+        description='List to iterate over',
+        json_schema_extra={"from_upstream": "always"}
+    )
+    max_concurrency: PositiveInt = Field(
+        title='Max Concurrency',
+        description='Max number of parallel executions'
+    )
+
+class OutputMode(BaseModel):
+    output: List[Union[str, int, float, bool, dict, datetime]] = Field(
+        title='Output',
+        description='Output of the batch',
+    ) # TODO use output modifier?
+
+batch_piece_default_style = {
+    "label": "Batch Piece",
+    "iconClassName": "ic:baseline-loop",
+    "nodeType": "control",
+}
+class BatchPiece(BaseModel):
+    name: str = Field(title='Name', default='BatchPiece')
+    description: str = Field(title='Description', default='Piece to run batch processing with concurrency.')
+    input_schema: dict = Field(default=InputModel.model_json_schema())
+    secrets_schema: Optional[dict] = Field(default=None)
+    style: dict = Field(default=batch_piece_default_style)
+

rest/core/settings.py

@@ -93,6 +93,14 @@ class Settings(BaseSettings):
         version="0.0.1",
         url="domino-default/default_storage_repository"
     )
+    DEFAULT_CONTROL_REPOSITORY: dict = dict(
+        name="default_control_repository",
+        path="default_control_repository",
+        source=getattr(RepositorySource, 'default').value,
+        version="0.0.1",
+        label="Default Control Repository",
+        url="domino-default/default_control_repository"
+    )
 
     DEPLOY_MODE: str = os.environ.get('DOMINO_DEPLOY_MODE', 'local-k8s')
 

rest/database/models/piece_repository.py

@@ -2,14 +2,14 @@
 from sqlalchemy import Column, String, Integer, DateTime, Enum, JSON, ForeignKey
 from database.models.enums import RepositorySource
 from sqlalchemy.orm import relationship
-from datetime import datetime
+from datetime import datetime, timezone
 
 
 class PieceRepository(Base, BaseDatabaseModel):
     __tablename__ = "piece_repository"
 
     id = Column(Integer, primary_key=True)
-    created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.utcnow)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now(tz=timezone.utc))
     name = Column(String(50), unique=False)
     label = Column(String(50), unique=False)
     source = Column(Enum(RepositorySource), nullable=True, default=RepositorySource.github.value)