Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Silence errors in //fbpcs
Browse files Browse the repository at this point in the history
Summary: Silencing these

Differential Revision: D64497617
Maggie Moss authored and facebook-github-bot committed Oct 16, 2024

Verified

This commit was signed with the committer’s verified signature.
neilvcarvalho Neil Carvalho
1 parent fcf4d60 commit 8c6a4d7
Showing 20 changed files with 221 additions and 3 deletions.
5 changes: 4 additions & 1 deletion fbpcs/bolt/bolt_runner.py
Original file line number Diff line number Diff line change
@@ -341,7 +341,10 @@ async def _run_hooks(
{stage, None}, {role, None}, {when, None}, {event, None}
)
for hook in job.hooks.get(
BoltHookKey(event=e, when=w, stage=s.name if s else s, role=r), []
# pyre-fixme[6]: For 3rd argument expected `Optional[str]` but got
# `Union[None, PrivateComputationBaseStageFlow, str]`.
BoltHookKey(event=e, when=w, stage=s.name if s else s, role=r),
[],
)
)

5 changes: 5 additions & 0 deletions fbpcs/bolt/read_config.py
Original file line number Diff line number Diff line change
@@ -14,6 +14,9 @@
from fbpcs.bolt.bolt_runner import BoltRunner
from fbpcs.bolt.constants import DEFAULT_POLL_INTERVAL_SEC
from fbpcs.bolt.oss_bolt_pcs import BoltPCSClient, BoltPCSCreateInstanceArgs

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation_cli.private_computation_service_wrapper`.
from fbpcs.private_computation_cli.private_computation_service_wrapper import (
build_private_computation_service,
)
@@ -45,6 +48,7 @@ def create_bolt_runner(
runner_config["partner_client_config"]
)
publisher_client = BoltPCSClient(
# pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation_cli`.
build_private_computation_service(
publisher_client_config["private_computation"],
publisher_client_config["mpc"],
@@ -54,6 +58,7 @@ def create_bolt_runner(
)
)
partner_client = BoltPCSClient(
# pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation_cli`.
build_private_computation_service(
partner_client_config["private_computation"],
partner_client_config["mpc"],
37 changes: 37 additions & 0 deletions fbpcs/common/feature/pcs_feature_gate_utils.py
Original file line number Diff line number Diff line change
@@ -7,53 +7,90 @@
# pyre-strict
from typing import Optional, Set, Type

# pyre-fixme[21]: Could not find module `fbpcs.private_computation.entity.infra_config`.
from fbpcs.private_computation.entity.infra_config import PrivateComputationGameType

# pyre-fixme[21]: Could not find module `fbpcs.private_computation.entity.pcs_feature`.
from fbpcs.private_computation.entity.pcs_feature import PCSFeature

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.stage_flows.private_computation_base_stage_flow`.
from fbpcs.private_computation.stage_flows.private_computation_base_stage_flow import (
PrivateComputationBaseStageFlow,
)

# pyre-fixme[21]: Could not find module `fbpcs.private_computation.stage_flows.privat...
from fbpcs.private_computation.stage_flows.private_computation_mr_pid_pcf2_lift_stage_flow import (
PrivateComputationMrPidPCF2LiftStageFlow,
)

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.stage_flows.private_computation_mr_stage_flow`.
from fbpcs.private_computation.stage_flows.private_computation_mr_stage_flow import (
PrivateComputationMRStageFlow,
)

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.stage_flows.private_computation_pcf2_lift_udp_stage_flow`.
from fbpcs.private_computation.stage_flows.private_computation_pcf2_lift_udp_stage_flow import (
PrivateComputationPCF2LiftUDPStageFlow,
)

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.stage_flows.private_computation_pcf2_stage_flow`.
from fbpcs.private_computation.stage_flows.private_computation_pcf2_stage_flow import (
PrivateComputationPCF2StageFlow,
)

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.stage_flows.private_computation_stage_flow`.
from fbpcs.private_computation.stage_flows.private_computation_stage_flow import (
PrivateComputationStageFlow,
)
from fbpcs.utils.optional import unwrap_or_default


def get_stage_flow(
# pyre-fixme[11]: Annotation `PrivateComputationGameType` is not defined as a type.
game_type: PrivateComputationGameType,
# pyre-fixme[11]: Annotation `PCSFeature` is not defined as a type.
pcs_feature_enums: Set[PCSFeature],
# pyre-fixme[11]: Annotation `PrivateComputationBaseStageFlow` is not defined as
# a type.
stage_flow_cls: Optional[Type[PrivateComputationBaseStageFlow]] = None,
) -> Type[PrivateComputationBaseStageFlow]:
selected_stage_flow_cls = unwrap_or_default(
optional=stage_flow_cls,
default=(
# pyre-fixme[16]: Module `private_computation` has no attribute
# `stage_flows`.
PrivateComputationPCF2StageFlow
# pyre-fixme[16]: Module `entity` has no attribute `infra_config`.
if game_type is PrivateComputationGameType.ATTRIBUTION
# pyre-fixme[16]: Module `private_computation` has no attribute
# `stage_flows`.
else PrivateComputationStageFlow
),
)

# warning, enabled feature gating will override stage flow, Please contact PSI team to have a similar adoption
if PCSFeature.PRIVATE_ATTRIBUTION_MR_PID in pcs_feature_enums:
selected_stage_flow_cls = (
# pyre-fixme[16]: Module `private_computation` has no attribute
# `stage_flows`.
PrivateComputationMRStageFlow
# pyre-fixme[16]: Module `entity` has no attribute `infra_config`.
if game_type is PrivateComputationGameType.ATTRIBUTION
# pyre-fixme[16]: Module `private_computation` has no attribute
# `stage_flows`.
else PrivateComputationMrPidPCF2LiftStageFlow
)
if PCSFeature.PRIVATE_LIFT_UNIFIED_DATA_PROCESS in pcs_feature_enums:
selected_stage_flow_cls = (
# pyre-fixme[16]: Module `private_computation` has no attribute
# `stage_flows`.
PrivateComputationPCF2LiftUDPStageFlow
# pyre-fixme[16]: Module `entity` has no attribute `infra_config`.
if game_type is PrivateComputationGameType.LIFT
else selected_stage_flow_cls
)
8 changes: 8 additions & 0 deletions fbpcs/data_processing/pid_preparer/preparer.py
Original file line number Diff line number Diff line change
@@ -11,8 +11,13 @@
import pathlib
from typing import Optional

# pyre-fixme[21]: Could not find module `fbpcp.entity.container_instance`.
from fbpcp.entity.container_instance import ContainerInstance

# pyre-fixme[21]: Could not find module `fbpcp.service.onedocker`.
from fbpcp.service.onedocker import OneDockerService

# pyre-fixme[21]: Could not find module `fbpcp.service.storage`.
from fbpcp.service.storage import StorageService


@@ -24,6 +29,7 @@ def prepare(
output_path: str,
log_path: Optional[pathlib.Path] = None,
log_level: int = logging.INFO,
# pyre-fixme[11]: Annotation `StorageService` is not defined as a type.
storage_svc: Optional[StorageService] = None,
) -> None:
pass
@@ -34,10 +40,12 @@ def prepare_on_container(
input_path: str,
output_path: str,
# TODO: Support custom log path
# pyre-fixme[11]: Annotation `OneDockerService` is not defined as a type.
onedocker_svc: OneDockerService,
binary_version: str,
tmp_directory: str = "/tmp/",
wait_for_container: bool = True,
# pyre-fixme[11]: Annotation `ContainerInstance` is not defined as a type.
) -> ContainerInstance:
pass

20 changes: 20 additions & 0 deletions fbpcs/data_processing/pid_preparer/union_pid_preparer_cpp.py
Original file line number Diff line number Diff line change
@@ -15,14 +15,27 @@
import tempfile
from typing import Dict, Optional

# pyre-fixme[21]: Could not find module `fbpcp.entity.container_instance`.
from fbpcp.entity.container_instance import ContainerInstance, ContainerInstanceStatus

# pyre-fixme[21]: Could not find module `fbpcp.error.pcp`.
from fbpcp.error.pcp import ThrottlingError

# pyre-fixme[21]: Could not find module `fbpcp.service.onedocker`.
from fbpcp.service.onedocker import OneDockerService

# pyre-fixme[21]: Could not find module `fbpcp.service.storage`.
from fbpcp.service.storage import PathType, StorageService

# pyre-fixme[21]: Could not find module `fbpcs.common.service.retry_handler`.
from fbpcs.common.service.retry_handler import RetryHandler
from fbpcs.data_processing.pid_preparer.preparer import UnionPIDDataPreparerService

# pyre-fixme[21]: Could not find module `fbpcs.onedocker_binary_names`.
from fbpcs.onedocker_binary_names import OneDockerBinaryNames

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.service.run_binary_base_service`.
from fbpcs.private_computation.service.run_binary_base_service import (
RunBinaryBaseService,
)
@@ -43,6 +56,7 @@ def prepare(
output_path: str,
log_path: Optional[pathlib.Path] = None,
log_level: int = logging.INFO,
# pyre-fixme[11]: Annotation `StorageService` is not defined as a type.
storage_svc: Optional[StorageService] = None,
) -> None:
if log_path is not None:
@@ -95,13 +109,15 @@ def prepare_on_container(
self,
input_path: str,
output_path: str,
# pyre-fixme[11]: Annotation `OneDockerService` is not defined as a type.
onedocker_svc: OneDockerService,
binary_version: str,
max_column_count: int = 1,
tmp_directory: str = "/tmp/",
container_timeout: Optional[int] = None,
wait_for_container: bool = True,
env_vars: Optional[Dict[str, str]] = None,
# pyre-fixme[11]: Annotation `ContainerInstance` is not defined as a type.
) -> ContainerInstance:
return asyncio.run(
self.prepare_on_container_async(
@@ -145,6 +161,7 @@ async def prepare_on_container_async(

current_retry = 0
status = ContainerInstanceStatus.UNKNOWN
# pyre-fixme[16]: Module `fbpcs` has no attribute `onedocker_binary_names`.
exe = OneDockerBinaryNames.UNION_PID_PREPARER.value
container = None
while status is not ContainerInstanceStatus.COMPLETED:
@@ -162,6 +179,7 @@ async def prepare_on_container_async(
env_vars=env_vars,
)

# pyre-fixme[16]: Module `fbpcs` has no attribute `common`.
with RetryHandler(
ThrottlingError, logger=logger, backoff_seconds=30
) as retry_handler:
@@ -175,6 +193,8 @@ async def prepare_on_container_async(
# Busy wait until the container is finished
if wait_for_container:
container = (
# pyre-fixme[16]: Module `fbpcs` has no attribute
# `private_computation`.
await RunBinaryBaseService.wait_for_containers_async(
onedocker_svc, [container]
)
12 changes: 12 additions & 0 deletions fbpcs/data_processing/service/id_spine_combiner.py
Original file line number Diff line number Diff line change
@@ -8,10 +8,17 @@

from typing import List, Optional

# pyre-fixme[21]: Could not find module `fbpcp.util.arg_builder`.
from fbpcp.util.arg_builder import build_cmd_args

# pyre-fixme[21]: Could not find module `fbpcs.private_computation.service.constants`.
from fbpcs.private_computation.service.constants import DEFAULT_SORT_STRATEGY

# pyre-fixme[21]: Could not find module `fbpcs.private_computation.service.pid_utils`.
from fbpcs.private_computation.service.pid_utils import get_sharded_filepath

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.service.run_binary_base_service`.
from fbpcs.private_computation.service.run_binary_base_service import (
RunBinaryBaseService,
)
@@ -21,6 +28,7 @@
DEFAULT_CONTAINER_TIMEOUT_IN_SEC = 10800


# pyre-fixme[11]: Annotation `RunBinaryBaseService` is not defined as a type.
class IdSpineCombinerService(RunBinaryBaseService):
@staticmethod
def build_args(
@@ -31,6 +39,7 @@ def build_args(
tmp_directory: str,
protocol_type: str,
max_id_column_cnt: int = 1,
# pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation`.
sort_strategy: str = DEFAULT_SORT_STRATEGY,
# TODO T106159008: padding_size and run_name are only temporarily optional
# because Lift does not use them. It should and will be required to use them.
@@ -46,8 +55,11 @@ def build_args(
# own ThreadPoolExecutor here and instead use async primitives
cmd_args_list = []
for shard in range(num_shards):
# pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation`.
next_spine_path = get_sharded_filepath(spine_path, shard)
# pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation`.
next_data_path = get_sharded_filepath(data_path, shard)
# pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation`.
next_output_path = get_sharded_filepath(output_path, shard)
cmd_args = build_cmd_args(
spine_path=next_spine_path,
14 changes: 14 additions & 0 deletions fbpcs/data_processing/service/pid_run_protocol_binary_service.py
Original file line number Diff line number Diff line change
@@ -12,6 +12,9 @@
from fbpcs.onedocker_binary_names import OneDockerBinaryNames

from fbpcs.pid.entity.pid_instance import PIDProtocol

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.entity.private_computation_instance`.
from fbpcs.private_computation.entity.private_computation_instance import (
PrivateComputationRole,
)
@@ -35,6 +38,7 @@ def build_args(
output_path: str,
port: int,
tls_args: TlsArgs,
# pyre-fixme[11]: Annotation `PrivateComputationRole` is not defined as a type.
pc_role: PrivateComputationRole,
use_row_numbers: bool = False,
server_endpoint: Optional[str] = None,
@@ -66,12 +70,18 @@ def build_args(
if use_row_numbers:
cmd_ls.append("--use-row-numbers")

# pyre-fixme[16]: Module `entity` has no attribute
# `private_computation_instance`.
if tls_args.ca_cert_path and pc_role is PrivateComputationRole.PARTNER:
cmd_ls.append(f"--tls-ca {tls_args.ca_cert_path}")

# pyre-fixme[16]: Module `entity` has no attribute
# `private_computation_instance`.
if tls_args.server_cert_path and pc_role is PrivateComputationRole.PUBLISHER:
cmd_ls.append(f"--tls-cert {tls_args.server_cert_path}")

# pyre-fixme[16]: Module `entity` has no attribute
# `private_computation_instance`.
if tls_args.private_key_path and pc_role is PrivateComputationRole.PUBLISHER:
cmd_ls.append(f"--tls-key {tls_args.private_key_path}")

@@ -82,10 +92,14 @@ def build_args(

@staticmethod
def get_binary_name(protocol: PIDProtocol, pc_role: PrivateComputationRole) -> str:
# pyre-fixme[16]: Module `entity` has no attribute
# `private_computation_instance`.
if pc_role is PrivateComputationRole.PUBLISHER:
binary_name = OneDockerBinaryNames.PID_SERVER.value
if protocol is PIDProtocol.UNION_PID_MULTIKEY:
binary_name = OneDockerBinaryNames.PID_MULTI_KEY_SERVER.value
# pyre-fixme[16]: Module `entity` has no attribute
# `private_computation_instance`.
elif pc_role is PrivateComputationRole.PARTNER:
binary_name = OneDockerBinaryNames.PID_CLIENT.value
if protocol is PIDProtocol.UNION_PID_MULTIKEY:
Original file line number Diff line number Diff line change
@@ -9,6 +9,9 @@
from typing import Optional

from fbpcs.infra.certificate.certificate_provider import CertificateProvider

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.entity.private_computation_instance`.
from fbpcs.private_computation.entity.private_computation_instance import (
PrivateComputationInstance,
)
@@ -20,7 +23,9 @@ class PCInstanceCaCertificateProvider(CertificateProvider):
from PC instance repo.
"""

# pyre-fixme[11]: Annotation `PrivateComputationInstance` is not defined as a type.
def __init__(self, pc_instance: PrivateComputationInstance) -> None:
# pyre-fixme[4]: Attribute must be annotated.
self.pc_instance = pc_instance

def get_certificate(self) -> Optional[str]:
5 changes: 5 additions & 0 deletions fbpcs/infra/certificate/pc_instance_server_certificate.py
Original file line number Diff line number Diff line change
@@ -9,6 +9,9 @@
from typing import Optional

from fbpcs.infra.certificate.certificate_provider import CertificateProvider

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.entity.private_computation_instance`.
from fbpcs.private_computation.entity.private_computation_instance import (
PrivateComputationInstance,
)
@@ -20,7 +23,9 @@ class PCInstanceServerCertificateProvider(CertificateProvider):
from PC instance repo.
"""

# pyre-fixme[11]: Annotation `PrivateComputationInstance` is not defined as a type.
def __init__(self, pc_instance: PrivateComputationInstance) -> None:
# pyre-fixme[4]: Attribute must be annotated.
self.pc_instance = pc_instance

def get_certificate(self) -> Optional[str]:
2 changes: 2 additions & 0 deletions fbpcs/pc_pre_validation/tests/pc_pre_validation_cli_test.py
Original file line number Diff line number Diff line change
@@ -91,6 +91,8 @@ def test_parsing_all_args(
expected_access_key_id = "access_key_id2"
expected_access_key_data = "access_key_data3"
expected_binary_version = "binary_version"
# pyre-fixme[9]: expected_pc_computation_role has type
# `PrivateComputationRole`; used as `str`.
expected_pc_computation_role: PrivateComputationRole = (
PrivateComputationRole.PARTNER.name
)
3 changes: 3 additions & 0 deletions fbpcs/pl_coordinator/token_validator.py
Original file line number Diff line number Diff line change
@@ -77,6 +77,9 @@ def validate_rule(self, rule: TokenValidationRule) -> None:
instance_id="NO_INSTANCE",
checkpoint_name="FAIL_FAST_VALIDATION",
status=CheckpointStatus.FAILED,
# pyre-fixme[6]: For 5th argument expected
# `Optional[Dict[str, str]]` but got `Dict[str,
# Union[TokenValidationRuleType, str]]`.
checkpoint_data={
"rule_name": rule.name,
"rule_type": rule.rule_type,
4 changes: 4 additions & 0 deletions fbpcs/post_processing_handler/post_processing_handler.py
Original file line number Diff line number Diff line change
@@ -11,6 +11,8 @@
from typing import TYPE_CHECKING

if TYPE_CHECKING:
# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.entity.private_computation_instance`.
from fbpcs.private_computation.entity.private_computation_instance import (
PrivateComputationInstance,
)
@@ -29,6 +31,8 @@ class PostProcessingHandler(abc.ABC):
async def run(
self,
storage_svc: StorageService,
# pyre-fixme[11]: Annotation `PrivateComputationInstance` is not defined as
# a type.
private_computation_instance: "PrivateComputationInstance",
) -> None:
raise NotImplementedError
4 changes: 4 additions & 0 deletions fbpcs/post_processing_handler/tests/dummy_handler.py
Original file line number Diff line number Diff line change
@@ -15,6 +15,8 @@
from fbpcs.post_processing_handler.post_processing_handler import PostProcessingHandler

if TYPE_CHECKING:
# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.entity.private_computation_instance`.
from fbpcs.private_computation.entity.private_computation_instance import (
PrivateComputationInstance,
)
@@ -38,6 +40,8 @@ def __init__(
async def run(
self,
storage_svc: StorageService,
# pyre-fixme[11]: Annotation `PrivateComputationInstance` is not defined as
# a type.
private_computation_instance: "PrivateComputationInstance",
) -> None:
if random.random() >= self.probability_of_failure:
2 changes: 2 additions & 0 deletions fbpcs/private_computation/entity/pid_mr_config.py
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@
from dataclasses import dataclass
from enum import Enum

# pyre-fixme[21]: Could not find module `dataclasses_json`.
from dataclasses_json import dataclass_json, DataClassJsonMixin


@@ -26,6 +27,7 @@ class PidWorkflowConfigs:


@dataclass
# pyre-fixme[11]: Annotation `DataClassJsonMixin` is not defined as a type.
class PidMrConfig(DataClassJsonMixin):
runConfigs: PidRunConfigs
workflowConfigs: PidWorkflowConfigs
9 changes: 9 additions & 0 deletions fbpcs/private_computation/service/constants.py
Original file line number Diff line number Diff line change
@@ -8,8 +8,10 @@

import typing

# pyre-fixme[21]: Could not find module `fbpcs.pid.entity.pid_instance`.
from fbpcs.pid.entity.pid_instance import PIDProtocol

# pyre-fixme[21]: Could not find module `fbpcs.private_computation.entity.infra_config`.
from fbpcs.private_computation.entity.infra_config import PrivateComputationGameType

"""
@@ -32,14 +34,21 @@

DEFAULT_K_ANONYMITY_THRESHOLD_PL = 100
DEFAULT_K_ANONYMITY_THRESHOLD_PA = 0
# pyre-fixme[11]: Annotation `PIDProtocol` is not defined as a type.
# pyre-fixme[16]: Module `fbpcs` has no attribute `pid`.
DEFAULT_PID_PROTOCOL: PIDProtocol = PIDProtocol.UNION_PID
DEFAULT_HMAC_KEY: str = ""
DEFAULT_CONCURRENCY = 4
ATTRIBUTION_TEST_CONCURRENCY = 1
# pyre-fixme[11]: Annotation `PrivateComputationGameType` is not defined as a type.
DEFAULT_PADDING_SIZE: typing.Dict[PrivateComputationGameType, typing.Optional[int]] = {
# pyre-fixme[16]: Module `entity` has no attribute `infra_config`.
PrivateComputationGameType.LIFT: 25,
# pyre-fixme[16]: Module `entity` has no attribute `infra_config`.
PrivateComputationGameType.ATTRIBUTION: 4,
# pyre-fixme[16]: Module `entity` has no attribute `infra_config`.
PrivateComputationGameType.PRIVATE_ID_DFCA: None,
# pyre-fixme[16]: Module `entity` has no attribute `infra_config`.
PrivateComputationGameType.ANONYMIZER: None,
}
DEFAULT_LOG_COST_TO_S3 = True
16 changes: 16 additions & 0 deletions fbpcs/private_computation/service/mpc/mpc.py
Original file line number Diff line number Diff line change
@@ -9,27 +9,38 @@
import logging
from typing import Any, Dict, List, Optional, Tuple

# pyre-fixme[21]: Could not find module `fbpcp.service.container`.
from fbpcp.service.container import ContainerService

# pyre-fixme[21]: Could not find module `fbpcp.service.onedocker`.
from fbpcp.service.onedocker import OneDockerService

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.entity.private_computation_instance`.
from fbpcs.private_computation.entity.private_computation_instance import (
PrivateComputationRole,
)
from fbpcs.private_computation.service.mpc.entity.mpc_instance import MPCParty
from fbpcs.private_computation.service.mpc.mpc_game import MPCGameService

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.service.run_binary_base_service`.
from fbpcs.private_computation.service.run_binary_base_service import (
RunBinaryBaseService,
)

DEFAULT_BINARY_VERSION = "latest"


# pyre-fixme[11]: Annotation `RunBinaryBaseService` is not defined as a type.
class MPCService(RunBinaryBaseService):
"""MPCService is responsible for distributing a larger MPC game to multiple
MPC workers
"""

def __init__(
self,
# pyre-fixme[11]: Annotation `ContainerService` is not defined as a type.
container_svc: ContainerService,
task_definition: str,
mpc_game_svc: MPCGameService,
@@ -45,11 +56,13 @@ def __init__(
f"Dependency is missing. container_svc={container_svc}, mpc_game_svc={mpc_game_svc}"
)

# pyre-fixme[4]: Attribute must be annotated.
self.container_svc = container_svc
self.task_definition = task_definition
self.mpc_game_svc: MPCGameService = mpc_game_svc
self.logger: logging.Logger = logging.getLogger(__name__)

# pyre-fixme[4]: Attribute must be annotated.
self.onedocker_svc = OneDockerService(self.container_svc, self.task_definition)

"""
@@ -138,6 +151,7 @@ def convert_cmd_args_list(


def map_private_computation_role_to_mpc_party(
# pyre-fixme[11]: Annotation `PrivateComputationRole` is not defined as a type.
private_computation_role: PrivateComputationRole,
) -> MPCParty:
"""Convert PrivateComputationRole to MPCParty
@@ -151,8 +165,10 @@ def map_private_computation_role_to_mpc_party(
Exceptions:
ValueError: raised when there is no MPCParty associated with private_computation_role
"""
# pyre-fixme[16]: Module `private_computation` has no attribute `entity`.
if private_computation_role is PrivateComputationRole.PUBLISHER:
return MPCParty.SERVER
# pyre-fixme[16]: Module `private_computation` has no attribute `entity`.
elif private_computation_role is PrivateComputationRole.PARTNER:
return MPCParty.CLIENT
else:
1 change: 1 addition & 0 deletions fbpcs/private_computation/service/mpc/mpc_game.py
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@
import logging
from typing import Any, Dict, Optional, Tuple

# pyre-fixme[21]: Could not find module `fbpcp.util.arg_builder`.
from fbpcp.util.arg_builder import build_cmd_args

from fbpcs.private_computation.service.mpc.entity.mpc_game_config import MPCGameConfig
37 changes: 37 additions & 0 deletions fbpcs/private_computation/service/pre_validate_service.py
Original file line number Diff line number Diff line change
@@ -10,23 +10,48 @@
import logging
from typing import Any, Dict, List

# pyre-fixme[21]: Could not find module `fbpcp.entity.container_instance`.
from fbpcp.entity.container_instance import ContainerInstanceStatus

# pyre-fixme[21]: Could not find module `fbpcp.entity.container_type`.
from fbpcp.entity.container_type import ContainerType

# pyre-fixme[21]: Could not find module `fbpcs.onedocker_binary_names`.
from fbpcs.onedocker_binary_names import OneDockerBinaryNames

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.entity.private_computation_instance`.
from fbpcs.private_computation.entity.private_computation_instance import (
PrivateComputationRole,
)

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.service.pc_pre_validation_stage_service`.
from fbpcs.private_computation.service.pc_pre_validation_stage_service import (
PRE_VALIDATION_CHECKS_TIMEOUT,
)

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.service.pre_validation_util`.
from fbpcs.private_computation.service.pre_validation_util import get_cmd_args

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.service.private_computation`.
from fbpcs.private_computation.service.private_computation import (
PrivateComputationService,
)

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation.service.run_binary_base_service`.
from fbpcs.private_computation.service.run_binary_base_service import (
RunBinaryBaseService,
)

# pyre-fixme[21]: Could not find module `fbpcs.private_computation.service.utils`.
from fbpcs.private_computation.service.utils import generate_env_vars_dict

# pyre-fixme[21]: Could not find module
# `fbpcs.private_computation_cli.private_computation_service_wrapper`.
from fbpcs.private_computation_cli.private_computation_service_wrapper import (
build_private_computation_service,
)
@@ -35,46 +60,57 @@
class PreValidateService:
@staticmethod
async def run_pre_validate_async(
# pyre-fixme[11]: Annotation `PrivateComputationService` is not defined as a
# type.
pc_service: PrivateComputationService,
input_paths: List[str],
logger: logging.Logger,
) -> None:
region = pc_service.pc_validator_config.region
onedocker_svc = pc_service.onedocker_svc
# pyre-fixme[16]: Module `fbpcs` has no attribute `onedocker_binary_names`.
binary_name = OneDockerBinaryNames.PC_PRE_VALIDATION.value
binary_config = pc_service.onedocker_binary_config_map[binary_name]
# pyre-fixme[16]: Module `service` has no attribute `utils`.
env_vars = generate_env_vars_dict(repository_path=binary_config.repository_path)

"""
[BE] T147526958 pass private computation role from cli.
Since this service runs on partner PCE, we are hardcoding PrivateComputationRole to PARTNER in the code below.
"""
cmd_args = [
# pyre-fixme[16]: Module `service` has no attribute `pre_validation_util`.
get_cmd_args(
input_path=input_path,
region=region,
binary_config=binary_config,
pre_validation_file_stream_flag=True,
publisher_pc_pre_validation_flag=True,
partner_pc_pre_validation_flag=True,
# pyre-fixme[16]: Module `entity` has no attribute
# `private_computation_instance`.
private_computation_role=PrivateComputationRole.PARTNER,
input_path_start_ts=None,
input_path_end_ts=None,
)
for input_path in input_paths
]

# pyre-fixme[16]: Module `service` has no attribute `run_binary_base_service`.
container_instances = await RunBinaryBaseService().start_containers(
cmd_args,
onedocker_svc,
binary_config.binary_version,
binary_name,
# pyre-fixme[16]: Module `service` has no attribute
# `pc_pre_validation_stage_service`.
timeout=PRE_VALIDATION_CHECKS_TIMEOUT,
env_vars=env_vars,
container_type=ContainerType.LARGE,
)
logger.info("Started container instances")

# pyre-fixme[16]: Module `service` has no attribute `run_binary_base_service`.
completed_containers = await RunBinaryBaseService().wait_for_containers_async(
onedocker_svc,
container_instances,
@@ -119,6 +155,7 @@ def pre_validate(
input_paths: List[str],
logger: logging.Logger,
) -> None:
# pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation_cli`.
pc_service = build_private_computation_service(
config["private_computation"],
config["mpc"],
31 changes: 29 additions & 2 deletions fbpcs/private_computation/service/run_binary_base_service.py
Original file line number Diff line number Diff line change
@@ -10,13 +10,25 @@
import logging
from typing import Dict, List, Optional

# pyre-fixme[21]: Could not find module `fbpcp.entity.certificate_request`.
from fbpcp.entity.certificate_request import CertificateRequest

# pyre-fixme[21]: Could not find module `fbpcp.entity.container_instance`.
from fbpcp.entity.container_instance import ContainerInstance, ContainerInstanceStatus

# pyre-fixme[21]: Could not find module `fbpcp.entity.container_permission`.
from fbpcp.entity.container_permission import ContainerPermissionConfig

# pyre-fixme[21]: Could not find module `fbpcp.entity.container_type`.
from fbpcp.entity.container_type import ContainerType

# pyre-fixme[21]: Could not find module `fbpcp.error.pcp`.
from fbpcp.error.pcp import ThrottlingError

# pyre-fixme[21]: Could not find module `fbpcp.service.onedocker`.
from fbpcp.service.onedocker import OneDockerService

# pyre-fixme[21]: Could not find module `fbpcs.common.service.retry_handler`.
from fbpcs.common.service.retry_handler import BackoffType, RetryHandler
from fbpcs.private_computation.service.constants import DEFAULT_CONTAINER_TIMEOUT_IN_SEC

@@ -27,18 +39,24 @@ class RunBinaryBaseService:
async def start_containers(
self,
cmd_args_list: List[str],
# pyre-fixme[11]: Annotation `OneDockerService` is not defined as a type.
onedocker_svc: OneDockerService,
binary_version: str,
binary_name: str,
timeout: Optional[int] = None,
wait_for_containers_to_finish: bool = False,
env_vars: Optional[Dict[str, str]] = None,
wait_for_containers_to_start_up: bool = True,
# pyre-fixme[11]: Annotation `ContainerInstance` is not defined as a type.
existing_containers: Optional[List[ContainerInstance]] = None,
# pyre-fixme[11]: Annotation `ContainerType` is not defined as a type.
container_type: Optional[ContainerType] = None,
# pyre-fixme[11]: Annotation `CertificateRequest` is not defined as a type.
certificate_request: Optional[CertificateRequest] = None,
env_vars_list: Optional[List[Dict[str, str]]] = None,
opa_workflow_path: Optional[str] = None,
# pyre-fixme[11]: Annotation `ContainerPermissionConfig` is not defined as a
# type.
permission: Optional[ContainerPermissionConfig] = None,
) -> List[ContainerInstance]:
logger = logging.getLogger(__name__)
@@ -82,6 +100,7 @@ async def start_containers(
logger.info("Skipped container warm up")
return pending_containers

# pyre-fixme[16]: Module `fbpcs` has no attribute `common`.
with RetryHandler(
ThrottlingError, logger=logger, backoff_seconds=30
) as retry_handler:
@@ -150,8 +169,12 @@ async def wait_for_containers_async(
container_ids = [container.instance_id for container in containers]
finished_containers = []

# pyre-fixme[16]: Module `fbpcs` has no attribute `common`.
with RetryHandler(
ThrottlingError, backoff_type=BackoffType.LINEAR
# pyre-fixme[16]: Module `fbpcs` has no attribute `common`.
ThrottlingError,
# pyre-fixme[16]: Module `fbpcs` has no attribute `common`.
backoff_type=BackoffType.LINEAR,
) as retry_handler:
updated_containers = retry_handler.execute_sync(
onedocker_svc.get_containers,
@@ -164,8 +187,12 @@ async def wait_for_containers_async(
)
while pending_container_ids:
await asyncio.sleep(poll)
# pyre-fixme[16]: Module `fbpcs` has no attribute `common`.
with RetryHandler(
ThrottlingError, backoff_type=BackoffType.LINEAR
# pyre-fixme[16]: Module `fbpcs` has no attribute `common`.
ThrottlingError,
# pyre-fixme[16]: Module `fbpcs` has no attribute `common`.
backoff_type=BackoffType.LINEAR,
) as retry_handler:
updated_containers = retry_handler.execute_sync(
onedocker_svc.get_containers,
4 changes: 4 additions & 0 deletions fbpcs/service/workflow_sfn_fb.py
Original file line number Diff line number Diff line change
@@ -10,18 +10,22 @@

import botocore

# pyre-fixme[21]: Could not find module `fbpcp.intern.gateway.aws_fb`.
from fbpcp.intern.gateway.aws_fb import FBAWSGateway
from fbpcs.service.workflow_sfn import SfnWorkflowService


# pyre-fixme[11]: Annotation `FBAWSGateway` is not defined as a type.
class FBSfnWorkflowService(FBAWSGateway, SfnWorkflowService):
def __init__(
self,
region: str,
account: str,
role: Optional[str] = None,
) -> None:
# pyre-fixme[28]: Unexpected keyword argument `account`.
super().__init__(account=account, role=role, region=region)
# pyre-fixme[16]: `FBSfnWorkflowService` has no attribute `session_gen`.
self.client: botocore.client.BaseClient = self.session_gen.get_client(
"stepfunctions"
)

0 comments on commit 8c6a4d7

Please sign in to comment.