From 199f5f64b81acd6a602a48b8fd80baeefdcdd0c0 Mon Sep 17 00:00:00 2001 From: Tobias Pfandzelter Date: Tue, 26 Mar 2024 13:21:01 +0100 Subject: [PATCH] fix github actions --- .github/workflows/main.yml | 23 ++++--- .github/workflows/release.yml | 9 ++- .goreleaser.yml | 8 ++- .vscode/settings.json | 3 +- celestial.py | 12 ++-- celestial/host.py | 4 +- celestial/proto_util.py | 63 ++++++++++--------- .../{satgen.py => satgen_connstellation.py} | 0 celestial/shell.py | 2 +- pkg/ebpfem/ebpfem_test.go | 2 + pkg/peer/peer.go | 4 ++ pkg/virt/machine.go | 2 +- pkg/virt/net.go | 2 +- requirements.txt | 2 +- satgen.py | 6 +- setup.cfg | 6 -- 16 files changed, 81 insertions(+), 67 deletions(-) rename celestial/{satgen.py => satgen_connstellation.py} (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index df93abc..b2455c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,17 +34,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: mypy-check - uses: theCapypara/mypy-check@master + uses: jpetrucciani/mypy-check@master with: - path: "" - requirements: types-toml - mypy_flags: "-p celestial --config setup.cfg" - - name: mypy-check2 - uses: theCapypara/mypy-check@master - with: - requirements: types-toml numpy types-request mypy-protobuf + requirements: "types-toml numpy types-requests mypy-protobuf" path: "*.py" mypy_flags: "--config setup.cfg" @@ -52,16 +46,21 @@ jobs: name: go runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.21' + cache: false - name: golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v4 with: version: latest + args: --out-format=colored-line-number markdown: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: markdownlint-cli uses: nosborn/github-action-markdown-cli@v3.3.0 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8aa5fb0..c9d179e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,6 @@ name: goreleaser on: - pull_request: push: tags: - "*" @@ -28,13 +27,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: - go-version: "1.21" + go-version: '1.21' + cache: false - name: Run GoReleaser uses: goreleaser/goreleaser-action@v5 with: diff --git a/.goreleaser.yml b/.goreleaser.yml index 2bb98c1..abe62e8 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -16,4 +16,10 @@ # builds: - - skip: true + - id: "celestial.bin" + main: ./celestial.go + binary: celestial.bin + goos: + - linux + goarch: + - amd64 diff --git a/.vscode/settings.json b/.vscode/settings.json index 5074a8e..7ea4afb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,5 +5,6 @@ "go.toolsEnvVars": { "GOOS": "linux", "GOARCH": "amd64" - } + }, + "mypy.runUsingActiveInterpreter": true } \ No newline at end of file diff --git a/celestial.py b/celestial.py index 5ed318f..3b30d00 100644 --- a/celestial.py +++ b/celestial.py @@ -145,7 +145,7 @@ def get_diff( ) -> typing.List[proto.celestial.celestial_pb2.StateUpdateRequest]: # we get iterators of the deserialized diffs t1 = time.perf_counter() - l = [ + s = [ *celestial.proto_util.make_update_request_iter( serializer.diff_machines(t), serializer.diff_links(t) ) @@ -153,7 +153,7 @@ def get_diff( logging.debug(f"diffs took {time.perf_counter() - t1} seconds") - return l + return s # start the simulation timestep: celestial.types.timestamp_s = 0 @@ -171,9 +171,9 @@ def get_diff( logging.info(f"Updating for timestep {timestep}") with concurrent.futures.ThreadPoolExecutor() as e: - for h in range(len(hosts)): + for i in range(len(hosts)): # need to make some generators - e.submit(hosts[h].update, (u for u in updates)) + e.submit(hosts[i].update, (u for u in updates)) timestep += config.resolution @@ -194,8 +194,8 @@ def get_diff( finally: logging.info("got keyboard interrupt, stopping...") with concurrent.futures.ThreadPoolExecutor() as e: - for h in range(len(hosts)): + for i in range(len(hosts)): # need to make some generators - e.submit(hosts[h].stop) + e.submit(hosts[i].stop) logging.info("finished") diff --git a/celestial/host.py b/celestial/host.py index 76fddc7..fda1118 100644 --- a/celestial/host.py +++ b/celestial/host.py @@ -54,7 +54,9 @@ def register(self) -> proto.celestial.celestial_pb2.RegisterResponse: """ request = proto.celestial.celestial_pb2.RegisterRequest(host=self.num) - response = self.stub.Register(request) + response: proto.celestial.celestial_pb2.RegisterResponse = self.stub.Register( + request + ) # others currently not used self.peer_public_key = response.peer_public_key diff --git a/celestial/proto_util.py b/celestial/proto_util.py index 504b0d8..bbb7e34 100644 --- a/celestial/proto_util.py +++ b/celestial/proto_util.py @@ -30,6 +30,14 @@ MAX_DIFF_UPDATE_SIZE = 100_000 +def _machineID_group(m: celestial.types.MachineID_dtype) -> int: + return typing.cast(int, celestial.types.MachineID_group(m)) + + +def _machineID_id(m: celestial.types.MachineID_dtype) -> int: + return typing.cast(int, celestial.types.MachineID_id(m)) + + def make_init_request( hosts: typing.List[celestial.host.Host], machines: typing.Dict[ @@ -53,8 +61,8 @@ def make_init_request( for m in machines[h.num]: mid = proto.celestial.celestial_pb2.MachineID( - group=celestial.types.MachineID_group(m[0]), - id=celestial.types.MachineID_id(m[0]), + group=_machineID_group(m[0]), + id=_machineID_id(m[0]), ) mc = proto.celestial.celestial_pb2.InitRequest.Machine.MachineConfig( @@ -66,19 +74,22 @@ def make_init_request( boot_parameters=m[1].boot_parameters, ) - m = proto.celestial.celestial_pb2.InitRequest.Machine( + r = proto.celestial.celestial_pb2.InitRequest.Machine( id=mid, config=mc, host=h.num, name=celestial.types.MachineID_name(m[0]), ) - init_request.machines.append(m) + init_request.machines.append(r) return init_request -def _islice(iterable: typing.Iterator, n: int) -> typing.Iterator: +T = typing.TypeVar("T") + + +def _islice(iterable: typing.Iterator[T], n: int) -> typing.Iterator[T]: count = 0 for i in iterable: @@ -114,8 +125,8 @@ def make_update_request_iter( machine_diffs=[ proto.celestial.celestial_pb2.StateUpdateRequest.MachineDiff( id=proto.celestial.celestial_pb2.MachineID( - group=celestial.types.MachineID_group(m_id), - id=celestial.types.MachineID_id(m_id), + group=_machineID_group(m_id), + id=_machineID_id(m_id), ), active=proto.celestial.celestial_pb2.VM_STATE_STOPPED if m_state == celestial.types.VMState.STOPPED @@ -131,42 +142,36 @@ def make_update_request_iter( network_diffs=[ proto.celestial.celestial_pb2.StateUpdateRequest.NetworkDiff( source=proto.celestial.celestial_pb2.MachineID( - group=celestial.types.MachineID_group(source), - id=celestial.types.MachineID_id(source), + group=_machineID_group(source), + id=_machineID_id(source), ), target=proto.celestial.celestial_pb2.MachineID( - group=celestial.types.MachineID_group(target), - id=celestial.types.MachineID_id(target), + group=_machineID_group(target), + id=_machineID_id(target), + ), + latency=typing.cast(int, celestial.types.Link_latency_us(link)), + bandwidth=typing.cast( + int, celestial.types.Link_bandwidth_kbits(link) ), - latency=celestial.types.Link_latency_us(link), - bandwidth=celestial.types.Link_bandwidth_kbits(link), blocked=False, next=proto.celestial.celestial_pb2.MachineID( - group=celestial.types.MachineID_group( - celestial.types.Link_next_hop(link) - ), - id=celestial.types.MachineID_id( - celestial.types.Link_next_hop(link) - ), + group=_machineID_group(celestial.types.Link_next_hop(link)), + id=_machineID_id(celestial.types.Link_next_hop(link)), ), prev=proto.celestial.celestial_pb2.MachineID( - group=celestial.types.MachineID_group( - celestial.types.Link_prev_hop(link) - ), - id=celestial.types.MachineID_id( - celestial.types.Link_prev_hop(link) - ), + group=_machineID_group(celestial.types.Link_prev_hop(link)), + id=_machineID_id(celestial.types.Link_prev_hop(link)), ), ) if not celestial.types.Link_blocked(link) else proto.celestial.celestial_pb2.StateUpdateRequest.NetworkDiff( source=proto.celestial.celestial_pb2.MachineID( - group=celestial.types.MachineID_group(source), - id=celestial.types.MachineID_id(source), + group=_machineID_group(source), + id=_machineID_id(source), ), target=proto.celestial.celestial_pb2.MachineID( - group=celestial.types.MachineID_group(target), - id=celestial.types.MachineID_id(target), + group=_machineID_group(target), + id=_machineID_id(target), ), blocked=True, ) diff --git a/celestial/satgen.py b/celestial/satgen_connstellation.py similarity index 100% rename from celestial/satgen.py rename to celestial/satgen_connstellation.py diff --git a/celestial/shell.py b/celestial/shell.py index 4f01494..bffc263 100644 --- a/celestial/shell.py +++ b/celestial/shell.py @@ -543,7 +543,7 @@ def _init_ground_stations( ) init_pos[2] = (EARTH_RADIUS_M + 100.0) * math.sin(latitude) - temp: npt.NDArray[GROUNDPOINT_DTYPE] = np.zeros(1, dtype=GROUNDPOINT_DTYPE) + temp: npt.NDArray = np.zeros(1, dtype=GROUNDPOINT_DTYPE) # type: ignore temp[0]["ID"] = np.int16(-i - 1) diff --git a/pkg/ebpfem/ebpfem_test.go b/pkg/ebpfem/ebpfem_test.go index 65822a4..4b43577 100644 --- a/pkg/ebpfem/ebpfem_test.go +++ b/pkg/ebpfem/ebpfem_test.go @@ -65,6 +65,7 @@ func TestBasic(t *testing.T) { if _, err := cmd.CombinedOutput(); err != nil { // ignore + log.Debugf("could not delete tap %s: %s", tap, err) } // iptables -D FORWARD -i [TAP_NAME] -o [HOSTINTERFACE] -j ACCEPT @@ -73,6 +74,7 @@ func TestBasic(t *testing.T) { if _, err := cmd.CombinedOutput(); err != nil { // ignore + log.Debugf("could not delete iptables rule: %s", err) } // create a tap device diff --git a/pkg/peer/peer.go b/pkg/peer/peer.go index 85559d1..6f9e972 100644 --- a/pkg/peer/peer.go +++ b/pkg/peer/peer.go @@ -241,6 +241,10 @@ func (p *PeeringService) InitPeering(remotes map[orchestrator.Host]HostInfo) err portNum, err := strconv.ParseUint(port, 10, 16) + if err != nil { + return errors.WithStack(err) + } + r := &peer{ directAddr: net.ParseIP(addr), wgAddr: remoteWgAddr, diff --git a/pkg/virt/machine.go b/pkg/virt/machine.go index 18405ea..9e59eef 100644 --- a/pkg/virt/machine.go +++ b/pkg/virt/machine.go @@ -149,7 +149,7 @@ func (m *machine) initialize() error { return errors.WithStack(err) } - loglevel := "ERROR" + var loglevel string // unfortunately Firecracker is incredibly verbose switch log.GetLevel() { diff --git a/pkg/virt/net.go b/pkg/virt/net.go index 71dcbb5..0244ff0 100644 --- a/pkg/virt/net.go +++ b/pkg/virt/net.go @@ -104,7 +104,7 @@ func removeNetworkDevice(tapName string, hostInterface string) error { } // createNetworkDevice creates a new network device for a microVM. -func createNetworkDevice(gateway net.IPNet, tapName string, hostInterface string) error { +func createNetworkDevice(gateway net.IPNet, tapName string, _ string) error { // ip tuntap add [TAP_NAME] mode tap diff --git a/requirements.txt b/requirements.txt index 28183f2..63bbfcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ cerberus==1.3.5 grpcio==1.59.0 grpcio-tools==1.59.0 llvmlite==0.41.1 -mypy==1.6.1 +mypy==1.9.0 mypy-protobuf==3.5.0 numba==0.58.1 numpy==1.26.1 diff --git a/satgen.py b/satgen.py index 49de381..0990c11 100644 --- a/satgen.py +++ b/satgen.py @@ -53,7 +53,7 @@ import celestial.config import celestial.zip_serializer -import celestial.satgen +import celestial.satgen_connstellation if __name__ == "__main__": if len(sys.argv) > 3 or len(sys.argv) < 2: @@ -77,7 +77,9 @@ serializer = celestial.zip_serializer.ZipSerializer(config, output_file) # init the constellation - constellation = celestial.satgen.SatgenConstellation(config, serializer) + constellation = celestial.satgen_connstellation.SatgenConstellation( + config, serializer + ) # run the simulation i = 0 diff --git a/setup.cfg b/setup.cfg index 07d9953..8c3a88b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,18 +2,12 @@ python_version = 3.11 warn_unused_configs = True disallow_any_generics = True -disallow_subclassing_any = True disallow_untyped_calls = True disallow_untyped_defs = True disallow_incomplete_defs = True check_untyped_defs = True -disallow_untyped_decorators = True -no_implicit_optional = True -warn_redundant_casts = True warn_return_any = True -no_implicit_reexport = True strict_equality = True ignore_missing_imports = True show_column_numbers = True -follow_imports = skip plugins = numpy.typing.mypy_plugin