Merge branch 'main' into vineet-work

.github/workflows/ci-test-debian.yml

@@ -38,8 +38,8 @@ jobs:
           tools/ci-build.sh
 
       - name: Run tests (Ubuntu 22.04)
-        # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        # Need to use sudo for the eBPF kernel tests and need to avoid p4tc_stf tests.
+        run: sudo -E ctest --output-on-failure --schedule-random -E "p4tc_samples_stf|p4tc_cleanup|p4tc_setup"
         working-directory: ./build
 
   # Build with GCC and test Tofino backend on Ubuntu 22.04.
@@ -116,7 +116,7 @@ jobs:
           tools/ci-build.sh
 
       - name: Run tests (Ubuntu 20.04)
-        # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        # Need to use sudo for the eBPF kernel tests and need to avoid p4tc_stf tests.
+        run: sudo -E ctest --output-on-failure --schedule-random -E "p4tc_samples_stf|p4tc_cleanup|p4tc_setup"
         working-directory: ./build
         if: matrix.unity == 'ON' && matrix.gtest == 'ON'

.github/workflows/ci-test-fedora.yml

@@ -50,5 +50,6 @@ jobs:
 
       - name: Run p4c tests (Fedora Linux)
         run: |
-          export PATH="$HOME/.local/bin:$PATH"; ctest --output-on-failure --schedule-random
+          # Avoid running p4tc stf tests for now
+          export PATH="$HOME/.local/bin:$PATH"; ctest --output-on-failure --schedule-random -E "p4tc_samples_stf|p4tc_cleanup|p4tc_setup"
         working-directory: ./build

.github/workflows/ci-test-mac.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith|p4tc"
         working-directory: ./build
 
   # Build and test p4c on MacOS 13 on x86.
@@ -105,5 +105,5 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith|p4tc"
         working-directory: ./build

.github/workflows/ci-ubuntu-18-nightly-p4tc-stf.yml

@@ -0,0 +1,43 @@
+name: "test-p4c-ubuntu-18.04"
+
+on:
+  schedule:
+    # Every day on midnight UTC
+    - cron: "0 0 * * *"
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+jobs:
+  # Run p4tc stf tests
+  test-ubuntu18-p4tc-stf:
+    # Only run on pull requests with the "p4tc" label.
+    if: ${{ github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'p4tc') }}
+    runs-on: ubuntu-20.04
+    env:
+      CTEST_PARALLEL_LEVEL: 4
+      IMAGE_TYPE: test
+      ENABLE_GTESTS: ${{ matrix.gtest }}
+      CMAKE_UNITY_BUILD: ${{ matrix.unity }}
+      BUILD_GENERATOR: Ninja
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          key: test-${{ matrix.unity }}-${{ runner.os }}-gcc
+          max-size: 1000M
+
+      - name: Build (Ubuntu 20.04, GCC)
+        run: |
+          tools/ci-build.sh
+
+      - name: Run tests (Ubuntu 20.04)
+        # Need to use sudo for the eBPF kernel tests.
+        run: sudo -E ctest --output-on-failure --schedule-random -R "p4tc_samples_stf|p4tc_cleanup|p4tc_setup"
+        working-directory: ./build

.github/workflows/ci-ubuntu-18-nightly.yml

@@ -39,4 +39,5 @@ jobs:
         # this is needed to create network namespaces for the ebpf tests.
       - name: Run tests (Ubuntu 18.04)
         run: |
-          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c ctest --output-on-failure --schedule-random
+          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c ctest --output-on-failure --schedule-random \
+          -E "p4tc_samples_stf|p4tc_cleanup|p4tc_setup"

.github/workflows/ci-ubuntu-20-sanitizer-nightly.yml

@@ -40,5 +40,5 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: sudo -E ctest --output-on-failure --schedule-random -E "p4tc_samples_stf|p4tc_cleanup|p4tc_setup"
         working-directory: ./build

backends/ebpf/codeGen.h

@@ -82,8 +82,8 @@ class CodeGenInspector : public Inspector {
     }
     bool isPointerVariable(cstring name) { return asPointerVariables.count(name) > 0; }
 
-    bool notSupported(const IR::Expression *expression) {
-        ::P4::error(ErrorType::ERR_UNSUPPORTED, "%1%: not yet implemented", expression);
+    bool notSupported(const IR::Node *n) {
+        ::P4::error(ErrorType::ERR_UNSUPPORTED, "%1%: not yet implemented", n);
         return false;
     }
 
@@ -117,6 +117,7 @@ class CodeGenInspector : public Inspector {
     bool preorder(const IR::Type_Enum *type) override;
     void emitAssignStatement(const IR::Type *ltype, const IR::Expression *lexpr, cstring lpath,
                              const IR::Expression *rexpr);
+    bool preorder(const IR::BaseAssignmentStatement *s) override { return notSupported(s); }
     bool preorder(const IR::AssignmentStatement *s) override;
     bool preorder(const IR::BlockStatement *s) override;
     bool preorder(const IR::MethodCallStatement *s) override;

backends/ebpf/ebpfParser.h

@@ -62,6 +62,7 @@ class StateTranslationVisitor : public CodeGenInspector {
         builder->endOfStatement(true);
         return false;
     }
+    bool preorder(const IR::BaseAssignmentStatement *stat) override { return notSupported(stat); }
     bool preorder(const IR::AssignmentStatement *stat) override;
 };
 

backends/ebpf/psa/ebpfPsaControl.h

@@ -31,6 +31,7 @@ class ControlBodyTranslatorPSA : public ControlBodyTranslator {
  public:
     explicit ControlBodyTranslatorPSA(const EBPFControlPSA *control);
 
+    bool preorder(const IR::BaseAssignmentStatement *a) override { return notSupported(a); }
     bool preorder(const IR::AssignmentStatement *a) override;
 
     void processMethod(const P4::ExternMethod *method) override;

backends/graphs/controls.cpp

@@ -226,7 +226,7 @@ bool ControlGraphs::preorder(const IR::MethodCallStatement *statement) {
     return false;
 }
 
-bool ControlGraphs::preorder(const IR::AssignmentStatement *statement) {
+bool ControlGraphs::preorder(const IR::BaseAssignmentStatement *statement) {
     statementsStack.push_back(statement);
     return false;
 }

backends/graphs/controls.h

@@ -45,7 +45,7 @@ class ControlGraphs : public Graphs {
     bool preorder(const IR::IfStatement *statement) override;
     bool preorder(const IR::SwitchStatement *statement) override;
     bool preorder(const IR::MethodCallStatement *statement) override;
-    bool preorder(const IR::AssignmentStatement *statement) override;
+    bool preorder(const IR::BaseAssignmentStatement *statement) override;
     bool preorder(const IR::ReturnStatement *) override;
     bool preorder(const IR::ExitStatement *) override;
     bool preorder(const IR::P4Table *table) override;

backends/p4fmt/p4formatter.cpp

@@ -920,6 +920,16 @@ bool P4Formatter::preorder(const IR::AssignmentStatement *a) {
     return false;
 }
 
+bool P4Formatter::preorder(const IR::OpAssignmentStatement *a) {
+    visit(a->left);
+    builder.append(" ");
+    builder.append(a->getStringOp());
+    builder.append("= ");
+    visit(a->right);
+    builder.endOfStatement();
+    return false;
+}
+
 bool P4Formatter::preorder(const IR::BlockStatement *s) {
     if (printAnnotations(s)) builder.spc();
     builder.blockStart();

backends/p4fmt/p4formatter.h

@@ -248,6 +248,7 @@ class P4Formatter : public Inspector, ::P4::ResolutionContext {
 
     // statements
     bool preorder(const IR::AssignmentStatement *s) override;
+    bool preorder(const IR::OpAssignmentStatement *s) override;
     bool preorder(const IR::BlockStatement *s) override;
     bool preorder(const IR::MethodCallStatement *s) override;
     bool preorder(const IR::EmptyStatement *s) override;

backends/p4test/p4test.cpp

@@ -87,6 +87,39 @@ class P4TestOptions : public CompilerOptions {
     }
 };
 
+class P4TestPragmas : public P4::P4COptionPragmaParser {
+    std::optional<IOptionPragmaParser::CommandLineOptions> tryToParse(
+        const IR::Annotation *annotation) {
+        if (annotation->name == "test_keep_opassign") {
+            test_keepOpAssign = true;
+            return std::nullopt;
+        }
+        return P4::P4COptionPragmaParser::tryToParse(annotation);
+    }
+
+ public:
+    P4TestPragmas() : P4::P4COptionPragmaParser(true) {}
+
+    bool test_keepOpAssign = false;
+};
+
+class TestFEPolicy : public P4::FrontEndPolicy {
+    const P4TestPragmas &pragmas;
+
+    P4::ParseAnnotations *getParseAnnotations() const {
+        return new P4::ParseAnnotations("p4test", true,
+                                        P4::ParseAnnotations::HandlerMap({
+                                            PARSE_EMPTY("test_keep_opassign"_cs),
+                                        }),
+                                        false);
+    }
+
+    bool removeOpAssign() const { return !pragmas.test_keepOpAssign; }
+
+ public:
+    explicit TestFEPolicy(const P4TestPragmas &pragmas) : pragmas(pragmas) {}
+};
+
 using P4TestContext = P4CContextWithOptions<P4TestOptions>;
 
 static void log_dump(const IR::Node *node, const char *head) {
@@ -135,13 +168,14 @@ int main(int argc, char *const argv[]) {
         info.emitInfo("PARSER");
 
         if (program != nullptr && ::P4::errorCount() == 0) {
-            P4::P4COptionPragmaParser optionsPragmaParser(true);
-            program->apply(P4::ApplyOptionsPragmas(optionsPragmaParser));
+            P4TestPragmas testPragmas;
+            program->apply(P4::ApplyOptionsPragmas(testPragmas));
             info.emitInfo("PASS P4COptionPragmaParser");
 
             if (!options.parseOnly) {
                 try {
-                    P4::FrontEnd fe;
+                    TestFEPolicy fe_policy(testPragmas);
+                    P4::FrontEnd fe(&fe_policy);
                     fe.addDebugHook(hook);
                     // use -TdiagnosticCountInPass:1 / -TdiagnosticCountInPass:4 to get output of
                     // this hook

backends/p4tools/modules/smith/common/statements.cpp

@@ -166,10 +166,6 @@ void StatementGenerator::removeLval(const IR::Expression *left, const IR::Type *
 }
 
 IR::Statement *StatementGenerator::genAssignmentStatement() {
-    IR::AssignmentStatement *assignstat = nullptr;
-    IR::Expression *left = nullptr;
-    IR::Expression *right = nullptr;
-
     std::vector<int64_t> percent = {
         Probabilities::get().ASSIGNMENTORMETHODCALLSTATEMENT_ASSIGN_BIT,
         Probabilities::get().ASSIGNMENTORMETHODCALLSTATEMENT_ASSIGN_STRUCTLIKE};
@@ -183,19 +179,19 @@ IR::Statement *StatementGenerator::genAssignmentStatement() {
                 // TODO(fruffy): Find a more meaningful assignment statement
                 return nullptr;
             }
-            left = target().expressionGenerator().pickLvalOrSlice(bitType);
+            auto *left = target().expressionGenerator().pickLvalOrSlice(bitType);
             if (P4Scope::constraints.single_stage_actions) {
                 removeLval(left, bitType);
             }
-            right = target().expressionGenerator().genExpression(bitType);
+            auto *right = target().expressionGenerator().genExpression(bitType);
             return new IR::AssignmentStatement(left, right);
         }
         case 1:
             // TODO(fruffy): Compound types
             break;
     }
 
-    return assignstat;
+    return nullptr;
 }
 
 IR::Statement *StatementGenerator::genMethodCallExpression(const IR::PathExpression *methodName,

backends/p4tools/modules/testgen/lib/collect_coverable_nodes.cpp

@@ -48,7 +48,7 @@ bool CoverableNodesScanner::preorder(const IR::ParserState *parserState) {
     return true;
 }
 
-bool CoverableNodesScanner::preorder(const IR::AssignmentStatement *stmt) {
+bool CoverableNodesScanner::preorder(const IR::BaseAssignmentStatement *stmt) {
     // Only track statements, which have a valid source position in the P4 program.
     if (coverageOptions.coverStatements && stmt->getSourceInfo().isValid()) {
         coverableNodes.insert(stmt);

backends/p4tools/modules/testgen/lib/collect_coverable_nodes.h

@@ -34,7 +34,7 @@ class CoverableNodesScanner : public Inspector {
 
     /// Statement coverage.
     bool preorder(const IR::ParserState *parserState) override;
-    bool preorder(const IR::AssignmentStatement *stmt) override;
+    bool preorder(const IR::BaseAssignmentStatement *stmt) override;
     bool preorder(const IR::MethodCallStatement *stmt) override;
     bool preorder(const IR::ExitStatement *stmt) override;
     bool preorder(const IR::MethodCallExpression *call) override;

backends/tc/CMakeLists.txt

@@ -14,6 +14,49 @@
 # and limitations under the License.
 #*****************************************************************************/
 
+if(NOT APPLE)
+  # Fetch and declare the libbpf library. Print out download state while setting up libbpf.
+  set(FETCHCONTENT_QUIET_PREV ${FETCHCONTENT_QUIET})
+  set(FETCHCONTENT_QUIET OFF)
+  fetchcontent_declare(
+    p4cbpfrepo
+    URL https://github.com/libbpf/libbpf/archive/refs/tags/v1.5.0.tar.gz
+    URL_HASH SHA256=53492aff6dd47e4da04ef5e672d753b9743848bdb38e9d90eafbe190b7983c44
+    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/runtime/libbpf
+    USES_TERMINAL_DOWNLOAD TRUE
+    GIT_PROGRESS TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+  )
+  fetchcontent_makeavailable(p4cbpfrepo)
+  set(FETCHCONTENT_QUIET ${FETCHCONTENT_QUIET_PREV})
+  message("Building libbpf...")
+  execute_process(
+    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/runtime/build-libbpf
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    COMMAND_ECHO STDOUT
+  )
+  message("Done with setting up libbpf for P4C.")
+
+  fetchcontent_declare(
+    iproute2repo
+    URL https://github.com/p4tc-dev/iproute2-p4tc-pub/archive/refs/tags/release-v17-rc6.tar.gz
+    URL_HASH SHA256=624c32a571f9f30d1070d9b23e96121ac79f9273df9ff6db4ee6d034ab983c5d
+    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/runtime/iproute2-p4tc-pub
+    USES_TERMINAL_DOWNLOAD TRUE
+    GIT_PROGRESS TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+  )
+  fetchcontent_makeavailable(iproute2repo)
+  set(FETCHCONTENT_QUIET ${FETCHCONTENT_QUIET_PREV})
+  message("Building iproute2...")
+  execute_process(
+    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/runtime/build-iproute2 ${CMAKE_CURRENT_SOURCE_DIR}/runtime
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    COMMAND_ECHO STDOUT
+  )
+  message("Done with setting up iproute2 for P4C.")
+endif()
+
 set(P4TC_BACKEND_SOURCES
     backend.cpp
     ebpfCodeGen.cpp
@@ -123,4 +166,21 @@ set(P4TC_COMPILER_DRIVER "${CMAKE_CURRENT_SOURCE_DIR}/run-tc-test.py")
 set (P4_16_SUITES
   "${P4C_SOURCE_DIR}/testdata/p4tc_samples/*.p4")
 
+# Setup fixture
+add_test(NAME p4tc_setup COMMAND bash ${P4C_SOURCE_DIR}/backends/tc/runtime/setup "https://api.github.com/repos/p4tc-dev/linux-p4tc-pub/releases/latest")
+set_tests_properties(p4tc_setup PROPERTIES FIXTURES_SETUP P4TCFixture)
+
+add_test(NAME p4tc_cleanup COMMAND bash ${P4C_SOURCE_DIR}/backends/tc/runtime/cleanup)
+set_tests_properties(p4tc_cleanup PROPERTIES FIXTURES_CLEANUP P4TCFixture)
+
+macro(p4tc_add_test_with_args tag driver isXfail alias p4test test_args cmake_args)
+  p4c_add_test_with_args(${tag} ${driver} ${isXfail} ${alias} ${p4test} ${test_args} "")
+  p4c_test_set_name(__testname ${tag} ${alias})
+  set_tests_properties(${__testname} PROPERTIES FIXTURES_REQUIRED P4TCFixture RESOURCE_LOCK "shared_lock")
+  set_tests_properties(${__testname} PROPERTIES RESOURCE_LOCK "shared_lock")
+  set_tests_properties(${__testname} PROPERTIES TIMEOUT 1000)
+endmacro(p4tc_add_test_with_args)
+
 p4c_add_tests("p4tc" ${P4TC_COMPILER_DRIVER} "${P4_16_SUITES}" "")
+p4tc_add_test_with_args("p4tc" ${P4TC_COMPILER_DRIVER} FALSE "testdata/p4tc_samples_stf/arp_respond.p4" "testdata/p4tc_samples_stf/arp_respond.p4" "-tf ${P4C_SOURCE_DIR}/testdata/p4tc_samples_stf/arp_respond.stf" "")
+p4tc_add_test_with_args("p4tc" ${P4TC_COMPILER_DRIVER} FALSE "testdata/p4tc_samples_stf/simple_l3.p4" "testdata/p4tc_samples_stf/simple_l3.p4" "-tf ${P4C_SOURCE_DIR}/testdata/p4tc_samples_stf/simple_l3.stf" "")