T0000: Test PR dont merge

.github/workflows/check-unused-imports.yml

@@ -10,5 +10,5 @@ permissions:
 
 jobs:
   check-unused-imports:
-    uses: vyos/.github/.github/workflows/check-unused-imports.yml@feature/check-pr-commenting
+    uses: vyos/.github/.github/workflows/check-unused-imports.yml@circinus
     secrets: inherit

CODEOWNERS

@@ -1 +1 @@
-* @kumvijaya
+* @vyos/reviewers

README.md

@@ -1,12 +1,80 @@
-# Validates the common reusable github action workflows
+# vyos-1x: VyOS command definitions, configuration scripts, and data
+TEST - 4
 
+VyOS 1.1.x had its codebase split into way too many submodules for no good
+reason, which made it hard to navigate or write meaningful changelogs. As the
+code undergoes rewrite in the new style in VyOS 1.2.0+, we consolidate the
+rewritten code in this package.
 
-## Package layout for validation and tests. Test 2
+If you just want to build a VyOS image, the repository you want is
+[vyos-build](https://github.com/vyos/vyos-build). If you also want to contribute
+to VyOS, read on.
+
+## Package layout
 
 ```
 data                   # Jinja2 templates
+debian                 # Our package build system
+interface-definitions  # Configuration interface (i.e. conf mode command) definitions
+op-mode-definitions    # Operational command definitions
+python                 # Programming library for accessing the config frontend/backend
+schema                 # XML schema definition for conf mode and op mode commands
+scripts                # Build-time scripts
+smoketest              # Python Unittests for the CLI and system
 src
+    conf_mode/  # Configuration mode scripts
+    op_mode/    # Operational mode scripts
+    completion/ # Completion helpers
     validators/ # Value validators
     helpers/    # Misc helpers
+    migration-scripts # Migration scripts
     tests/      # Unit tests
 ```
+
+## Interface/command definitions
+
+Raw `node.def` files for the old backend are no longer written by hand or
+generated by custom scripts. They are all now produced from a unified XML format
+that supports a strict subset of the old backend features. In particular, it
+intentionally does not support embedded shell scripts, default values, and value
+"types", instead delegating those tasks to external scripts.
+
+Configuration interface definitions must conform to the schema found in
+`schema/interface_definition.rng` and operational command definitions must
+conform to `schema/op-mode-definition.rng`. Schema checks are performed at build
+time, so a package with malformed interface definitions will not build.
+
+## Configuration scripts
+
+The guidelines in a nutshell:
+
+* Use separate functions for retrieving configuration data, validating it, and
+  generating taret config, see our
+  [documentation](https://docs.vyos.io/en/latest/contributing/development.html#python)
+  for the common structure
+* Use the `get_config_dict()` API as much as possible when retrieving values from the CLI
+* Use a template processor when the format is more complex than just one line
+  (our standard is Jinja2)
+
+## Tests
+
+Tests are executed at build time, you can also execute them by hand with:
+
+```
+pipenv install --dev
+pipenv shell
+make test
+```
+
+### Runtime (Smoke Tests)
+
+Runtime tests are executed by the CI system on a running VyOS instance inside
+QEMU. The testcases can be found inside the smoketest subdirectory which will
+be placed into the vyos-1x-smoketest package.
+
+
+### Thanks to all the people who already contributed!
+
+<a href="https://github.com/vyos/vyos-1x/graphs/contributors">
+  <img src="https://contributors-img.web.app/image?repo=vyos/vyos-1x" />
+</a>

data/config-mode-dependencies/vyos-1x.json

@@ -29,8 +29,11 @@
         "https": ["service_https"],
         "ipsec": ["vpn_ipsec"],
         "openconnect": ["vpn_openconnect"],
+        "reverse_proxy": ["load-balancing_reverse-proxy"],
         "rpki": ["protocols_rpki"],
-        "sstp": ["vpn_sstp"]
+        "sstp": ["vpn_sstp"],
+        "sstpc": ["interfaces_sstpc"],
+        "stunnel": ["service_stunnel"]
     },
     "vpn_ipsec": {
         "nhrp": ["protocols_nhrp"]
@@ -57,5 +60,18 @@
         "wireguard": ["interfaces_wireguard"],
         "wireless": ["interfaces_wireless"],
         "wwan": ["interfaces_wwan"]
+    },
+    "system_wireless": {
+        "wireless": ["interfaces_wireless"]
+    },
+    "system_ip": {
+        "sysctl": ["system_sysctl"]
+    },
+    "system_ipv6": {
+        "sysctl": ["system_sysctl"]
+    },
+    "system_option": {
+        "ip_ipv6": ["system_ip", "system_ipv6"],
+        "sysctl": ["system_sysctl"]
     }
 }

data/op-mode-standardized.json

@@ -23,6 +23,7 @@
 "openconnect.py",
 "openvpn.py",
 "otp.py",
+"qos.py",
 "reset_vpn.py",
 "reverseproxy.py",
 "route.py",

data/templates/accel-ppp/ipoe.config.j2

@@ -16,6 +16,9 @@ net-snmp
 {% if limits is vyos_defined %}
 connlimit
 {% endif %}
+{% if extended_scripts is vyos_defined %}
+pppd_compat
+{% endif %}
 
 [core]
 thread-count={{ thread_count }}
@@ -29,7 +32,9 @@ max-starting={{ max_concurrent_sessions }}
 [log]
 syslog=accel-ipoe,daemon
 copy=1
-level=5
+{% if log.level is vyos_defined %}
+level={{ log.level }}
+{% endif %}
 
 [ipoe]
 verbose=1
@@ -51,7 +56,7 @@ verbose=1
 {%         set relay = ',' ~ 'relay=' ~ iface_config.external_dhcp.dhcp_relay  if iface_config.external_dhcp.dhcp_relay is vyos_defined else '' %}
 {%         set giaddr = ',' ~ 'giaddr=' ~ iface_config.external_dhcp.giaddr if iface_config.external_dhcp.giaddr is vyos_defined else '' %}
 {{ tmp }},{{ shared }}mode={{ iface_config.mode | upper }},ifcfg=1,{{ range }}start=dhcpv4,ipv6=1{{ relay }}{{ giaddr }}
-{%         if iface_config.vlan is vyos_defined %}
+{%         if iface_config.vlan_mon is vyos_defined %}
 vlan-mon={{ iface }},{{ iface_config.vlan | join(',') }}
 {%         endif %}
 {%     endfor %}

data/templates/accel-ppp/l2tp.config.j2

@@ -16,6 +16,9 @@ net-snmp
 {% if limits is vyos_defined %}
 connlimit
 {% endif %}
+{% if extended_scripts is vyos_defined %}
+pppd_compat
+{% endif %}
 
 [core]
 thread-count={{ thread_count }}
@@ -28,7 +31,9 @@ max-starting={{ max_concurrent_sessions }}
 [log]
 syslog=accel-l2tp,daemon
 copy=1
-level=5
+{% if log.level is vyos_defined %}
+level={{ log.level }}
+{% endif %}
 
 [client-ip-range]
 0.0.0.0/0

data/templates/accel-ppp/pppoe.config.j2

@@ -27,12 +27,17 @@ thread-count={{ thread_count }}
 [log]
 syslog=accel-pppoe,daemon
 copy=1
-level=5
+{% if log.level is vyos_defined %}
+level={{ log.level }}
+{% endif %}
 
-{% if authentication.mode is vyos_defined("noauth") %}
 [auth]
+{% if authentication.mode is vyos_defined("noauth") %}
 noauth=1
 {% endif %}
+{% if authentication.any_login is vyos_defined %}
+any-login=1
+{% endif %}
 
 [client-ip-range]
 0.0.0.0/0
@@ -56,13 +61,21 @@ interface={{ iface }}
 {%             for vlan in iface_config.vlan %}
 interface=re:^{{ iface }}\.{{ vlan | range_to_regex }}$
 {%             endfor %}
+{%             if iface_config.vlan_mon is vyos_defined %}
 vlan-mon={{ iface }},{{ iface_config.vlan | join(',') }}
+{%             endif %}
 {%         endif %}
 {%     endfor %}
 {% endif %}
 {% if service_name %}
 service-name={{ service_name | join(',') }}
 {% endif %}
+{% if accept_any_service is vyos_defined %}
+accept-any-service=1
+{% endif %}
+{% if accept_blank_service is vyos_defined %}
+accept-blank-service=1
+{% endif %}
 {% if pado_delay %}
 {%     set delay_without_sessions = pado_delay.delays_without_sessions[0] | default('0') %}
 {%     set pado_delay_param = namespace(value=delay_without_sessions) %}

data/templates/accel-ppp/pptp.config.j2

@@ -16,6 +16,9 @@ net-snmp
 {% if limits is vyos_defined %}
 connlimit
 {% endif %}
+{% if extended_scripts is vyos_defined %}
+pppd_compat
+{% endif %}
 
 [core]
 thread-count={{ thread_count }}
@@ -28,7 +31,9 @@ max-starting={{ max_concurrent_sessions }}
 [log]
 syslog=accel-pptp,daemon
 copy=1
-level=5
+{% if log.level is vyos_defined %}
+level={{ log.level }}
+{% endif %}
 
 [client-ip-range]
 0.0.0.0/0

data/templates/accel-ppp/sstp.config.j2

@@ -16,6 +16,9 @@ net-snmp
 {% if limits is vyos_defined %}
 connlimit
 {% endif %}
+{% if extended_scripts is vyos_defined %}
+pppd_compat
+{% endif %}
 
 [core]
 thread-count={{ thread_count }}
@@ -29,7 +32,9 @@ max-starting={{ max_concurrent_sessions }}
 [log]
 syslog=accel-sstp,daemon
 copy=1
-level=5
+{% if log.level is vyos_defined %}
+level={{ log.level }}
+{% endif %}
 
 [client-ip-range]
 0.0.0.0/0

data/templates/chrony/chrony.conf.j2

@@ -42,7 +42,7 @@ user {{ user }}
 {%         if config.pool is vyos_defined %}
 {%             set association = 'pool' %}
 {%         endif %}
-{{ association }} {{ server | replace('_', '-') }} iburst {{ 'nts' if config.nts is vyos_defined }} {{ 'noselect' if config.noselect is vyos_defined }} {{ 'prefer' if config.prefer is vyos_defined }}
+{{ association }} {{ server | replace('_', '-') }} iburst {{- ' nts' if config.nts is vyos_defined }} {{- ' noselect' if config.noselect is vyos_defined }} {{- ' prefer' if config.prefer is vyos_defined }} {{- ' xleave' if config.interleave is vyos_defined }} {{- ' port ' ~ ptp.port if ptp.port is vyos_defined and config.ptp is vyos_defined }}
 {%     endfor %}
 {% endif %}
 
@@ -66,3 +66,18 @@ bindaddress {{ address }}
 binddevice {{ interface }}
 {%     endif %}
 {% endif %}
+
+{% if ptp.timestamp.interface is vyos_defined %}
+# Enable hardware timestamping on the specified interfaces
+{%     for iface, iface_config in ptp.timestamp.interface.items() %}
+{%         if iface == "all" %}
+{%             set iface = "*" %}
+{%         endif %}
+hwtimestamp {{ iface }} {{- ' rxfilter ' ~ iface_config.receive_filter if iface_config.receive_filter is vyos_defined }}
+{%     endfor %}
+{% endif %}
+
+{% if ptp.port is vyos_defined %}
+# Enable sending and receiving NTP over PTP packets (PTP transport)
+ptpport {{ ptp.port }}
+{% endif %}

data/templates/conntrack/sysctl.conf.j2

@@ -3,25 +3,8 @@
 
 net.netfilter.nf_conntrack_expect_max = {{ expect_table_size }}
 net.netfilter.nf_conntrack_max = {{ table_size }}
-
 net.ipv4.tcp_max_syn_backlog = {{ tcp.half_open_connections }}
-
 net.netfilter.nf_conntrack_tcp_loose = {{ '1' if tcp.loose is vyos_defined('enable') else '0' }}
 net.netfilter.nf_conntrack_tcp_max_retrans = {{ tcp.max_retrans }}
-
-net.netfilter.nf_conntrack_icmp_timeout = {{ timeout.icmp }}
-net.netfilter.nf_conntrack_generic_timeout = {{ timeout.other }}
-
-net.netfilter.nf_conntrack_tcp_timeout_close_wait = {{ timeout.tcp.close_wait }}
-net.netfilter.nf_conntrack_tcp_timeout_close = {{ timeout.tcp.close }}
-net.netfilter.nf_conntrack_tcp_timeout_established = {{ timeout.tcp.established }}
-net.netfilter.nf_conntrack_tcp_timeout_fin_wait = {{ timeout.tcp.fin_wait }}
-net.netfilter.nf_conntrack_tcp_timeout_last_ack = {{ timeout.tcp.last_ack }}
-net.netfilter.nf_conntrack_tcp_timeout_syn_recv = {{ timeout.tcp.syn_recv }}
-net.netfilter.nf_conntrack_tcp_timeout_syn_sent = {{ timeout.tcp.syn_sent }}
-net.netfilter.nf_conntrack_tcp_timeout_time_wait = {{ timeout.tcp.time_wait }}
-
-net.netfilter.nf_conntrack_udp_timeout = {{ timeout.udp.other }}
-net.netfilter.nf_conntrack_udp_timeout_stream = {{ timeout.udp.stream }}
-
 net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }}
+net.netfilter.nf_conntrack_timestamp = {{ '1' if log.timestamp is vyos_defined else '0' }}

data/templates/dns-forwarding/recursor.conf.lua.j2

@@ -6,3 +6,31 @@ dofile("/usr/share/pdns-recursor/lua-config/rootkeys.lua")
 
 -- Load lua from vyos-hostsd --
 dofile("{{ config_dir }}/recursor.vyos-hostsd.conf.lua")
+
+-- ZoneToCache --
+{% if zone_cache is vyos_defined %}
+{%     set option_mapping = {
+    'refresh': 'refreshPeriod',
+    'retry_interval': 'retryOnErrorPeriod',
+    'max_zone_size': 'maxReceivedMBytes'
+} %}
+{%     for name, conf in zone_cache.items() %}
+{%         set source = conf.source.items() | first %}
+{%         set settings = [] %}
+{%         for key, val in conf.options.items() %}
+{%             set mapped_key = option_mapping.get(key, key) %}
+{%             if key == 'refresh' %}
+{%                 set val = val['interval'] %}
+{%             endif %}
+{%             if key in ['dnssec', 'zonemd'] %}
+{%                 set _ = settings.append(mapped_key ~ ' = "' ~ val ~ '"') %}
+{%             else %}
+{%                 set _ = settings.append(mapped_key ~ ' = ' ~ val) %}
+{%             endif %}
+{%         endfor %}
+
+zoneToCache("{{ name }}", "{{ source[0] }}", "{{ source[1] }}", { {{ settings | join(', ') }} })
+
+{%     endfor %}
+
+{% endif %}