Draft support for iRODS 4.3 (not yet complete)

UtrechtUniversity · Nov 22, 2023 · 41d34b4 · 41d34b4
1 parent fc36de9
commit 41d34b4
Show file tree

Hide file tree

Showing 8 changed files with 204 additions and 71 deletions.
diff --git a/README.md b/README.md
@@ -10,12 +10,11 @@ This repository contains Vagrant configurations for local iRODS VMs.
 
 # Included configurations
 
-- irods-single-server: a basic plain vanilla iRODS server for local testing. It can run on either a CentOS 7 image or a Ubuntu 18.04 LTS image.
-- irods-provider-consumer: an iRODS zone consisting of a provider and a single consumer. It can run on either a CentOS 7 image or a Ubuntu 18.04 LTS image. The VMs are meant for local testing, and run with default key values.
-- irods-icommands: a VM which contains the icommands tools for remote administration of iRODS. It can run on either a CentOS 7 image or a Ubuntu 18.04 LTS image.
+- irods-single-server: a basic plain vanilla iRODS server for local testing. It can run on either a CentOS 7 image or a Ubuntu image (18.04 LTS for iRODS 4.2.x; 20.04 LTS for iRODS 4.3.x).
+- irods-provider-consumer: an iRODS zone consisting of a provider and a single consumer. It can run on either a CentOS 7 image or a Ubuntu image (18.04 LTS for iRODS 4.2.x; 20.04 LTS for iRODS 4.3.x). The VMs are meant for local testing, and run with default key values.
+- irods-icommands: a VM which contains the icommands tools for remote administration of iRODS. It can run on either a CentOS 7 image or a Ubuntu image (18.04 LTS for iRODS 4.2.x; 20.04 LTS for iRODS 4.3.x).
 
-These scripts should support the 4.2.x iRODS versions that are available through the package repositories. As of 20 May 2023, versions 4.2.2 through 4.2.12 are available in the repositories.
-Version 4.3.0 is available in the repository, but not yet supported.
+These scripts should support the 4.2.x and 4.3.x iRODS versions that are available through the package repositories. As of 22 November 2023, versions 4.2.2 through 4.2.12, as well as 4.3.0 through 4.3.1, are available in the repositories.
 
 # Usage
 

diff --git a/_common/common_functions.sh b/_common/common_functions.sh
@@ -20,16 +20,156 @@ function get_package_version()
          then package_version="4.2.11.1"
          elif [ "$IRODS_VERSION" == "4.2.12" ] && [ "$distro" == "centos" ]
          then package_version="4.2.12.0"
+         elif [ "$IRODS_VERSION" == "4.3.0" ] && [  "$distro" == "ubuntu" ]
+         then package_version="4.3.0.0-1~focal"
+         elif [ "$IRODS_VERSION" == "4.3.0" ] && [ "$distro" == "centos" ]
+         then package_version="4.3.0.0"
+         elif [ "$IRODS_VERSION" == "4.3.1" ] && [  "$distro" == "ubuntu" ]
+         then package_version="4.3.1.0-0~focal"
+         elif [ "$IRODS_VERSION" == "4.3.1" ] && [ "$distro" == "centos" ]
+         then package_version="4.3.1.0"
          else package_version="$IRODS_VERSION"
          fi
      else
          if [ "$IRODS_VERSION" == "4.2.11" ] && [  "$distro" == "ubuntu" ]
          then package_version="4.2.11-1~xenial"
 	 elif [ "$IRODS_VERSION" == "4.2.12" ] && [ "$distro" == "ubuntu" ]
 	 then package_version="4.2.12-1~bionic"
+         elif [ "$IRODS_VERSION" == "4.3.0" ] && [  "$distro" == "ubuntu" ]
+         then package_version="4.3.0-1~focal"
+         elif [ "$IRODS_VERSION" == "4.3.1" ] && [  "$distro" == "ubuntu" ]
+         then package_version="4.3.1-0~focal"
          else # shellcheck disable=SC2034
               package_version="$IRODS_VERSION"
          fi
      fi
 }
 
+function write_irods_config()
+{
+
+cat > /etc/irods/setup_irods.json << IRODS_CONFIG_END
+{
+  "admin_password": "$ADMIN_PASSWORD",
+  "service_account_environment": {
+    "irods_host": "provider.local",
+    "irods_port": 1247,
+    "irods_user_name": "rods",
+    "irods_zone_name": "$ZONE_NAME",
+    "irods_client_server_negotiation": "request_server_negotiation",
+    "irods_client_server_policy": "CS_NEG_REFUSE",
+    "irods_cwd": "/$ZONE_NAME/home/rods",
+    "irods_default_hash_scheme": "SHA256",
+    "irods_default_resource": "demoResc",
+    "irods_encryption_algorithm": "AES-256-CBC",
+    "irods_encryption_key_size": 32,
+    "irods_encryption_salt_size": 8,
+    "irods_encryption_num_hash_rounds": 16,
+    "irods_home": "/$ZONE_NAME/home/rods",
+    "irods_match_hash_policy": "compatible"
+  },
+  "host_access_control_config": {
+    "access_entries": []
+  },
+  "hosts_config": {
+    "host_entries": []
+  },
+  "host_system_information": {
+    "service_account_user_name": "vagrant",
+    "service_account_group_name": "vagrant"
+  },
+  "server_config": {
+    "advanced_settings": {
+      "default_log_rotation_in_days": 5,
+      "default_number_of_transfer_threads": 4,
+      "default_temporary_password_lifetime_in_seconds": 120,
+      "maximum_number_of_concurrent_rule_engine_server_processes": 4,
+      "maximum_size_for_single_buffer_in_megabytes": 32,
+      "maximum_temporary_password_lifetime_in_seconds": 1000,
+      "rule_engine_server_execution_time_in_seconds": 120,
+      "rule_engine_server_sleep_time_in_seconds": 10,
+      "transfer_buffer_size_for_parallel_transfer_in_megabytes": 4,
+      "transfer_chunk_size_for_parallel_transfer_in_megabytes": 40
+    },
+    "catalog_provider_hosts": [
+      "provider.local"
+    ],
+    "catalog_service_role": "provider",
+    "default_dir_mode": "0750",
+    "default_file_mode": "0600",
+    "default_hash_scheme": "SHA256",
+    "default_resource_name": "demoResc",
+    "environment_variables": {},
+    "federation": [],
+    "host_resolution": {
+            "host_entries": []
+    },
+    "match_hash_policy": "compatible",
+    "negotiation_key": "$NEG_KEY",
+    "plugin_configuration": {
+      "authentication": {},
+      "database": {
+        "postgres": {
+          "db_host": "localhost",
+          "db_name": "ICAT",
+          "db_odbc_driver": "PostgreSQL",
+          "db_password": "$DB_PASSWORD",
+          "db_port": 5432,
+          "db_username": "irods"
+        }
+      },
+      "network": {},
+      "resource": {},
+      "rule_engines": [{
+          "instance_name": "irods_rule_engine_plugin-irods_rule_language-instance",
+          "plugin_name": "irods_rule_engine_plugin-irods_rule_language",
+          "plugin_specific_configuration": {
+            "re_data_variable_mapping_set": [
+              "core"
+            ],
+            "re_function_name_mapping_set": [
+              "core"
+            ],
+            "re_rulebase_set": [
+              "core"
+            ],
+            "regexes_for_supported_peps": [
+              "ac[^ ]*",
+              "msi[^ ]*",
+              "[^ ]*pep_[^ ]*_(pre|post)"
+            ]
+          },
+          "shared_memory_instance": "irods_rule_language_rule_engine"
+        },
+        {
+          "instance_name": "irods_rule_engine_plugin-cpp_default_policy-instance",
+          "plugin_name": "irods_rule_engine_plugin-cpp_default_policy",
+          "plugin_specific_configuration": {}
+        }
+      ]
+    },
+    "rule_engine_namespaces": [
+      ""
+    ],
+    "schema_name": "server_config",
+    "schema_validation_base_uri": "https://schemas.irods.org/configuration",
+    "schema_version": "v3",
+    "server_control_plane_encryption_algorithm": "AES-256-CBC",
+    "server_control_plane_encryption_num_hash_rounds": 16,
+    "server_control_plane_key": "$CP_KEY",
+    "server_control_plane_port": 1248,
+    "server_control_plane_timeout_milliseconds": 10000,
+    "server_port_range_end": 20000,
+    "server_port_range_start": 20199,
+    "xmsg_port": 1279,
+    "zone_auth_scheme": "native",
+    "zone_key": "$ZONE_KEY",
+    "zone_name": "$ZONE_NAME",
+    "zone_port": 1247,
+    "zone_user": "rods"
+  },
+  "default_resource_name": "demoResc"
+}
+IRODS_CONFIG_END
+
+}
diff --git a/_common/install-irods-consumer.sh b/_common/install-irods-consumer.sh
@@ -4,6 +4,7 @@
 set -e
 set -o pipefail
 set -u
+set -x
 
 export DEBIAN_FRONTEND=noninteractive
 
@@ -55,9 +56,16 @@ then
      sudo yum versionlock "$package"
   done
 
+  PYTHON_INTERPRETER="/usr/bin/python3"
+
 elif lsb_release -i | grep -q Ubuntu
 then
 
+  if [ "$BOXNAME" == "ubuntu/bionic64" ]
+  then APT_IRODS_REPO_DISTRIBUTION="xenial"
+  else APT_IRODS_REPO_DISTRIBUTION="focal"
+  fi
+
   echo "Downloading and installing iRODS repository signing key ..."
   wget -qO - "$APT_IRODS_REPO_SIGNING_KEY_LOC" | sudo apt-key add -
 
@@ -118,26 +126,8 @@ then echo "Provider was not ready in time."
      exit 1
 fi
 
-sudo python /var/lib/irods/scripts/setup_irods.py << IRODS_SETUP_END
-vagrant
-vagrant
-0
-consumer
-$ZONE_NAME
-$PROVIDER_HOSTNAME
-1247
-20000
-20199
-1248
-
-rods
-
-$ZONE_KEY
-$NEG_KEY
-$CP_KEY
-rods
-/var/lib/irods/Vault
-IRODS_SETUP_END
+write_irods_config
+sudo "$PYTHON_INTERPRETER" /var/lib/irods/scripts/setup_irods.py --json_configuration_file=/etc/irods/setup_irods.json
 
 # Restart is needed for iRODS 4.2.9+
 sudo /etc/init.d/irods restart

diff --git a/_common/install-irods-provider.sh b/_common/install-irods-provider.sh
@@ -4,6 +4,7 @@
 set -e
 set -o pipefail
 set -u
+set -x
 
 export DEBIAN_FRONTEND=noninteractive
 
@@ -68,24 +69,40 @@ then
   sudo systemctl start postgresql
   sudo systemctl enable postgresql
 
+  if [[ "$IRODS_VERSION" =~ ^4\.3\. ]]
+  then PYTHON_INTERPRETER="/usr/bin/python3"
+       sudo yum -y install gcc gcc-c++ python36-devel unixODBC-devel
+       sudo python3 -m pip install pyodbc
+  else PYTHON_INTERPRETER="/usr/bin/python"
+  fi
+
 elif lsb_release -i | grep -q Ubuntu
 then
 
+  if [ "$IRODS_VERSION" == "4.2.12" ]
+  then APT_IRODS_REPO_DISTRIBUTION="bionic"
+  elif [[ "$IRODS_VERSION" =~ ^4\.2\. ]]
+  then APT_IRODS_REPO_DISTRIBUTION="xenial"
+  else APT_IRODS_REPO_DISTRIBUTION="focal"
+  fi
+
   echo "Downloading and installing iRODS repository signing key ..."
   wget -qO - "$APT_IRODS_REPO_SIGNING_KEY_LOC" | sudo apt-key add -
 
-  echo "Adding iRODS repository ..."
- if [ "$IRODS_VERSION" == "4.2.12" ]
-  then APT_IRODS_REPO_DISTRIBUTION="bionic"
-  else APT_IRODS_REPO_DISTRIBUTION="xenial"
-  fi
 cat << ENDAPTREPO | sudo tee /etc/apt/sources.list.d/irods.list
 deb [arch=${APT_IRODS_REPO_ARCHITECTURE}] $APT_IRODS_REPO_URL $APT_IRODS_REPO_DISTRIBUTION $APT_IRODS_REPO_COMPONENT
 ENDAPTREPO
   sudo apt-get update
 
-  echo "Installing dependencies of installation script ..."
-  sudo apt-get -y install aptitude pwgen
+  if [[ "$IRODS_VERSION" =~ ^4\.3\. ]]
+  then PYTHON_INTERPRETER="/usr/bin/python3"
+       sudo apt -y install python3-pyodbc
+  else PYTHON_INTERPRETER="/usr/bin/python"
+       sudo apt -y install python-pyodbc
+  fi
+
+  echo "Installing dependencies of installation script and misc dependencies ..."
+  sudo apt-get -y install aptitude pwgen python3-pip
 
   for package in $APT_DATABASE_PACKAGES
   do echo "Installing database package $package and its dependencies ..."
@@ -122,6 +139,10 @@ if [ -z "$DB_PASSWORD" ]
 then DB_PASSWORD=$(pwgen -N 1 -n 16)
 fi
 
+if [ -z "$ADMIN_PASSWORD" ]
+then ADMIN_PASSWORD=$(pwgen -N 1 -n 16)
+fi
+
 set -u
 
 sudo -u postgres psql <<PSQL_END
@@ -131,35 +152,9 @@ GRANT ALL PRIVILEGES ON DATABASE "ICAT" TO irods;
 \q
 PSQL_END
 
-# Test configuration has been adapted from:
-# https://github.com/irods/irods/blob/4-2-stable/plugins/database/packaging/localhost_setup_postgres.input
-sudo python /var/lib/irods/scripts/setup_irods.py << IRODS_SETUP_END
-vagrant
-vagrant
-
-
-localhost
-5432
-ICAT
-irods
-y
-$DB_PASSWORD
-
-$ZONE_NAME
-1247
-20000
-20199
-1248
-
-rods
-y
-$ZONE_KEY
-$NEG_KEY
-$CP_KEY
-rods
-
-
-IRODS_SETUP_END
+write_irods_config
+set
+sudo "$PYTHON_INTERPRETER" /var/lib/irods/scripts/setup_irods.py --json_configuration_file=/etc/irods/setup_irods.json
 
 # Restart is needed for iRODS 4.2.9+
 sudo /etc/init.d/irods restart

diff --git a/irods-icommands/.env b/irods-icommands/.env
@@ -2,8 +2,9 @@
 
 # Tested/supported box names: 
 # - centos/7
-# - ubuntu/bionic64
-BOXNAME=ubuntu/bionic64
+# - ubuntu/bionic64 (for iRODS 4.2.x)
+# - ubuntu/focal64  (for iRODS 4.3.x)
+BOXNAME=ubuntu/focal64
 
 # Amount of memory for VM (MB)
 MEMORY=1024
@@ -23,6 +24,7 @@ APT_IRODS_REPO_SIGNING_KEY_LOC=https://packages.irods.org/irods-signing-key.asc
 APT_IRODS_REPO_URL=https://packages.irods.org/apt/
 APT_IRODS_REPO_ARCHITECTURE=amd64
 APT_IRODS_REPO_COMPONENT=main
+
 # Packages to be installed (separated by whitespace).
 # Dependencies do not have to be listed. They are resolved by the script
 APT_PACKAGES="irods-runtime irods-icommands"

diff --git a/irods-icommands/install-icommands.sh b/irods-icommands/install-icommands.sh
@@ -45,22 +45,26 @@ then
 
   echo "Installing irods-commands on Ubuntu."
 
-  echo "Installing dependencies ..."
-  sudo apt-get -y install aptitude
-
   echo "Downloading and installing iRODS repository signing key ..."
   wget -qO - "$APT_IRODS_REPO_SIGNING_KEY_LOC" | sudo apt-key add -
 
   echo "Adding iRODS repository ..."
   if [ "$IRODS_VERSION" == "4.2.12" ]
   then APT_IRODS_REPO_DISTRIBUTION="bionic"
-  else APT_IRODS_REPO_DISTRIBUTION="xenial"
+  elif [[ "$IRODS_VERSION" =~ ^4\.2\. ]]
+  then APT_IRODS_REPO_DISTRIBUTION="xenial"
+  else APT_IRODS_REPO_DISTRIBUTION="focal"
   fi
+
 cat << ENDAPTREPO | sudo tee /etc/apt/sources.list.d/irods.list
 deb [arch=${APT_IRODS_REPO_ARCHITECTURE}] $APT_IRODS_REPO_URL $APT_IRODS_REPO_DISTRIBUTION $APT_IRODS_REPO_COMPONENT
 ENDAPTREPO
+
   sudo apt-get update
 
+  echo "Installing dependencies ..."
+  sudo apt-get -y install aptitude
+
   for package in $APT_PACKAGES
   do echo "Installing package $package and its dependencies"
      get_package_version "$package" "$IRODS_VERSION" "ubuntu"