diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e4320b0..c5d6281d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: "3.10" - uses: pre-commit/action@v3.0.0 # docs: diff --git a/.gitignore b/.gitignore index e45a1725..29d75b33 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ submodules/roles/marvel-nccr* submodules/roles/gantsign* submodules/roles/geerlingguy* submodules/ansible_collections/ + +local/modules/__pycache__/*.pyc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4375cdf..ee94cb5f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,24 +2,31 @@ # See usage instructions at https://pre-commit.com repos: -- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.3.0 - hooks: - - id: pretty-format-yaml - args: [--autofix, --indent, "2", --preserve-quotes] - - repo: https://github.com/adrienverge/yamllint - rev: v1.26.3 + rev: v1.33.0 hooks: - id: yamllint - repo: https://github.com/ansible/ansible-lint - rev: v6.3.0 + rev: v6.22.1 hooks: - id: ansible-lint - args: [--exclude, roles/] + # these should bring back after fixing the ansible-lint issues + # we exclude most of the roles and tasks, as they are not update to the new ansible version + args: [ + --exclude, + roles/, + submodules/roles/, + playbook-aiidalab-qe.yml, + playbook-build.yml, + playbook-package.yml, + .pre-commit-config.yaml, + local/tasks/, + .github/workflows/ci.yml, + submodules/ansible_collections/, + ] - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.11.0 hooks: - id: black diff --git a/Vagrantfile b/Vagrantfile index c417bcfa..f082a1db 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -79,7 +79,7 @@ Vagrant.configure(2) do |config| # Disable the default shared folder of vagrant config.vm.synced_folder ".", "/vagrant", disabled: true - # provisioner: set up VM via ansible. To (re-)run this step: + # provisioner: set up VM with ansible. To (re-)run this step: # vagrant provision --provision-with ansible # Note we use a static inventory, see: https://www.vagrantup.com/docs/provisioning/ansible_intro#static-inventory config.vm.network :private_network, ip: gconfig["ansible_host"] diff --git a/docs/conf.py b/docs/conf.py index 15e6bdfc..2fadd2de 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,7 @@ project = "Quantum Mobile" copyright = "2020, NCCR MARVEL" -author = "Chris Sewell, Giovanni Pizzi, Leopold Talirz" +author = "Jusong Yu, Chris Sewell, Giovanni Pizzi, Leopold Talirz" version = inventory["all"]["vars"]["vm_version"] release = version diff --git a/docs/developers/build-apple-silicon.md b/docs/developers/build-apple-silicon.md new file mode 100644 index 00000000..315cfc70 --- /dev/null +++ b/docs/developers/build-apple-silicon.md @@ -0,0 +1,96 @@ +# Build for Apple Silicon + +Apple Silicon is the new CPU architecture for Apple Macs. +The new architecture is based on ARM64, which means that software compiled for x86_64 will not run on Apple Silicon without translation. +VirtualBox does not yet support Apple Silicon, so instead we use a different virtualization software called [UTM](https://mac.getutm.app/). + +## Prerequisites + +To build the UTM image, first: + +- install the latest version of UTM [from the Apple store](https://apps.apple.com/us/app/utm-virtual-machines/id1538878817). +- download the "64-bit ARM (ARMv8/AArch64) server install image" from https://cdimage.ubuntu.com/releases/focal/release/. + + +Then, clone the `quantum-mobile` repository: + +```console +git clone https://github.com/marvel-nccr/quantum-mobile.git +cd quantum-mobile +``` + +Create a fresh Python environment with a manager of your choosing and simply install `tox`: + +``` +pip install tox +``` + +Finally, we need the `sshpass` tool to have password access to the VM when running the Ansible playbook: + +```console +brew install hudochenkov/sshpass/sshpass +``` + +## Install and start the VM + +Create a new virtual machine from the downloaded Ubuntu server ISO using "File -> New". +Select the following: + +- Virtualize +- Operating system: Select Custom -> Other, then use "Browse" to find the ISO image and continue. +- select the following hardware/storage settings: 4096MB RAM (default), 4 CPU cores and 64GB disk space (default). + +Next, **create** (be careful not delete the existing one, it is used for connecting the internet) a new network setting with type "Emulated VLAN" and forward port 22 to 2200 of localhost so you can ssh to VM from localhost. +Check [here](https://github.com/utmapp/UTM/discussions/2465#discussioncomment-6931047) for detailed instructions. + + + +Finally. run the VM and follow the installation instructions. +In most steps you can take the default option, keeping in mind the following exceptions: + +- Remember to install OpenSSH server during setup, so we can ssh to the VM during the Ansible deployment. +- Create system user `max` with password `moritz` and enable auto login. + +**Important**: Once the installation is complete, select reboot and then _shut down the VM_. +Rebooting with the ISO image still selected for the CD/DVD input will result in the installation process starting again. +Hence, unselect the image before booting the machine after the installation. + +## Configure the VM with Ansible + +Set the following SSH configuration in the `~/.ssh/config`: + +``` +Host qmobile + HostName 127.0.0.1 + User max + Port 2200 +``` + +In the localhost (control machine), activate the Python environment and run the Ansible playbook in the root directory of the repository: + +``` +BUILD_PLAYBOOK=playbook-aiidalab-qe.yml tox -e ansible -- --extra-vars "build_hosts=utm" -kK +``` + +It will ask for the password of the `max` user, which should be `moritz`. + +## Troubleshooting + +### Import from UTM copy + +- If you see "Failed to access data from shortcut", try the methods from https://github.com/utmapp/UTM/discussions/3774 + +### Failing to connect via SSH using password + +In case you run into the following error: + +``` +fatal: [utm]: FAILED! => + msg: to use the 'ssh' connection type with passwords, you must install the sshpass program +``` + +It means that the `sshpass` tool has not been installed correctly using: + +``` +brew install hudochenkov/sshpass/sshpass +``` \ No newline at end of file diff --git a/docs/developers/build-vagrant.md b/docs/developers/build-vagrant.md index 370b214d..0d924e61 100644 --- a/docs/developers/build-vagrant.md +++ b/docs/developers/build-vagrant.md @@ -98,6 +98,13 @@ Building the Desktop Edition is tested on GitHub Actions on every commit to the For the tested steps see the `.github/workflows/build.yml` file. ::: +### Build QeApp dedicated VM + +To build a VM dedicated to the QeApp, run: + +```bash +BUILD_PLAYBOOK=playbook-aiidalab-qe.yml tox -e vagrant +``` ### Continuing a failed build diff --git a/docs/developers/images/utm_ports_mapping.png b/docs/developers/images/utm_ports_mapping.png new file mode 100644 index 00000000..700422dd Binary files /dev/null and b/docs/developers/images/utm_ports_mapping.png differ diff --git a/docs/index.md b/docs/index.md index 9a3d3be1..fca1347f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -170,6 +170,7 @@ users/troubleshoot.md developers/customize.md developers/build-vagrant.md developers/build-cloud.md +developers/build-apple-silicon.md developers/roles.md ``` diff --git a/inventory.yml b/inventory.yml index 50283041..8dcd711e 100644 --- a/inventory.yml +++ b/inventory.yml @@ -11,7 +11,7 @@ all: vars: # VM Metadata vm_name: "Quantum Mobile" - vm_version: "23.04.03" + vm_version: "23.10.0" vm_description: "A Virtual Machine for Computational Materials Science" vm_url: "https://github.com/marvel-nccr/marvel-virtualmachine" vm_author: "MARVEL NCCR and MaX CoE" @@ -22,11 +22,12 @@ all: vm_hostname: "qmobile" vm_user: "max" vm_password: "moritz" - vm_memory: 1536 - vm_cpus: 2 - vm_vram: 128 + vm_memory: 4096 + vm_cpus: 4 + vm_vram: 256 vm_headless: false - vm_browser: chromium-browser # 'chromium-browser' or 'firefox' + vm_browser: firefox # 'chromium-browser' or 'firefox' + vm_wm_package: ubuntu-desktop-minimal # 'lxde' or 'ubuntu-desktop-minimal' vm_data_folder: "/usr/local/share" # this is used to put shared data files in # vm_shared_folder: "/shared" # this is currently inactive in Vagrant config vm_timezone: "Europe/Zurich" @@ -39,6 +40,9 @@ all: build_dir: "/tmp" run_tests: true + # AiiDAlab options + vm_qeapp_image: "ghcr.io/aiidalab/qe:v23.10.0" + # Readme and Release Notes (populated by roles during build) qm_homepage: "https://quantum-mobile.readthedocs.io" readme_vm_path: "/home/{{ vm_user }}/README.md" @@ -108,3 +112,9 @@ all: ansible_user: max vm_headless: true # add_user_public_key: "{{ lookup('file', './keys/user-key.pub') }}" + utm: + cloud_platform: utm + ansible_host: 127.0.0.1 + ansible_port: 2200 + ansible_user: max + vm_headless: false diff --git a/local/tasks/aiida-jupyter.yml b/local/tasks/aiida-jupyter.yml index 8b5dcbd9..5b38991e 100644 --- a/local/tasks/aiida-jupyter.yml +++ b/local/tasks/aiida-jupyter.yml @@ -54,7 +54,7 @@ - name: create desktop shortcut to Jupyter Lab copy: dest: "${HOME}/Desktop/jupyterlab.desktop" - mode: "0753" + mode: "0644" content: | [Desktop Entry] Encoding=UTF-8 diff --git a/local/tasks/aiidalab.yml b/local/tasks/aiidalab.yml new file mode 100644 index 00000000..0d3a0e78 --- /dev/null +++ b/local/tasks/aiidalab.yml @@ -0,0 +1,47 @@ +- name: Create folder for AiiDAlab QE + become: true + become_user: "{{ vm_user }}" + file: + path: /home/{{ vm_user }}/qeapp-home + state: directory + mode: 0775 + +- name: check if qeapp container is running + become: true + become_user: "{{ vm_user }}" + shell: "docker inspect -f '{{ '{{' }} .State.Running {{ '}}' }}' qeapp" + register: qe_container_stat + ignore_errors: true + +- name: Pull qeapp image + command: "docker pull {{ qeapp_image }}" + +- name: echo qe_container + debug: + var: qe_container + +- name: Create and run qeapp container + become: true + become_user: "{{ vm_user }}" + command: "docker run -d --name qeapp -p 8899:8888 --volume /home/{{ vm_user }}/qeapp-home:/home/jovyan -e JUPYTER_TOKEN='max' -e NB_UMASK=002 --restart always {{ qeapp_image }}" + when: qe_container_stat.stdout == "" + +- name: Copy AiiDAlab logo + become: true + become_user: "{{ root_user }}" + copy: + src: images/aiidalab-qe-logo.png + dest: /usr/share/icons/ + +- name: create desktop shortcut to AiiDALab + copy: + dest: "${HOME}/Desktop/aiidalab.desktop" + mode: "0644" + content: | + [Desktop Entry] + Name=AiiDAlab QE + Comment=Launch AiiDAlab QE + Exec={{ vm_browser }} "http://localhost:8899/apps/apps/quantum-espresso/qe.ipynb?token=max" + Terminal=false + Icon=/usr/share/icons/aiidalab-qe-logo.png + Type=Application diff --git a/local/tasks/customise-bash.yml b/local/tasks/customise-bash.yml index 6b0dd550..7bb3fdcb 100644 --- a/local/tasks/customise-bash.yml +++ b/local/tasks/customise-bash.yml @@ -33,6 +33,14 @@ - regexp: HISTFILESIZE= line: HISTFILESIZE=100000 +- name: Set umask to 002 + blockinfile: + path: "${HOME}/.bashrc" + marker: "# {mark} ANSIBLE MANAGED BLOCK (umask)" + block: | + # Set umask to 002 + umask 002 + - name: hide possible sudo message file: path: ~/.sudo_as_admin_successful diff --git a/local/tasks/customise-gui.yml b/local/tasks/customise-gui.yml index de451f5c..4d9a4ddd 100644 --- a/local/tasks/customise-gui.yml +++ b/local/tasks/customise-gui.yml @@ -11,7 +11,7 @@ become: true become_user: "{{ root_user }}" copy: - src: images/quantum-mobile-bg.png + src: "images/{{ bg_image | default('quantum-mobile-bg.png')}}" dest: "{{ vm_data_folder }}/qm-customizations/quantum-mobile-bg.png" mode: 0755 @@ -55,7 +55,7 @@ - name: create desktop shortcut to QM homepage copy: dest: "${HOME}/Desktop/homepage.desktop" - mode: "0753" + mode: "0644" content: | [Desktop Entry] Encoding=UTF-8 diff --git a/local/tasks/docker.yml b/local/tasks/docker.yml new file mode 100644 index 00000000..cd9f9973 --- /dev/null +++ b/local/tasks/docker.yml @@ -0,0 +1,13 @@ +- name: Install docker + include_role: + name: geerlingguy.docker + vars: + docker_users: + - "{{ vm_user }}" + +- name: Install docker pip + include_role: + name: geerlingguy.pip + vars: + pip_install_packages: + - docker diff --git a/local/tasks/files/README-qeapp.md.j2 b/local/tasks/files/README-qeapp.md.j2 new file mode 100644 index 00000000..bf840bbe --- /dev/null +++ b/local/tasks/files/README-qeapp.md.j2 @@ -0,0 +1,23 @@ +# Welcome to {{ vm_name }} version {{ vm_version }} + +*Quantum Mobile* is a Virtual Machine for computational materials science, distributed by the {{ vm_author }}. + +It comes with a collection of software packages for quantum mechanical calculations, including: + +{% for name, url in software_urls.items() %} +* [{{ name }}]({{ url }}) +{% endfor %} + +all of which are set up and ready to be used on their own or through [AiiDA](http://www.aiida.net). + +# Getting started + +* Start the Quantum ESPRESSO app by double-clicking on the icon on the desktop. + +* If the virtual machine is just started, you may need to wait a few minutes for the server to be fully ready. If you see an error message, try again after a few minutes. + +* The data you create over the app can be accessed from the `~/qeapp-home` folder. Vise versa, you can copy files into this folder to make them available in the app. + +* For more information on how to use the app, see the [Quantum ESPRESSO app documentation](https://aiidalab-quantumespresso.readthedocs.io/en/latest/). + +For troubleshooting and other information, see: {{ qm_homepage }}. diff --git a/local/tasks/images/aiidalab-qe-logo.png b/local/tasks/images/aiidalab-qe-logo.png new file mode 100644 index 00000000..0c09f79a Binary files /dev/null and b/local/tasks/images/aiidalab-qe-logo.png differ diff --git a/local/tasks/images/quantum-mobile-logo.png b/local/tasks/images/quantum-mobile-logo.png index 884654ea..a32dd0c2 100644 Binary files a/local/tasks/images/quantum-mobile-logo.png and b/local/tasks/images/quantum-mobile-logo.png differ diff --git a/local/tasks/images/quantum-mobile-qeapp-bg.svg b/local/tasks/images/quantum-mobile-qeapp-bg.svg new file mode 100644 index 00000000..e8fec95f --- /dev/null +++ b/local/tasks/images/quantum-mobile-qeapp-bg.svg @@ -0,0 +1,905 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + QUANTUMMOBILE + + + + + + + + Open AiiDAlab by double clicking on the desktop icon. + +However, please wait ~2 minutes after starting this virtual machine to let the AiiDAlab server be fully ready. + +If you double click too early and see an error message, just refresh the page after a few seconds. + + + + diff --git a/playbook-aiidalab-qe.yml b/playbook-aiidalab-qe.yml new file mode 100644 index 00000000..82092058 --- /dev/null +++ b/playbook-aiidalab-qe.yml @@ -0,0 +1,128 @@ +- name: Prepare for VM build + + hosts: "{{ build_hosts | default('vagrant-ssh') }}" + gather_facts: false + + pre_tasks: + + - name: test configuration + tags: [init] + debug: + msg: RUNNING PLAYBOOK FOR '{{ vm_name }}' VERSION '{{ vm_version }}' + + - name: testing ansible environment + tags: [init] + debug: + msg: Connecting to host '{{ inventory_hostname }}' as user '{{ ansible_user }}' + + - name: Install linux-headers-generic for guest-additions + tags: [init] + when: "inventory_hostname.startswith('vagrant')" + become: true + apt: + cache_valid_time: 86400 # One day + name: + - linux-headers-generic + + - name: Make local dist folder + tags: [init] + local_action: + module: file + state: directory + path: "{{ local_dist_folder }}/" + when: release_notes_locally is defined and release_notes_locally + +- name: Build VM + hosts: "{{ build_hosts | default('vagrant-ssh') }}" + + tasks: + + # Generic Tasks + # These tasks are run for any VM build, regardless of the software installed + # They setup the VM user, commandline interface, and (optionally) desktop GUI + + - name: Update system package managers (apt, pip) + tags: [init] + include_tasks: local/tasks/ensure-apt-pip.yml + + - name: "Add QM user '{{ vm_user }}'" + tags: [add_user, ci_test] + import_tasks: local/tasks/add-qm-user.yml + + - name: "Add release notes section for the operating system" + tags: [release_notes] + import_tasks: local/tasks/release-notes-system.yml + + - name: Customise bash terminal for QM + tags: [customise-bash] + become: true + become_user: "{{ vm_user }}" + import_tasks: local/tasks/customise-bash.yml + + - name: Add desktop GUI + tags: [ubuntu_desktop] + when: not vm_headless + import_role: + name: marvel-nccr.ubuntu_desktop + vars: + ubuntu_desktop_vm_user: "{{ vm_user }}" + ubuntu_desktop_browser: "{{ vm_browser }}" + ubuntu_desktop_wm_package: "{{ vm_wm_package }}" + + - name: Customise GUI for QM + tags: [customise-gui] + when: not vm_headless + become: true + become_user: "{{ vm_user }}" + import_tasks: local/tasks/customise-gui.yml + vars: + bg_image: "quantum-mobile-qeapp-bg.svg" + + - name: Setup QM to run simulations + tags: [simsetup] + import_tasks: local/tasks/simulation-setup.yml + + - name: "Add user README" + tags: [add_readme] + become: true + become_user: "{{ vm_user }}" + template: + src: local/tasks/files/README-qeapp.md.j2 + dest: "{{ readme_vm_path }}" + vars: + software_urls: + Quantum ESPRESSO: http://www.quantum-espresso.org/ + Quantum ESPRESSO App: https://aiidalab-qe.readthedocs.io/ + + + # Build Specific Tasks + # These tasks can be customised for specific VM builds, + # depending on the software one requires to be installed + - name: Install docker and docker-compose + tags: [docker] + import_tasks: local/tasks/docker.yml + become: true + vars: + vm_docker_compose_version: "1.26.0" + + - name: Create AiiDAlab service + tags: [aiidalab] + become: true + become_user: "{{ vm_user }}" + import_tasks: local/tasks/aiidalab.yml + vars: + qeapp_image: "{{ vm_qeapp_image }}" + + post_tasks: + + # These break idempotency, + # so only run once all other tasks have completed + + - name: "Clean caches" + tags: [never, cleanup] + import_tasks: local/tasks/clean-caches.yml + + - name: "Remove {{ build_dir }} content" + tags: [never, cleanup] + become: true + command: rm -rf {{ build_dir }}/* diff --git a/playbook-build.yml b/playbook-build.yml index 7b113a94..e6e7de79 100644 --- a/playbook-build.yml +++ b/playbook-build.yml @@ -227,6 +227,12 @@ become_user: "{{ vm_user }}" import_tasks: local/tasks/aiida-jupyter.yml + - name: Create AiiDAlab service + tags: [aiidalab] + become: true + become_user: "{{ vm_user }}" + import_tasks: local/tasks/aiidalab.yml + - name: "Create AiiDA computers" tags: [aiida-computers] become: true diff --git a/requirements.yml b/requirements.yml index e8c9f83f..412794e4 100644 --- a/requirements.yml +++ b/requirements.yml @@ -9,8 +9,12 @@ roles: - name: marvel-nccr.add_user version: v0.2.1 - name: marvel-nccr.ubuntu_desktop - version: v1.1.0 + version: v1.1.1 - name: marvel-nccr.editors version: v1.0.2 - name: marvel-nccr.slurm version: v2.0.2 +- name: geerlingguy.docker + version: 6.2.0 +- name: geerlingguy.pip + version: 2.2.0 diff --git a/resources/desktop-background/quantum-mobile-qeapp-bg.svg b/resources/desktop-background/quantum-mobile-qeapp-bg.svg new file mode 100644 index 00000000..e8fec95f --- /dev/null +++ b/resources/desktop-background/quantum-mobile-qeapp-bg.svg @@ -0,0 +1,905 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + QUANTUMMOBILE + + + + + + + + Open AiiDAlab by double clicking on the desktop icon. + +However, please wait ~2 minutes after starting this virtual machine to let the AiiDAlab server be fully ready. + +If you double click too early and see an error message, just refresh the page after a few seconds. + + + + diff --git a/resources/logos/quantum-mobile-v4-square.svg b/resources/logos/quantum-mobile-v4-square.svg new file mode 100644 index 00000000..0e19290b --- /dev/null +++ b/resources/logos/quantum-mobile-v4-square.svg @@ -0,0 +1,843 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tox.ini b/tox.ini index 0a2213fa..449276fb 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = vagrant [testenv] -basepython = python3 +basepython = python3.9 skip_install = true [testenv:vagrant] @@ -16,7 +16,7 @@ passenv = VAGRANT_NO_GUI VAGRANT_ON_GH BUILD_PLAYBOOK -whitelist_externals = vagrant +allowlist_externals = vagrant commands_pre = ansible-galaxy install -r requirements.yml commands = vagrant {posargs:up} @@ -27,7 +27,7 @@ deps = -rrequirements.txt passenv = HOME BUILD_PLAYBOOK -whitelist_externals = vagrant +allowlist_externals = vagrant commands_pre = ansible-galaxy install -r requirements.yml -vvv ; vagrant ssh-config > vagrant-ssh @@ -48,7 +48,7 @@ commands = ansible-galaxy {posargs:install -r requirements.yml --force} description = Update aspects of vagrant passenv = HOME -whitelist_externals = vagrant +allowlist_externals = vagrant commands = vagrant box update vagrant box prune @@ -72,7 +72,7 @@ description = clean: Build the documentation (remove any existing build) update: Build the documentation (modify any existing build) deps = -rrequirements-docs.txt -whitelist_externals = rm +allowlist_externals = rm commands = clean: rm -rf docs/_build sphinx-build -nW --keep-going -b {posargs:html} docs/ docs/_build/{posargs:html}