From 930d86115d5f3e51ba81e5e9031a555d47f5ae80 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Thu, 20 Aug 2020 11:22:33 +0100 Subject: [PATCH 001/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index bde0861e8a..693eb49a52 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,6 +1,21 @@ UNRELEASED: * -2020-05-27 +2020-08-20: + * raspi-config - added selection of boot device order + * raspi-config - added selection of boot EEPROM version + * SD Card Copier - copy is now immediately aborted if drives are connected or disconnected while copying + * Version 32.0.0.414 of Flash player included + * User feedback survey removed from first run of Chromium + * Recommended Software - now allows multiple install and reinstall operations without having to close between each one + * Bug fix - misleading file browser from panel menu icon selection dialog - icons must now be in icon theme rather than arbitrary files + * Bug fix - items in main menu not being translated + * Bug fix - raspi-config not detecting audio devices in non-English locales + * Bug fix - Bookshelf claiming no disk space in non-English locales + * Bug fix - failed installation of both 32 and 64 bit versions of packages by Recommended Software on 64-bit images + * Italian translations added (thanks to Emanuele Goldoni and the Italian translation team) + * Raspberry Pi firmware ef72c17bcaaeb89093d87bcf71f3228e1b5e1fff + * Linux kernel 5.4.51 +2020-05-27: * Added Bookshelf application * Added Raspberry Pi Diagnostics application * Added magnifier plugin to taskbar - needs magnifier application installed from Recommended Software to enable From 78c9e2cf02c312069701dc21f0496a2091b351f4 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Mon, 24 Aug 2020 15:45:57 +0100 Subject: [PATCH 002/222] Raspbian -> Raspberry Pi OS --- stage2/EXPORT_NOOBS | 2 +- stage4/EXPORT_NOOBS | 2 +- stage5/EXPORT_NOOBS | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stage2/EXPORT_NOOBS b/stage2/EXPORT_NOOBS index 136cc77778..0d78ecfd03 100644 --- a/stage2/EXPORT_NOOBS +++ b/stage2/EXPORT_NOOBS @@ -1,2 +1,2 @@ -NOOBS_NAME="Raspbian Lite" +NOOBS_NAME="Raspberry Pi OS Lite (32-bit)" NOOBS_DESCRIPTION="A port of Debian with no desktop environment" diff --git a/stage4/EXPORT_NOOBS b/stage4/EXPORT_NOOBS index 2ef6d7b486..f197ee1db7 100644 --- a/stage4/EXPORT_NOOBS +++ b/stage4/EXPORT_NOOBS @@ -1,2 +1,2 @@ -NOOBS_NAME="Raspbian" +NOOBS_NAME="Raspberry Pi OS (32-bit)" NOOBS_DESCRIPTION="A port of Debian with the Raspberry Pi Desktop" diff --git a/stage5/EXPORT_NOOBS b/stage5/EXPORT_NOOBS index cb1020328e..ac7051f0c5 100644 --- a/stage5/EXPORT_NOOBS +++ b/stage5/EXPORT_NOOBS @@ -1,2 +1,2 @@ -NOOBS_NAME="Raspbian Full" +NOOBS_NAME="Raspberry Pi OS Full (32-bit)" NOOBS_DESCRIPTION="A port of Debian with desktop and recommended applications" From d6207a620a10ba0ec25f95a637d3fdd272b47e84 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Mon, 24 Aug 2020 17:01:54 +0100 Subject: [PATCH 003/222] Update OS.png --- export-noobs/00-release/files/OS.png | Bin 1399 -> 1752 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/export-noobs/00-release/files/OS.png b/export-noobs/00-release/files/OS.png index 02b67cf7e9fa5352b828649bb11f5a135a0e91f0..3091bf95064eeb6364b8d1e22faae101eaf1ecf5 100644 GIT binary patch delta 1747 zcmV;^1}ypa3fK*h8Gi-<004~sxNQIc010qNS#tmY3labT3lag+-G2N400xRlL_t(o zg{@g%OjB1J{#~SEp|;g(T9pOZz$p(FLX3N;vh870Uo_FVM{&vCMA_rqgYE^lrvdu7 z==Q2E!6ln)$u|LS*+SlK`1R1i=5>oOg z-#PbcFviFUhGDc2(Lo}5Q{2{0MAJlcU0nT)v2i;ShGEnzbAzUz<=nWVe>;MLR1Xo= zZ)pkDVQkypTH4$aqQ_%ds`6FROuE27-;Yn=?+qgAlFsD(Dxaqvb1)ZJ8}U+=&qK+f zG~JI+(z9HV9)I7>-kX~#{;ke0hv1opSmbo{a6N5~h8!b2lPS=H_yqm*SprdQ5ySD= z@#X-?ZFtS^IR8Onf)ZE8LHbk1*ctQh6`Fb1XVSB-^RimN)CVz_!23%Ku3_~az%QUpSO(LjR8B4HX(fnh=1t7&W^3F#N6yj=marMhQCQt)}dTR z2JWHZuEJ;81OQHw8NXdq1t$zy95nnwut}fNu*t0BsKbxY#LT1s1 zY2H5&bzNh;5J3^a{z#>_?gM`l?O79{nu=0gotwHtjjPwvzUo@rxz}b>bS;0M`Zn(3 z-_y?vbZc&cE@Ve3JzoIHgD(|98M!8pzwX?|Hx`Kj@wrLro4QRG!ZF7Mc0U=W)~fZ( z7Jrdnn5WK1ztLzh>l#V7CP8=Aod`Bt$bCHV2R(dV;LhC8Jvq#yDFIvE8`|)#PWaHm zjo`s1kDpqq)-Mjg5mujDDDn%Rhm=~AKsr$Z*YYJ|W|EOrUfS&qlQ@oPVJ}ikXpdAb z=^q?k$|ZSI?5o*An^slR!{@VfDL2j|ihprnw3y{-J>qMi&fxZCwRdZ7l1>In8NuQQ z3z>E&0@@lB_4fp;++STwz0QHwc42t7VR2@IHeuuVW;rwv=7oPUZo8ICnbh{4`L>=cAiV2*$?MQ0RgT4w+^nYSF zMgyDQAzy{3lvWKn5`_i)q(=4d`J&l^NNk-+J`np&=lBn0`p ze2Th)&2(zrj%7x$80;SCYFWRyU$uvWL}$>?YgUZ+o5@&oP`nLDf(TA8$A2mMUyzBw<=i+=k0gsl zm?LKfQi@2*`{gz#7qIuik8;LGj4{iziLOa_sF4~qI`8@F>Dam=ltS3V#qjYqQTf0VEJPhXo{bFQA-crXO-(2M8@9;wC*L^b<9w zMoxv08{6o)62w1tXFXugnlLYHJ1LPsjX@Lfm|zFY74`h$1jVz$4XIgX$Ooj(N53yS zh)O_aOBUFHa@ml8yx&SXP?lz+NY4A$rV289|j~q|zI}30aCR4w_PQOFPK&j>h2@donZ-&omHs98{I3I#+UK5u~L9EVS{o zIu5dU_;EV_UCfS>O_}tHhj!QJ;<644FeKi7wn7r*I3njn4lqd-=C(4%y2>F}HbI4G zoH2%!AA>Wb*{(q(piBTFdLVWTbKO#EMp-G=PBsi|O(*564>SrQW_tC}i-@h*2?)N- pCH{*6I)*Rmg)ub2%Iz~o{{i3Q1??G5KNkQ1002ovPDHLkV1fkNI|Kj# delta 1391 zcmV-#1(5pK4fhI=8Gix*000A=FFF7K010qNS#tmY4s-wj4s-!)B(v54000McNliru z;{^^5Aqf28ixdC=1r|v}K~z}7?N}{%BRLFxz3+gA*(`>JQVW7TW{UpY-pp$$j4~IkB;2J%3AnlI+?0`ST}!h5UwJ zB)<&!_rIU62E)etAQG7YD5WW*OsU)DVpHv8Sb67?NW)A-L>T~W)a`Qlk>$OQ&iG_l zEf;r!K$gfr+YEq2Bu>!_nEJ!P`0}zk9JsZM5>!rlKcfx+NQ1CU0K~)QcD7h-;qlt7 zU{Zc;w0$*05P$GOZYI}6)G$*ob<5Xk_J~qMQ;VCr?Lr+nP>u(40tld@^Wdj1u!9%87(iI1&5^JYcfrQw*SEJ@=kb%ARwwm*1`;#W5QKG# zD*=!b8Gix`oCp9iFbA3pxK7bYp69dWV)x1gz`M1Gs-pm`MUI_svwGxs?75Dz z7Fmi+nt1Gdhu}p{oM!=mBco#sqtnw}&bVPFKm*ezPB?TGom;(66o6?H&-_Dz9x0mU zHUkoH)5K%vN7j^z0N}mY`~z@3uOT8pf8`_4S9E4gJEeisj*2PGqvS-R$MFoPI?528 zd4IM90IS9F`{?Z3_JiJ?@~W@>xPZ-pTMTuS(0 zPXG*|uc%@EEN}=he{@m+e$V<12LN>xet$!tfagK0v%t+~fhnUWKCVUPQNl}wS&Vgj z&=`5YIh%kfho@YR0VHhFvglH}7SUU1k{=ww-wQYXuijA8ZI91j#}AONc0K)7>Wl?f90=$svRT#! zk_C4mctKxLt}_=>9fg!$^8M~tpXUeM>MlN}(=@l{&DnkRd=~|PGP>v Date: Tue, 8 Sep 2020 16:16:07 -0400 Subject: [PATCH 004/222] Enable adding SSH pubkey and setting pubkey only (#380) This commit add the ability to specify an SSH public key as well as the option to disable password authentication and only allow public key authentication for SSH. Signed-off-by: Daniel F. Dickinson --- README.md | 12 ++++++++++++ build.sh | 8 ++++++++ stage2/01-sys-tweaks/01-run.sh | 12 ++++++++++++ 3 files changed, 32 insertions(+) diff --git a/README.md b/README.md index 29d81667d4..e7f224ac8e 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,18 @@ The following environment variables are supported: Setting to `1` will enable ssh server for remote log in. Note that if you are using a common password such as the defaults there is a high risk of attackers taking over you Raspberry Pi. + * `PUBKEY_SSH_FIRST_USER` (Default: unset) + + Setting this to a value will make that value the contents of the FIRST_USER_NAME's ~/.ssh/authorized_keys. Obviously the value should + therefore be a valid authorized_keys file. Note that this does not + automatically enable SSH. + + * `PUBKEY_ONLY_SSH` (Default: `0`) + + * Setting to `1` will disable password authentication for SSH and enable + public key authentication. Note that if SSH is not enabled this will take + effect when SSH becomes enabled. + * `STAGE_LIST` (Default: `stage*`) If set, then instead of working through the numeric stages in order, this list will be followed. For example setting to `"stage0 stage1 mystage stage2"` will run the contents of `mystage` before stage2. Note that quotes are needed around the list. An absolute or relative path can be given for stages outside the pi-gen directory. diff --git a/build.sh b/build.sh index a8247ab040..6f720b1f50 100755 --- a/build.sh +++ b/build.sh @@ -171,6 +171,7 @@ export WPA_ESSID export WPA_PASSWORD export WPA_COUNTRY export ENABLE_SSH="${ENABLE_SSH:-0}" +export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}" export LOCALE_DEFAULT="${LOCALE_DEFAULT:-en_GB.UTF-8}" @@ -181,6 +182,8 @@ export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}" export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"} +export PUBKEY_SSH_FIRST_USER + export CLEAN export IMG_NAME export APT_PROXY @@ -226,6 +229,11 @@ if [[ -n "${WPA_PASSWORD}" && ${#WPA_PASSWORD} -lt 8 || ${#WPA_PASSWORD} -gt 63 exit 1 fi +if [[ "${PUBKEY_ONLY_SSH}" = "1" && -z "${PUBKEY_SSH_FIRST_USER}" ]]; then + echo "Must set 'PUBKEY_SSH_FIRST_USER' to a valid SSH public key if using PUBKEY_ONLY_SSH" + exit 1 +fi + mkdir -p "${WORK_DIR}" log "Begin ${BASE_DIR}" diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index 8d28adccc1..c1836f9ca6 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -11,6 +11,18 @@ install -m 644 files/console-setup "${ROOTFS_DIR}/etc/default/" install -m 755 files/rc.local "${ROOTFS_DIR}/etc/" +if [ -n "${PUBKEY_SSH_FIRST_USER}" ]; then + install -v -m 0700 -o 1000 -g 1000 -d "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh + echo "${PUBKEY_SSH_FIRST_USER}" >"${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys + chown 1000:1000 "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys + chmod 0600 "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys +fi + +if [ "${PUBKEY_ONLY_SSH}" = "1" ]; then + sed -i -Ee 's/^#?[[:blank:]]*PubkeyAuthentication[[:blank:]]*no[[:blank:]]*$/PubkeyAuthentication yes/ +s/^#?[[:blank:]]*PasswordAuthentication[[:blank:]]*yes[[:blank:]]*$/PasswordAuthentication no/' "${ROOTFS_DIR}"/etc/ssh/sshd_config +fi + on_chroot << EOF systemctl disable hwclock.sh systemctl disable nfs-common From d6ee1e5469184e0a88c22e6c8f5d209350a34723 Mon Sep 17 00:00:00 2001 From: andrum99 <58046090+andrum99@users.noreply.github.com> Date: Wed, 16 Sep 2020 20:46:56 +0100 Subject: [PATCH 005/222] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e7f224ac8e..8b7f875af5 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ # pi-gen -_Tool used to create the raspberrypi.org Raspbian images_ +Tool used to create Raspberry Pi OS images. (Previously known as Raspbian). ## Dependencies -pi-gen runs on Debian based operating systems. Currently it is only supported on +pi-gen runs on Debian-based operating systems. Currently it is only supported on either Debian Buster or Ubuntu Xenial and is known to have issues building on earlier releases of these systems. On other Linux distributions it may be possible to use the Docker build described below. -To install the required dependencies for pi-gen you should run: +To install the required dependencies for `pi-gen` you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ @@ -68,7 +68,7 @@ The following environment variables are supported: system for each build stage, amounting to tens of gigabytes in the case of Raspbian. - **CAUTION**: If your working directory is on an NTFS partition you probably won't be able to build. Make sure this is a proper Linux filesystem. + **CAUTION**: If your working directory is on an NTFS partition you probably won't be able to build: make sure this is a proper Linux filesystem. * `DEPLOY_DIR` (Default: `"$BASE_DIR/deploy"`) @@ -124,7 +124,7 @@ The following environment variables are supported: * `WPA_ESSID`, `WPA_PASSWORD` and `WPA_COUNTRY` (Default: unset) - If these are set, they are use to configure `wpa_supplicant.conf`, so that the Raspberry Pi can automatically connect to a wifi network on first boot. If `WPA_ESSID` is set and `WPA_PASSWORD` is unset an unprotected wifi network will be configured. If set, `WPA_PASSWORD` must be between 8 and 63 characters. + If these are set, they are use to configure `wpa_supplicant.conf`, so that the Raspberry Pi can automatically connect to a wireless network on first boot. If `WPA_ESSID` is set and `WPA_PASSWORD` is unset an unprotected wireless network will be configured. If set, `WPA_PASSWORD` must be between 8 and 63 characters. * `ENABLE_SSH` (Default: `0`) @@ -276,7 +276,7 @@ maintenance and allows for more easy customization. - **Stage 2** - lite system. This stage produces the Raspbian-Lite image. It installs some optimized memory functions, sets timezone and charmap - defaults, installs fake-hwclock and ntp, wifi and bluetooth support, + defaults, installs fake-hwclock and ntp, wireless LAN and bluetooth support, dphys-swapfile, and other basics for managing the hardware. It also creates necessary groups and gives the pi user access to sudo and the standard console hardware permission groups. From be8daaa7210d8a8df77792e7bcb3ec509d8e995e Mon Sep 17 00:00:00 2001 From: Raphael Nestler Date: Fri, 18 Sep 2020 15:40:33 +0200 Subject: [PATCH 006/222] Only disable wifi if WPA_COUNTRY is not set (#416) * Only disable wifi if WPA_COUNTRY is not set * Use '-n' instead of '-v' to test if WPA_COUNTRY is set Apparently -v doesn't work in all shells. --- stage2/02-net-tweaks/01-run.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/stage2/02-net-tweaks/01-run.sh b/stage2/02-net-tweaks/01-run.sh index 092975531f..d82381b1b7 100755 --- a/stage2/02-net-tweaks/01-run.sh +++ b/stage2/02-net-tweaks/01-run.sh @@ -25,7 +25,12 @@ network={ EOL fi -# Disable wifi on 5GHz models +# Disable wifi on 5GHz models if WPA_COUNTRY is not set mkdir -p "${ROOTFS_DIR}/var/lib/systemd/rfkill/" -echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan" -echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan" +if [ -n "$WPA_COUNTRY" ]; then + echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan" + echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan" +else + echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan" + echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan" +fi From d1596451b9d81c360c6cf89a10ed1c30e19e4e02 Mon Sep 17 00:00:00 2001 From: kdoren Date: Wed, 23 Sep 2020 11:54:37 -0700 Subject: [PATCH 007/222] workaround build failure on AWS M6g instances (#449) --- export-image/prerun.sh | 28 ++++++++++++++++++++++++++-- export-noobs/prerun.sh | 28 ++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/export-image/prerun.sh b/export-image/prerun.sh index 8bbc566e06..4b5cf8cd5c 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -39,8 +39,32 @@ BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) -BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") -ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") +echo "Mounting BOOT_DEV..." +cnt=0 +until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do + if [ $cnt -lt 5 ]; then + cnt=$((cnt + 1)) + echo "Error in losetup for BOOT_DEV. Retrying..." + sleep 5 + else + echo "ERROR: losetup for BOOT_DEV failed; exiting" + exit 1 + fi +done + +echo "Mounting ROOT_DEV..." +cnt=0 +until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do + if [ $cnt -lt 5 ]; then + cnt=$((cnt + 1)) + echo "Error in losetup for ROOT_DEV. Retrying..." + sleep 5 + else + echo "ERROR: losetup for ROOT_DEV failed; exiting" + exit 1 + fi +done + echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 54e0c59f07..c161ee6213 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -16,8 +16,32 @@ BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) -BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") -ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") +echo "Mounting BOOT_DEV..." +cnt=0 +until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do + if [ $cnt -lt 5 ]; then + cnt=$((cnt + 1)) + echo "Error in losetup for BOOT_DEV. Retrying..." + sleep 5 + else + echo "ERROR: losetup for BOOT_DEV failed; exiting" + exit 1 + fi +done + +echo "Mounting ROOT_DEV..." +cnt=0 +until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do + if [ $cnt -lt 5 ]; then + cnt=$((cnt + 1)) + echo "Error in losetup for ROOT_DEV. Retrying..." + sleep 5 + else + echo "ERROR: losetup for ROOT_DEV failed; exiting" + exit 1 + fi +done + echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" From dd96ca1c8629da6589b381546e2b7677c7611744 Mon Sep 17 00:00:00 2001 From: Ani Date: Tue, 6 Oct 2020 02:58:18 -0700 Subject: [PATCH 008/222] Automagically use i386/debian:buster (#415) * Autmagically use 1386/debian:buster when running on 64-bit host to prevent error #271 --- Dockerfile | 3 ++- build-docker.sh | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2e53149b52..3d5874d17b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM debian:buster +ARG BASE_IMAGE=debian:buster +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND noninteractive diff --git a/build-docker.sh b/build-docker.sh index b6a9ea3d81..350f722bde 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -73,7 +73,17 @@ fi # Modify original build-options to allow config file to be mounted in the docker container BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')" -${DOCKER} build -t pi-gen "${DIR}" +# Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead +case "$(uname -m)" in + x86_64|aarch64) + BASE_IMAGE=i386/debian:buster + ;; + *) + BASE_IMAGE=debian:buster + ;; +esac +${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" + if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ From 6c21cc1e4ddcf177b7bcf584e23cad734ea5d7e3 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Tue, 17 Nov 2020 15:13:18 +0000 Subject: [PATCH 009/222] stage4: add printing support --- stage4/05-print-support/00-packages | 2 ++ stage4/05-print-support/01-run.sh | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 stage4/05-print-support/00-packages create mode 100755 stage4/05-print-support/01-run.sh diff --git a/stage4/05-print-support/00-packages b/stage4/05-print-support/00-packages new file mode 100644 index 0000000000..e9c36bda29 --- /dev/null +++ b/stage4/05-print-support/00-packages @@ -0,0 +1,2 @@ +cups +system-config-printer diff --git a/stage4/05-print-support/01-run.sh b/stage4/05-print-support/01-run.sh new file mode 100755 index 0000000000..dc9e2b2c3c --- /dev/null +++ b/stage4/05-print-support/01-run.sh @@ -0,0 +1,5 @@ +#!/bin/bash -e + +on_chroot < Date: Tue, 17 Nov 2020 15:22:12 +0000 Subject: [PATCH 010/222] export-image: get apt lists from public repo --- export-image/01-set-sources/01-run.sh | 2 ++ export-image/04-finalise/01-run.sh | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/export-image/01-set-sources/01-run.sh b/export-image/01-set-sources/01-run.sh index 86dea2d879..5f512099b3 100755 --- a/export-image/01-set-sources/01-run.sh +++ b/export-image/01-set-sources/01-run.sh @@ -1,5 +1,7 @@ #!/bin/bash -e +rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache" +find "${ROOTFS_DIR}/var/lib/apt/lists/" -type f -delete on_chroot << EOF apt-get update apt-get -y dist-upgrade diff --git a/export-image/04-finalise/01-run.sh b/export-image/04-finalise/01-run.sh index 0864639b23..91264e5d32 100755 --- a/export-image/04-finalise/01-run.sh +++ b/export-image/04-finalise/01-run.sh @@ -16,7 +16,6 @@ if [ -d "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config" ]; then chmod 700 "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config" fi -rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache" rm -f "${ROOTFS_DIR}/usr/bin/qemu-arm-static" if [ "${USE_QEMU}" != "1" ]; then From cce27bd6f44a3b2e83855645986b3e21f771e852 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Fri, 27 Nov 2020 15:50:50 +0000 Subject: [PATCH 011/222] Update release notes --- .../00-release/files/release_notes.txt | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 693eb49a52..be998d9216 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,26 @@ UNRELEASED: - * + * PulseAudio now included and running by default + * Bluealsa Bluetooth interface removed - Bluetooth audio is now handled by PulseAudio + * LXPanel volume control plugin replaced with PulseAudio version + * Version 84.0.4147.105 of Chromium web browser included + * Version 3.3.0 of Thonny included + * Version 32.0.0.453 of Flash player included - note that this will be the final release of Flash, as it is end-of-lifed at the end of 2020 + * CUPS printer system included, along with system-config-printer CUPS GUI and HP printer drivers + * raspi-config menu structure rearranged to match Raspberry Pi Configuration tabs + * Control for GPIO-connected fans added to raspi-config and Raspberry Pi Configuration + * Control for power / activity LED on Pi 400 and Pi Zero added to raspi-config and Raspberry Pi Configuration + * Improved screen reader voice prompts in several applications + * Added ctrl-alt-space shortcut to install Orca screen reader at any point + * Low voltage warnings added to battery monitor plugin + * Magnifier plugin zoom can now be changed with scroll wheel when pointer is over icon + * Change to notification popups - now will only close when clicked on directly, not by clicking anywhere + * Bookshelf now made compatible with translated versions of books and magazines, and will offer translated versions where available, based on system language setting + * Bug fix - crash in CPU temperature plugin when throttling detection fails + * Bug fix - if Orca is running, shutdown commands and shutdown dialog will force kill it to prevent it locking up the reboot or shutdown process + * Various additional language translations added + * Various minor bug fixes and UI tweaks + * Raspberry Pi firmware b324aea801f669b6ab18441f970e74a5a7346684 + * Linux kernel 5.4.79 2020-08-20: * raspi-config - added selection of boot device order * raspi-config - added selection of boot EEPROM version From 94a4e4b41cf82b64c1997a50d5d9ce05a8e34e24 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Wed, 2 Dec 2020 16:53:07 +0000 Subject: [PATCH 012/222] Finalise 2020-12-02 --- export-noobs/00-release/files/release_notes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index be998d9216..d5b6629fc2 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,6 @@ UNRELEASED: + * +2020-12-02: * PulseAudio now included and running by default * Bluealsa Bluetooth interface removed - Bluetooth audio is now handled by PulseAudio * LXPanel volume control plugin replaced with PulseAudio version From 225f69828fa05361d6028edf2d7a69db73fe2b45 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Wed, 2 Dec 2020 16:56:25 +0000 Subject: [PATCH 013/222] os.json: Update kernel version automatically --- export-noobs/00-release/00-run.sh | 1 + export-noobs/00-release/files/os.json | 2 +- export-noobs/prerun.sh | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/00-run.sh b/export-noobs/00-release/00-run.sh index 18a3a5dcd6..c2a4a51789 100755 --- a/export-noobs/00-release/00-run.sh +++ b/export-noobs/00-release/00-run.sh @@ -37,6 +37,7 @@ sed "${NOOBS_DIR}/os.json" -i -e "s|UNRELEASED|${IMG_DATE}|" sed "${NOOBS_DIR}/os.json" -i -e "s|NOOBS_NAME|${NOOBS_NAME}|" sed "${NOOBS_DIR}/os.json" -i -e "s|NOOBS_DESCRIPTION|${NOOBS_DESCRIPTION}|" sed "${NOOBS_DIR}/os.json" -i -e "s|RELEASE|${RELEASE}|" +sed "${NOOBS_DIR}/os.json" -i -e "s|KERNEL|$(cat "${STAGE_WORK_DIR}/kernel_version")|" sed "${NOOBS_DIR}/release_notes.txt" -i -e "s|UNRELEASED|${IMG_DATE}|" diff --git a/export-noobs/00-release/files/os.json b/export-noobs/00-release/files/os.json index e231238be6..6cc0e36d80 100644 --- a/export-noobs/00-release/files/os.json +++ b/export-noobs/00-release/files/os.json @@ -1,7 +1,7 @@ { "description": "NOOBS_DESCRIPTION", "feature_level": 35120124, - "kernel": "4.19", + "kernel": "KERNEL", "name": "NOOBS_NAME", "password": "raspberry", "release_date": "UNRELEASED", diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index c161ee6213..3ac53c49a6 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -53,6 +53,9 @@ mount "$BOOT_DEV" "${STAGE_WORK_DIR}/rootfs/boot" ln -sv "/lib/systemd/system/apply_noobs_os_config.service" "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/apply_noobs_os_config.service" +KERNEL_VER="$(zgrep -oPm 1 "Linux version \K(.*)$" "${STAGE_WORK_DIR}/rootfs/usr/share/doc/raspberrypi-kernel/changelog.Debian.gz" | cut -f-2 -d.)" +echo "$KERNEL_VER" > "${STAGE_WORK_DIR}/kernel_version" + bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs/boot" -cpf - . | xz -T0 > "${NOOBS_DIR}/boot.tar.xz" umount "${STAGE_WORK_DIR}/rootfs/boot" bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs" --one-file-system -cpf - . | xz -T0 > "${NOOBS_DIR}/root.tar.xz" From 69ee26fb34687bc0b03101af266a234988b71d84 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Mon, 4 Jan 2021 11:42:42 +0000 Subject: [PATCH 014/222] stage5: Remove scratch2 --- stage5/00-install-extras/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage5/00-install-extras/00-packages b/stage5/00-install-extras/00-packages index 609e718125..38c5c8d57d 100644 --- a/stage5/00-install-extras/00-packages +++ b/stage5/00-install-extras/00-packages @@ -1,6 +1,6 @@ mu-editor sonic-pi -scratch nuscratch scratch2 scratch3 +scratch nuscratch scratch3 smartsim minecraft-pi python-minecraftpi python-picraft python3-picraft From 96982d119a4aff8516fbf248f66021c611a85356 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Fri, 8 Jan 2021 19:23:47 +0000 Subject: [PATCH 015/222] Add 1.2 multiplier to nominal root partition size --- export-noobs/00-release/00-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export-noobs/00-release/00-run.sh b/export-noobs/00-release/00-run.sh index c2a4a51789..488098ab29 100755 --- a/export-noobs/00-release/00-run.sh +++ b/export-noobs/00-release/00-run.sh @@ -20,7 +20,7 @@ BOOT_SIZE="$(( BOOT_SIZE / 1024 / 1024 + 1))" ROOT_SIZE="$(( ROOT_SIZE / 1024 / 1024 + 1))" BOOT_NOM="256" -ROOT_NOM="$(( ROOT_SIZE + 400 ))" +ROOT_NOM="$(echo "$ROOT_SIZE" | awk '{printf "%.0f", (($1 + 400) * 1.2) + 0.5 }')" mv "${NOOBS_DIR}/OS.png" "${NOOBS_DIR}/${NOOBS_NAME// /_}.png" From 21090519d85bdaa1615d5d5057d37b09368ea5d2 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Mon, 4 Jan 2021 11:42:55 +0000 Subject: [PATCH 016/222] Update release notes --- .../00-release/files/release_notes.txt | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index d5b6629fc2..abc7bfcca9 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,22 @@ UNRELEASED: - * + * Chromium version 86.0.4240.197 included + * Screen reader support enabled in Chromium + * Adobe have end-of-lifed Flash Player, so it has been removed + * Scratch 2 required Flash, so it has been removed + * Added Epson printer drivers + * Added timeout to hide messages from USB device monitor after 5 seconds + * Bug fix - PulseAudio output was in mono + * Bug fix - brief audio interruptions at start of playback in VLC + * Bug fix - old ALSA output settings being used instead of PulseAudio settings by some applications + * Bug fix - crash in PulseAudio volume controller when used on multichannel devices + * Bug fix - battery monitor failing to load on x86 platforms + * Bug fix - setting of password in startup wizard failed if language was changed + * Bug fix - Chromium video playback lockup on small number of devices + * Bug fix - Chromium Google Maps 3D view artefacts + * Slovak, Italian and Norwegian translations updated + * Added Epson printer drivers + * Raspberry Pi firmware 70f1581eec2c036b7e9309f1af41c651fb125447 + * Linux kernel 5.4.83 2020-12-02: * PulseAudio now included and running by default * Bluealsa Bluetooth interface removed - Bluetooth audio is now handled by PulseAudio From bf8c9f53833016e1b9410d2eb3ed0149390975a7 Mon Sep 17 00:00:00 2001 From: Holger Pandel Date: Wed, 10 Feb 2021 11:37:15 +0100 Subject: [PATCH 017/222] Add QCOW2 build mechanism (#349) --- Dockerfile | 2 +- README.md | 93 ++++++++- build-docker.sh | 8 + build.sh | 147 +++++++++++++- depends | 2 + export-image/03-set-partuuid/00-run.sh | 19 +- export-image/04-finalise/01-run.sh | 25 ++- export-image/prerun.sh | 160 ++++++++-------- export-noobs/00-release/00-run.sh | 6 +- export-noobs/prerun.sh | 16 +- imagetool.sh | 112 +++++++++++ scripts/qcow2_handling | 256 +++++++++++++++++++++++++ stage0/prerun.sh | 2 +- 13 files changed, 734 insertions(+), 114 deletions(-) create mode 100755 imagetool.sh create mode 100644 scripts/qcow2_handling diff --git a/Dockerfile b/Dockerfile index 3d5874d17b..cb2b4ba785 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get -y update && \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ - binfmt-support ca-certificates \ + binfmt-support ca-certificates qemu-utils kpartx \ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ diff --git a/README.md b/README.md index 8b7f875af5..59f6a65fe1 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ To install the required dependencies for `pi-gen` you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ -dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl bc +dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl bc \ +qemu-utils kpartx ``` The file `depends` contains a list of tools needed. The format of this @@ -36,7 +37,30 @@ The following environment variables are supported: but you should use something else for a customized version. Export files in stages may add suffixes to `IMG_NAME`. - * `RELEASE` (Default: buster) +* `USE_QCOW2`(Default: `1` ) + + Instead of using traditional way of building the rootfs of every stage in + single subdirectories and copying over the previous one to the next one, + qcow2 based virtual disks with backing images are used in every stage. + This speeds up the build process and reduces overall space consumption + significantly. + + Additional optional parameters regarding qcow2 build: + + * `BASE_QCOW2_SIZE` (Default: 12G) + + Size of the virtual qcow2 disk. + Note: it will not actually use that much of space at once but defines the + maximum size of the virtual disk. If you change the build process by adding + a lot of bigger packages or additional build stages, it can be necessary to + increase the value because the virtual disk can run out of space like a normal + hard drive would. + + **CAUTION:** Although the qcow2 build mechanism will run fine inside Docker, it can happen + that the network block device is not disconnected correctly after the Docker process has + ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong) + +* `RELEASE` (Default: buster) The release version to build images against. Valid values are jessie, stretch buster, bullseye, and testing. @@ -340,6 +364,71 @@ follows: * Once you're happy with the image you can remove the SKIP_IMAGES files and export your image to test +# Regarding Qcow2 image building + +### Get infos about the image in use + +If you issue the two commands shown in the example below in a second command shell while a build +is running you can find out, which network block device is currently being used and which qcow2 image +is bound to it. + +Example: + +```bash +root@build-machine:~/$ lsblk | grep nbd +nbd1 43:32 0 10G 0 disk +├─nbd1p1 43:33 0 10G 0 part +└─nbd1p1 253:0 0 10G 0 part + +root@build-machine:~/$ ps xa | grep qemu-nbd + 2392 pts/6 S+ 0:00 grep --color=auto qemu-nbd +31294 ? Ssl 0:12 qemu-nbd --discard=unmap -c /dev/nbd1 image-stage4.qcow2 +``` + +Here you can see, that the qcow2 image `image-stage4.qcow2` is currently connected to `/dev/nbd1` with +the associated partition map `/dev/mapper/nbd1p1`. Don't worry that `lsblk` shows two entries. It is totally fine, because the device map is accessible via `/dev/mapper/nbd1p1` and also via `/dev/dm-0`. This is all part of the device mapper functionality of the kernel. See `dmsetup` for further information. + +### Mount a qcow2 image + +If you want to examine the content of a a single stage, you can simply mount the qcow2 image found in the `WORK_DIR` directory with the tool `./imagetool.sh`. + +See `./imagetool.sh -h` for further details on how to use it. + +### Disconnect an image if something went wrong + +It can happen, that your build stops in case of an error. Normally `./build.sh` should handle image disconnection appropriately, but in rare cases, especially during a Docker build, this may not work as expected. If that happens, starting a new build will fail and you may have to disconnect the image and/or device yourself. + +A typical message indicating that there are some orphaned device mapper entries is this: + +``` +Failed to set NBD socket +Disconnect client, due to: Unexpected end-of-file before all bytes were read +``` + +If that happens go through the following steps: + +1. First, check if the image is somehow mounted to a directory entry and umount it as you would any other block device, like i.e. a hard disk or USB stick. + +2. Second, to disconnect an image from `qemu-nbd`, the QEMU Disk Network Block Device Server, issue the following command (be sure to change the device name to the one actually used): + + ```bash + sudo qemu-nbd -d /dev/nbd1 + ``` + + Note: if you use Docker build, normally no active `qemu-nbd` process exists anymore as it will be terminated when the Docker container stops. + +3. To disconnect a device partition map from the network block device, execute: + + ```bash + sudo kpartx -d /dev/nbd1 + or + sudo ./imagetool.sh --cleanup + ``` + + Note: The `imagetool.sh` command will cleanup any /dev/nbdX that is not connected to a running `qemu-nbd` daemon. Be careful if you use network block devices for other tasks utilizing NBDs on your build machine as well. + +Now you should be able to start a new build without running into troubles again. Most of the time, especially when using Docker build, you will only need no. 3 to get everything up and running again. + # Troubleshooting ## `64 Bit Systems` diff --git a/build-docker.sh b/build-docker.sh index 350f722bde..5d31f86225 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,4 +1,5 @@ #!/bin/bash -eu + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" BUILD_OPTS="$*" @@ -87,6 +88,9 @@ ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ + --cap-add=ALL \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ @@ -98,6 +102,9 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then else trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \ + --cap-add=ALL \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ pi-gen \ @@ -106,6 +113,7 @@ else rsync -av work/*/build.log deploy/" & wait "$!" fi + echo "copying results from deploy/" ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy . ls -lah deploy diff --git a/build.sh b/build.sh index 6f720b1f50..ff008be167 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,5 @@ #!/bin/bash -e + # shellcheck disable=SC2119 run_sub_stage() { @@ -13,7 +14,7 @@ $(cat "${i}-debconf") SELEOF EOF - log "End ${SUB_STAGE_DIR}/${i}-debconf" + log "End ${SUB_STAGE_DIR}/${i}-debconf" fi if [ -f "${i}-packages-nr" ]; then log "Begin ${SUB_STAGE_DIR}/${i}-packages-nr" @@ -22,6 +23,11 @@ EOF on_chroot << EOF apt-get -o APT::Acquire::Retries=3 install --no-install-recommends -y $PACKAGES EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF +apt-get clean +EOF + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages-nr" fi @@ -32,6 +38,11 @@ EOF on_chroot << EOF apt-get -o APT::Acquire::Retries=3 install -y $PACKAGES EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF +apt-get clean +EOF + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages" fi @@ -82,17 +93,30 @@ EOF run_stage(){ log "Begin ${STAGE_DIR}" STAGE="$(basename "${STAGE_DIR}")" + pushd "${STAGE_DIR}" > /dev/null - unmount "${WORK_DIR}/${STAGE}" + STAGE_WORK_DIR="${WORK_DIR}/${STAGE}" ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs + + if [ "${USE_QCOW2}" = "1" ]; then + if [ ! -f SKIP ]; then + load_qimage + fi + else + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi + fi + if [ ! -f SKIP_IMAGES ]; then if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}" fi fi if [ ! -f SKIP ]; then - if [ "${CLEAN}" = "1" ]; then + if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then if [ -d "${ROOTFS_DIR}" ]; then rm -rf "${ROOTFS_DIR}" fi @@ -103,13 +127,21 @@ run_stage(){ log "End ${STAGE_DIR}/prerun.sh" fi for SUB_STAGE_DIR in "${STAGE_DIR}"/*; do - if [ -d "${SUB_STAGE_DIR}" ] && - [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then + if [ -d "${SUB_STAGE_DIR}" ] && [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then run_sub_stage fi done fi - unmount "${WORK_DIR}/${STAGE}" + + if [ "${USE_QCOW2}" = "1" ]; then + unload_qimage + else + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi + fi + PREV_STAGE="${STAGE}" PREV_STAGE_DIR="${STAGE_DIR}" PREV_ROOTFS_DIR="${ROOTFS_DIR}" @@ -143,6 +175,15 @@ do esac done +term() { + if [ "${USE_QCOW2}" = "1" ]; then + log "Unloading image" + unload_qimage + fi +} + +trap term EXIT INT TERM + export PI_GEN=${PI_GEN:-pi-gen} export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen} @@ -211,6 +252,18 @@ source "${SCRIPT_DIR}/common" # shellcheck source=scripts/dependencies_check source "${SCRIPT_DIR}/dependencies_check" +export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" +export USE_QCOW2="${USE_QCOW2:-1}" +export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G} +source "${SCRIPT_DIR}/qcow2_handling" +if [ "${USE_QCOW2}" = "1" ]; then + NO_PRERUN_QCOW2=1 +else + NO_PRERUN_QCOW2=0 +fi + +export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" + dependencies_check "${BASE_DIR}/depends" #check username is valid @@ -250,22 +303,98 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_IMAGE" EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs - run_stage + if [ "${USE_QCOW2}" = "1" ]; then + USE_QCOW2=0 + EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}" + echo "------------------------------------------------------------------------" + echo "Running export stage for ${EXPORT_NAME}" + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true + EXPORT_STAGE=$(basename "${EXPORT_DIR}") + for s in $STAGE_LIST; do + TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}") + done + FIRST_STAGE=${TMP_LIST%% *} + FIRST_IMAGE="image-${FIRST_STAGE}.qcow2" + + pushd "${WORK_DIR}" > /dev/null + echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}" + cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2" + + ARR=($TMP_LIST) + # rebase stage images to new export base + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="${EXPORT_NAME}" + continue + fi + echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done + + # commit current export stage into base export image + echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2" + qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2 + + # rebase stage images back to original first stage for easy re-run + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="image-${CURR_STAGE}" + continue + fi + echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done + popd > /dev/null + + mkdir -p "${WORK_DIR}/export-image/rootfs" + mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/" + echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs" + mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs" + + CLEAN=0 + run_stage + CLEAN=1 + USE_QCOW2=1 + + else + run_stage + fi if [ "${USE_QEMU}" != "1" ]; then if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_NOOBS" STAGE_DIR="${BASE_DIR}/export-noobs" - run_stage + if [ "${USE_QCOW2}" = "1" ]; then + USE_QCOW2=0 + run_stage + USE_QCOW2=1 + else + run_stage + fi fi fi done -if [ -x ${BASE_DIR}/postrun.sh ]; then +if [ -x postrun.sh ]; then log "Begin postrun.sh" cd "${BASE_DIR}" ./postrun.sh log "End postrun.sh" fi +if [ "${USE_QCOW2}" = "1" ]; then + unload_qimage +fi + log "End ${BASE_DIR}" diff --git a/depends b/depends index a86bc82123..cb029fdddd 100644 --- a/depends +++ b/depends @@ -17,3 +17,5 @@ file git lsmod:kmod bc +qemu-nbd:qemu-utils +kpartx diff --git a/export-image/03-set-partuuid/00-run.sh b/export-image/03-set-partuuid/00-run.sh index 1538c07f24..16e1b158ef 100755 --- a/export-image/03-set-partuuid/00-run.sh +++ b/export-image/03-set-partuuid/00-run.sh @@ -1,13 +1,18 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +if [ "${NO_PRERUN_QCOW2}" = "0" ]; then -IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" -BOOT_PARTUUID="${IMGID}-01" -ROOT_PARTUUID="${IMGID}-02" + IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" -sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" -sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + BOOT_PARTUUID="${IMGID}-01" + ROOT_PARTUUID="${IMGID}-02" + + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" + +fi -sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" diff --git a/export-image/04-finalise/01-run.sh b/export-image/04-finalise/01-run.sh index 91264e5d32..d7ace58d85 100755 --- a/export-image/04-finalise/01-run.sh +++ b/export-image/04-finalise/01-run.sh @@ -76,25 +76,30 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" dpkg -l --root "$ROOTFS_DIR" } >> "$INFO_FILE" -ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" - -unmount "${ROOTFS_DIR}" -zerofree "${ROOT_DEV}" - -unmount_image "${IMG_FILE}" - mkdir -p "${DEPLOY_DIR}" rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +mv "$INFO_FILE" "$DEPLOY_DIR/" + +if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" + + unmount "${ROOTFS_DIR}" + zerofree "${ROOT_DEV}" + + unmount_image "${IMG_FILE}" +else + unload_qimage + make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE" +fi + if [ "${DEPLOY_ZIP}" == "1" ]; then pushd "${STAGE_WORK_DIR}" > /dev/null zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \ "$(basename "${IMG_FILE}")" popd > /dev/null else - cp "$IMG_FILE" "$DEPLOY_DIR" + mv "$IMG_FILE" "$DEPLOY_DIR/" fi - -cp "$INFO_FILE" "$DEPLOY_DIR" diff --git a/export-image/prerun.sh b/export-image/prerun.sh index 4b5cf8cd5c..fad7f80e79 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -1,85 +1,87 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" - -unmount_image "${IMG_FILE}" - -rm -f "${IMG_FILE}" - -rm -rf "${ROOTFS_DIR}" -mkdir -p "${ROOTFS_DIR}" - -BOOT_SIZE="$((256 * 1024 * 1024))" -ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) - -# All partition sizes and starts will be aligned to this size -ALIGN="$((4 * 1024 * 1024))" -# Add this much space to the calculated file size. This allows for -# some overhead (since actual space usage is usually rounded up to the -# filesystem block size) and gives some free space on the resulting -# image. -ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" - -BOOT_PART_START=$((ALIGN)) -BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) -ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) -ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) -IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) - -truncate -s "${IMG_SIZE}" "${IMG_FILE}" - -parted --script "${IMG_FILE}" mklabel msdos -parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" -parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" - -PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) -BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) -BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) - -ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) -ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) - -echo "Mounting BOOT_DEV..." -cnt=0 -until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do - if [ $cnt -lt 5 ]; then - cnt=$((cnt + 1)) - echo "Error in losetup for BOOT_DEV. Retrying..." - sleep 5 - else - echo "ERROR: losetup for BOOT_DEV failed; exiting" - exit 1 - fi -done - -echo "Mounting ROOT_DEV..." -cnt=0 -until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do - if [ $cnt -lt 5 ]; then - cnt=$((cnt + 1)) - echo "Error in losetup for ROOT_DEV. Retrying..." - sleep 5 - else - echo "ERROR: losetup for ROOT_DEV failed; exiting" - exit 1 - fi -done - -echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" -echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" - -ROOT_FEATURES="^huge_file" -for FEATURE in metadata_csum 64bit; do +if [ "${NO_PRERUN_QCOW2}" = "0" ]; then + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + + unmount_image "${IMG_FILE}" + + rm -f "${IMG_FILE}" + + rm -rf "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}" + + BOOT_SIZE="$((256 * 1024 * 1024))" + ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) + + # All partition sizes and starts will be aligned to this size + ALIGN="$((4 * 1024 * 1024))" + # Add this much space to the calculated file size. This allows for + # some overhead (since actual space usage is usually rounded up to the + # filesystem block size) and gives some free space on the resulting + # image. + ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" + + BOOT_PART_START=$((ALIGN)) + BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) + ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) + ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) + IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) + + truncate -s "${IMG_SIZE}" "${IMG_FILE}" + + parted --script "${IMG_FILE}" mklabel msdos + parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" + parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" + + PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) + BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) + BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) + + ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) + ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) + + echo "Mounting BOOT_DEV..." + cnt=0 + until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do + if [ $cnt -lt 5 ]; then + cnt=$((cnt + 1)) + echo "Error in losetup for BOOT_DEV. Retrying..." + sleep 5 + else + echo "ERROR: losetup for BOOT_DEV failed; exiting" + exit 1 + fi + done + + echo "Mounting ROOT_DEV..." + cnt=0 + until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do + if [ $cnt -lt 5 ]; then + cnt=$((cnt + 1)) + echo "Error in losetup for ROOT_DEV. Retrying..." + sleep 5 + else + echo "ERROR: losetup for ROOT_DEV failed; exiting" + exit 1 + fi + done + + echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" + echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" + + ROOT_FEATURES="^huge_file" + for FEATURE in metadata_csum 64bit; do if grep -q "$FEATURE" /etc/mke2fs.conf; then - ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" + ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" fi -done -mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null -mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null + done + mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null + mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null -mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 -mkdir -p "${ROOTFS_DIR}/boot" -mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat + mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 + mkdir -p "${ROOTFS_DIR}/boot" + mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat -rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" -rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" + rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" + rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" +fi diff --git a/export-noobs/00-release/00-run.sh b/export-noobs/00-release/00-run.sh index 488098ab29..8524730c0f 100755 --- a/export-noobs/00-release/00-run.sh +++ b/export-noobs/00-release/00-run.sh @@ -41,4 +41,8 @@ sed "${NOOBS_DIR}/os.json" -i -e "s|KERNEL|$(cat "${STAGE_WORK_DIR}/kernel_versi sed "${NOOBS_DIR}/release_notes.txt" -i -e "s|UNRELEASED|${IMG_DATE}|" -cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" +if [ "${USE_QCOW2}" = "1" ]; then + mv "${NOOBS_DIR}" "${DEPLOY_DIR}/" +else + cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" +fi diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 3ac53c49a6..bb9450a3a8 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -1,11 +1,15 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}" -unmount_image "${IMG_FILE}" - mkdir -p "${STAGE_WORK_DIR}" -cp "${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" "${STAGE_WORK_DIR}/" + +if [ "${DEPLOY_ZIP}" == "1" ]; then + IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" +else + IMG_FILE="${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +fi + +unmount_image "${IMG_FILE}" rm -rf "${NOOBS_DIR}" @@ -60,4 +64,8 @@ bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs/boot" -cpf - umount "${STAGE_WORK_DIR}/rootfs/boot" bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs" --one-file-system -cpf - . | xz -T0 > "${NOOBS_DIR}/root.tar.xz" +if [ "${USE_QCOW2}" = "1" ]; then + rm "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/apply_noobs_os_config.service" +fi + unmount_image "${IMG_FILE}" diff --git a/imagetool.sh b/imagetool.sh new file mode 100755 index 0000000000..adc1ab74d5 --- /dev/null +++ b/imagetool.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +if [ "$(id -u)" != "0" ]; then + echo "Please run as root" 1>&2 + exit 1 +fi + +progname=$(basename $0) + +function usage() +{ + cat << HEREDOC + +Usage: + Mount Image : $progname [--mount] [--image-name ] [--mount-point ] + Umount Image: $progname [--umount] [--mount-point ] + Cleanup NBD : $progname [--cleanup] + + arguments: + -h, --help show this help message and exit + -c, --cleanup cleanup orphaned device mappings + -m, --mount mount image + -u, --umount umount image + -i, --image-name path to qcow2 image + -p, --mount-point mount point for image + + This tool will use /dev/nbd1 as default for mounting an image. If you want to use another device, execute like this: + NBD_DEV=/dev/nbd2 ./$progname --mount --image --mount-point + +HEREDOC +} + +MOUNT=0 +UMOUNT=0 +IMAGE="" +MOUNTPOINT="" + +nbd_cleanup() { + DEVS="$(lsblk | grep nbd | grep disk | cut -d" " -f1)" + if [ ! -z "${DEVS}" ]; then + for d in $DEVS; do + if [ ! -z "${d}" ]; then + QDEV="$(ps xa | grep $d | grep -v grep)" + if [ -z "${QDEV}" ]; then + kpartx -d /dev/$d && echo "Unconnected device map removed: /dev/$d" + fi + fi + done + fi +} + +# As long as there is at least one more argument, keep looping +while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + -h|--help) + usage + exit + ;; + -c|--cleanup) + nbd_cleanup + ;; + -m|--mount) + MOUNT=1 + ;; + -u|--umount) + UMOUNT=1 + ;; + -i|--image-name) + shift + IMAGE="$1" + ;; + -p|--mount-point) + shift + MOUNTPOINT="$1" + ;; + *) + echo "Unknown option '$key'" + usage + exit + ;; + esac + # Shift after checking all the cases to get the next option + shift +done + +if [ "${MOUNT}" = "1" ] && [ "${UMOUNT}" = "1" ]; then + usage + echo "Concurrent mount options not possible." + exit +fi + +if [ "${MOUNT}" = "1" ] && ([ -z "${IMAGE}" ] || [ -z "${MOUNTPOINT}" ]); then + usage + echo "Can not mount image. Image path and/or mount point missing." + exit +fi + +if [ "${UMOUNT}" = "1" ] && [ -z "${MOUNTPOINT}" ]; then + usage + echo "Can not umount. Mount point parameter missing." + exit +fi + +export NBD_DEV="${NBD_DEV:-/dev/nbd1}" +source scripts/qcow2_handling + +if [ "${MOUNT}" = "1" ]; then + mount_qimage "${MOUNTPOINT}" "${IMAGE}" +elif [ "${UMOUNT}" = "1" ]; then + umount_qimage "${MOUNTPOINT}" +fi diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling new file mode 100644 index 0000000000..f722937b7b --- /dev/null +++ b/scripts/qcow2_handling @@ -0,0 +1,256 @@ +#!/bin/bash + +# QCOW2 Routines + +export CURRENT_IMAGE +export CURRENT_MOUNTPOINT + +export NBD_DEV +export MAP_BOOT_DEV +export MAP_ROOT_DEV + +# set in build.sh +# should be fairly enough for the beginning +# overwrite here by uncommenting following lines +# BASE_QCOW2_SIZE=12G + +# find and initialize free block device nodes +init_nbd() { + modprobe nbd max_part=16 + if [ -z "${NBD_DEV}" ]; then + for x in /sys/class/block/nbd* ; do + S=`cat $x/size` + if [ "$S" == "0" ] ; then + NBD_DEV=/dev/$(basename $x) + MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1 + MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2 + break + fi + done + fi +} +export -f init_nbd + +# connect image to block device +connect_blkdev() { + init_nbd + qemu-nbd --discard=unmap -c $NBD_DEV "$1" + sync + kpartx -a $NBD_DEV + sync + CURRENT_IMAGE="$1" +} +export -f connect_blkdev + +# disconnect image from block device +disconnect_blkdev() { + kpartx -d $NBD_DEV + qemu-nbd -d $NBD_DEV + NBD_DEV= + MAP_BOOT_DEV= + MAP_ROOT_DEV= + CURRENT_IMAGE= +} +export -f disconnect_blkdev + +# mount qcow2 image: mount_image +mount_qimage() { + connect_blkdev "$1" + mount -v -t ext4 $MAP_ROOT_DEV "$2" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "$2/boot" + CURRENT_MOUNTPOINT="$2" +} +export -f mount_qimage + +# umount qcow2 image: umount_image +umount_qimage() { + sync + #umount "$1/boot" + while mount | grep -q "$1"; do + local LOCS + LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r) + for loc in $LOCS; do + echo "$loc" + while mountpoint -q "$loc" && ! umount "$loc"; do + sleep 0.1 + done + done + done + CURRENT_MOUNTPOINT= + disconnect_blkdev +} +export -f umount_qimage + +# create base image / backing image / mount image +load_qimage() { + if [ -z "${CURRENT_MOUNTPOINT}" ]; then + if [ ! -d "${ROOTFS_DIR}" ]; then + mkdir -p "${ROOTFS_DIR}"; + fi + + if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then + rm -f "${WORK_DIR}/image-${STAGE}.qcow2"; + fi + + if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then + pushd ${WORK_DIR} > /dev/null + init_nbd + if [ -z "${PREV_STAGE}" ]; then + echo "Creating base image: image-${STAGE}.qcow2" + # -o preallocation=falloc + qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE + sync + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + sync + sfdisk $NBD_DEV << EOF +4MiB,250MiB,c,* +254MiB,,83; +EOF + sync + kpartx -a $NBD_DEV + mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV + mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV + sync + else + if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then + exit 1; + fi + echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2" + qemu-img create -f qcow2 \ + -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \ + ${WORK_DIR}/image-${STAGE}.qcow2 + sync + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + sync + kpartx -a $NBD_DEV + fi + + mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot" + CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2 + CURRENT_MOUNTPOINT=${ROOTFS_DIR} + popd > /dev/null + else + mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}" + fi + echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})" + fi +} +export -f load_qimage + +# umount current image and refresh mount point env var +unload_qimage() { + if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then + fstrim -v "${CURRENT_MOUNTPOINT}" || true + umount_qimage "${CURRENT_MOUNTPOINT}" + fi +} +export -f unload_qimage + +# based on: https://github.com/SirLagz/RaspberryPi-ImgAutoSizer +# helper function for make_bootable_image, do not call directly +function resize_qcow2() { + if [ -z "$CALL_FROM_MBI" ]; then + echo "resize_qcow2: cannot be called directly, use make_bootable_image instead" + return 1 + fi + + # ROOT_MARGIN=$((800*1024*1024)) + ROOT_MARGIN=$((1*1024*1024)) + PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print` + PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '` + PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '` + + e2fsck -y -f $MAP_ROOT_DEV || true + + DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '` + BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }') + BLOCK_SIZE=${BLOCK_SIZE// /} + + let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE + resize2fs -p $MAP_ROOT_DEV $DATA_SIZE + sleep 1 + + let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE + let PART_NEW_END=$PART_START+$PART_NEW_SIZE + ACT1=`parted -s "$NBD_DEV" rm 2` + ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END` + NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '` +} +export -f resize_qcow2 + +# create raw img from qcow2: make_bootable_image +function make_bootable_image() { + + EXPORT_QCOW2="$1" + EXPORT_IMAGE="$2" + + echo "Connect block device to source qcow2" + connect_blkdev "${EXPORT_QCOW2}" + + echo "Resize fs and partition" + CALL_FROM_MBI=1 + resize_qcow2 + sync + CALL_FROM_MBI= + + echo "Disconnect block device" + disconnect_blkdev + + if [ -z "$NEW_IMG_SIZE" ]; then + echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit." + exit 1 + fi + + echo "Shrinking qcow2 image" + qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE + sync + + echo "Convert qcow2 to raw image" + qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}" + sync + + echo "Get PARTUUIDs from image" + IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")" + + BOOT_PARTUUID="${IMGID}-01" + echo "Boot: $BOOT_PARTUUID" + ROOT_PARTUUID="${IMGID}-02" + echo "Root: $ROOT_PARTUUID" + + echo "Mount image" + MOUNTROOT=${WORK_DIR}/tmpimage + mkdir -p $MOUNTROOT + + MOUNTPT=$MOUNTROOT + PARTITION=2 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + + MOUNTPT=$MOUNTROOT/boot + PARTITION=1 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + + if [ ! -d "${MOUNTROOT}/root" ]; then + echo "Image damaged or not mounted. Exit." + exit 1 + fi + + echo "Setup PARTUUIDs" + if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then + echo "Set UUIDs to make it bootable" + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt" + fi + + echo "Umount image" + sync + umount "${MOUNTROOT}/boot" || exit 1 + umount "${MOUNTROOT}" || exit 1 + + echo "Remove qcow2 export image" + rm -f "${EXPORT_QCOW2}" +} +export -f make_bootable_image diff --git a/stage0/prerun.sh b/stage0/prerun.sh index 3b257836b8..d4dd0a1fb6 100755 --- a/stage0/prerun.sh +++ b/stage0/prerun.sh @@ -1,5 +1,5 @@ #!/bin/bash -e -if [ ! -d "${ROOTFS_DIR}" ]; then +if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then bootstrap ${RELEASE} "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/ fi From 461109415073d2eb67083709662ba983cc191f14 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Thu, 4 Mar 2021 22:43:11 +0000 Subject: [PATCH 018/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index abc7bfcca9..f7df4ca1d2 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,17 @@ UNRELEASED: + * Thonny upgraded to version 3.3.5 + * SD Card Copier made compatible with NVMe devices; now built against GTK+3 toolkit + * Composite video options removed from Raspberry Pi 4 in Raspberry Pi Configuration + * Boot order options in raspi-config adjusted for more flexibility + * Recommended Software now built against GTK+3 toolkit + * Fix for crash in volume plugin when using keyboard could push value out of range + * Fix for focus changing between windows in file manager when using keyboard to navigate directory view + * Fix for Raspberry Pi 400 keyboard country not being read correctly in startup wizard + * Armenian and Japanese translations added to several packages + * Automatically load aes-neon-bs on ARM64 to speed up OpenSSL + * Raspberry Pi firmware fcf8d2f7639ad8d0330db9c8db9b71bd33eaaa28 + * Linux kernel 5.10.17 +2021-01-11: * Chromium version 86.0.4240.197 included * Screen reader support enabled in Chromium * Adobe have end-of-lifed Flash Player, so it has been removed From 66cd2d17a0d2d04985b83a2ba830915c9a7d81dc Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Wed, 31 Mar 2021 12:28:30 +0100 Subject: [PATCH 019/222] Install gpiozero in lite images --- export-noobs/00-release/files/release_notes.txt | 2 ++ stage2/01-sys-tweaks/00-packages | 2 ++ stage4/00-install-packages/00-packages | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index f7df4ca1d2..3983c93e12 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,6 @@ UNRELEASED: + * Install gpiozero in lite images +2021-03-04: * Thonny upgraded to version 3.3.5 * SD Card Copier made compatible with NVMe devices; now built against GTK+3 toolkit * Composite video options removed from Raspberry Pi 4 in Raspberry Pi Configuration diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 83ec74ce53..a1f4b59573 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -2,6 +2,8 @@ ssh less fbset sudo psmisc strace ed ncdu crda console-setup keyboard-configuration debconf-utils parted unzip build-essential manpages-dev python bash-completion gdb pkg-config python-rpi.gpio v4l-utils +python-gpiozero +python3-gpiozero avahi-daemon lua5.1 luajit diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index f9f5ef4395..85a9f806f8 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -13,7 +13,7 @@ tree libgl1-mesa-dri libgles1 libgles2-mesa xcompmgr geany piclone -wiringpi pigpio python-pigpio python3-pigpio raspi-gpio python-gpiozero python3-gpiozero python3-rpi.gpio +wiringpi pigpio python-pigpio python3-pigpio raspi-gpio python3-rpi.gpio python-spidev python3-spidev python-twython python3-twython python-smbus python3-smbus From dcfd74d7d1fa293065ac6d565711e9ff891fe2b8 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Fri, 7 May 2021 13:58:09 +0100 Subject: [PATCH 020/222] Update release notes --- .../00-release/files/release_notes.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 3983c93e12..6e9db375a4 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,24 @@ UNRELEASED: + * Chromium upgraded to version 88.0.4324.187 + * NuScratch upgraded to version 20210507 + * Node-RED upgraded to version 1.3.4 + * pigpio upgraded to version 1.79 + * Thonny upgraded to version 3.3.6 + * Icelandic and Italian translations updated for several packages + * piclone: Remove hiding of application in other desktops + * agnostics: Remove hiding of app in other desktops + * rp-bookshelf: + - Remove hiding of app in other desktops + - GTK+3 version + * lxplug-bluetooth: + - Fix some memory leaks + - Add authorisation dialog required by some BT-LE pairings + * alsa-utils: Add custom init files for bcm2835 on Raspberry Pi to set volume correctly + * rp-prefapps: Remove hiding of app in other desktops + * OpenSSH and OpenSSL speed improvements * Install gpiozero in lite images + * Raspberry Pi firmware 518ee7c871aaa9aaa88116953d57e73787ee6e43 + * Linux kernel 5.10.17 2021-03-04: * Thonny upgraded to version 3.3.5 * SD Card Copier made compatible with NVMe devices; now built against GTK+3 toolkit From f271ce26e91434e7607137f309c2411aa345f7c5 Mon Sep 17 00:00:00 2001 From: Claude Warren Date: Wed, 16 Jun 2021 14:32:21 +0100 Subject: [PATCH 021/222] Added PIGEN_DOCKER_OPTS to build-docker.sh (#521) --- README.md | 4 ++++ build-docker.sh | 2 ++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 59f6a65fe1..f0a757a9c8 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,10 @@ fix is to ensure `binfmt-support` is installed on the host machine before starting the `./build-docker.sh` script (or using your own docker build solution). +### Passing arguments to Docker + +When the docker image is run various required command line arguments are provided. For example the system mounts the `/dev` directory to the `/dev` directory within the docker container. If other arguments are required they may be specified in the PIGEN_DOCKER_OPTS environment variable. For example setting `PIGEN_DOCKER_OPTS="--add-host foo:192.168.0.23"` will add '192.168.0.23 foo' to the `/etc/hosts` file in the container. The `--name` +and `--privileged` options are already set by the script and should not be redefined. ## Stage Anatomy diff --git a/build-docker.sh b/build-docker.sh index 5d31f86225..9538a8b838 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -91,6 +91,7 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then --cap-add=ALL \ -v /dev:/dev \ -v /lib/modules:/lib/modules \ + ${PIGEN_DOCKER_OPTS} \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ @@ -105,6 +106,7 @@ else --cap-add=ALL \ -v /dev:/dev \ -v /lib/modules:/lib/modules \ + ${PIGEN_DOCKER_OPTS} \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ pi-gen \ From 4ebe5490d13f23626b7a45ed337d11e0106ccb47 Mon Sep 17 00:00:00 2001 From: Serge Schneider Date: Thu, 17 Jun 2021 16:52:07 +0100 Subject: [PATCH 022/222] Update Beginner's Guide download link --- stage4/03-bookshelf/00-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage4/03-bookshelf/00-run.sh b/stage4/03-bookshelf/00-run.sh index 5bd9885ddd..8244568bd6 100755 --- a/stage4/03-bookshelf/00-run.sh +++ b/stage4/03-bookshelf/00-run.sh @@ -1,7 +1,7 @@ #!/bin/sh -e BOOKSHELF_URL="https://magpi.raspberrypi.org/bookshelf.xml" -GUIDE_URL="$(curl -s "$BOOKSHELF_URL" | awk -F '[<>]' "/Raspberry Pi Beginner's Guide v3<\/TITLE>/ {f=1; next} f==1 && /PDF/ {print \$3; exit}")" +GUIDE_URL="$(curl -s "$BOOKSHELF_URL" | awk -F '[<>]' "/<TITLE>Raspberry Pi Beginner's Guide 4th Edition<\/TITLE>/ {f=1; next} f==1 && /PDF/ {print \$3; exit}")" OUTPUT="$(basename "$GUIDE_URL" | cut -f1 -d'?')" if [ ! -f "files/$OUTPUT" ]; then From 54e1bb428fb52c9d6b67fd1f679a2e3571162a30 Mon Sep 17 00:00:00 2001 From: DBongiorno-CG <83716695+DBongiorno-CG@users.noreply.github.com> Date: Thu, 17 Jun 2021 17:44:30 -0400 Subject: [PATCH 023/222] Added simple catch for default case in which PIGEN_DOCKER_OPTS is unspecified during docker build (#522) --- build-docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build-docker.sh b/build-docker.sh index 9538a8b838..01fd5173fd 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -48,6 +48,7 @@ fi CONTAINER_NAME=${CONTAINER_NAME:-pigen_work} CONTINUE=${CONTINUE:-0} PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-0} +PIGEN_DOCKER_OPTS=${PIGEN_DOCKER_OPTS:-""} if [ -z "${IMG_NAME}" ]; then echo "IMG_NAME not set in 'config'" 1>&2 From d562f2a35b8917cacab9a461626becbce424b652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ciesielski?= <45674786+miccies3@users.noreply.github.com> Date: Tue, 22 Jun 2021 16:04:51 +0200 Subject: [PATCH 024/222] Fix: update dependencies (#526) --- Dockerfile | 2 +- README.md | 2 +- depends | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb2b4ba785..837e430a61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get -y update && \ apt-get -y install --no-install-recommends \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ - bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ + libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ binfmt-support ca-certificates qemu-utils kpartx \ && rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index f0a757a9c8..62938e9e31 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ To install the required dependencies for `pi-gen` you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ -dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl bc \ +dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \ qemu-utils kpartx ``` diff --git a/depends b/depends index cb029fdddd..eeb24902d9 100644 --- a/depends +++ b/depends @@ -7,7 +7,7 @@ zerofree zip mkdosfs:dosfstools capsh:libcap2-bin -bsdtar +bsdtar:libarchive-tools grep rsync xz:xz-utils From f9d44edb72e9a153c997266189c79cb1f96b69f3 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 22 Jun 2021 15:48:09 +0100 Subject: [PATCH 025/222] Copy debootstrap.log on failure --- scripts/common | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/common b/scripts/common index e2048d904f..300d30b1e7 100644 --- a/scripts/common +++ b/scripts/common @@ -21,8 +21,10 @@ bootstrap(){ setarch linux32 capsh --drop=cap_setfcap -- -c "'${BOOTSTRAP_CMD}' $BOOTSTRAP_STR" || true - if [ -d "$2/debootstrap" ]; then - rmdir "$2/debootstrap" + if [ -d "$2/debootstrap" ] && ! rmdir "$2/debootstrap"; then + cp "$2/debootstrap/debootstrap.log" "${STAGE_WORK_DIR}" + log "bootstrap failed: please check ${STAGE_WORK_DIR}/debootstrap.log" + return 1 fi } export -f bootstrap From 42bbe02c789acfc60c1b9588a2bd9aaec7cc47f2 Mon Sep 17 00:00:00 2001 From: Bruno Binet <bruno.binet@gmail.com> Date: Thu, 24 Jun 2021 14:32:01 +0200 Subject: [PATCH 026/222] Fix broken imagetool.sh --mount (#529) --- imagetool.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/imagetool.sh b/imagetool.sh index adc1ab74d5..e8467e1873 100755 --- a/imagetool.sh +++ b/imagetool.sh @@ -103,10 +103,12 @@ if [ "${UMOUNT}" = "1" ] && [ -z "${MOUNTPOINT}" ]; then fi export NBD_DEV="${NBD_DEV:-/dev/nbd1}" +export MAP_BOOT_DEV=/dev/mapper/nbd1p1 +export MAP_ROOT_DEV=/dev/mapper/nbd1p2 source scripts/qcow2_handling if [ "${MOUNT}" = "1" ]; then - mount_qimage "${MOUNTPOINT}" "${IMAGE}" + mount_qimage "${IMAGE}" "${MOUNTPOINT}" elif [ "${UMOUNT}" = "1" ]; then umount_qimage "${MOUNTPOINT}" fi From ff1e8b1aac4a6680ff36092121b20def300556f7 Mon Sep 17 00:00:00 2001 From: MichaIng <micha@dietpi.com> Date: Wed, 1 Sep 2021 12:10:26 +0200 Subject: [PATCH 027/222] Remove deprecated elevator key from cmdline (#535) With recent kernel versions, the "deadline" I/O scheduler has been replaced with "mq-deadline", and the "elevator" kernel command line option has been deprecated in general with Linux 5.4: https://kernelnewbies.org/Linux_5.4?highlight=%28elevator%29#Block_layer Signed-off-by: MichaIng <micha@dietpi.com> --- stage1/00-boot-files/files/cmdline.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage1/00-boot-files/files/cmdline.txt b/stage1/00-boot-files/files/cmdline.txt index b815bd89db..03cde688e6 100644 --- a/stage1/00-boot-files/files/cmdline.txt +++ b/stage1/00-boot-files/files/cmdline.txt @@ -1 +1 @@ -console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 elevator=deadline fsck.repair=yes rootwait +console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 fsck.repair=yes rootwait From f6048c77456f9be6ae9a94776699969e3a91cd99 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 1 Sep 2021 14:03:11 +0100 Subject: [PATCH 028/222] Update 07-resize-init.diff --- stage2/01-sys-tweaks/00-patches/07-resize-init.diff | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/01-sys-tweaks/00-patches/07-resize-init.diff b/stage2/01-sys-tweaks/00-patches/07-resize-init.diff index 8c41357e8f..cb160ae3ed 100644 --- a/stage2/01-sys-tweaks/00-patches/07-resize-init.diff +++ b/stage2/01-sys-tweaks/00-patches/07-resize-init.diff @@ -1,5 +1,5 @@ --- stage2.orig/rootfs/boot/cmdline.txt +++ stage2/rootfs/boot/cmdline.txt @@ -1 +1 @@ --console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 elevator=deadline fsck.repair=yes rootwait -+console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 elevator=deadline fsck.repair=yes rootwait quiet init=/usr/lib/raspi-config/init_resize.sh +-console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 fsck.repair=yes rootwait ++console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 fsck.repair=yes rootwait quiet init=/usr/lib/raspi-config/init_resize.sh From 029f258cdd5787254f5c48c6c7eb041840786b4d Mon Sep 17 00:00:00 2001 From: Rob Smith <kormoc@gmail.com> Date: Sat, 19 Dec 2020 18:34:20 -0800 Subject: [PATCH 029/222] Don't check for binfmt_misc on arm platforms When building on a native ARM platform, binfmt_misc is not required to be loaded. This change checks the machine type and if it's a ARM platform, skip the binfmt_misc validation. --- scripts/dependencies_check | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/dependencies_check b/scripts/dependencies_check index 375512b751..32c42eff96 100644 --- a/scripts/dependencies_check +++ b/scripts/dependencies_check @@ -28,11 +28,26 @@ dependencies_check() false fi + # If we're building on a native arm platform, we don't need to check for + # binfmt_misc or require it to be loaded. - if ! grep -q "/proc/sys/fs/binfmt_misc" /proc/mounts; then - echo "Module binfmt_misc not loaded in host" - echo "Please run:" - echo " sudo modprobe binfmt_misc" - exit 1 + binfmt_misc_required=1 + + case $(uname -m) in + aarch64) + binfmt_misc_required=0 + ;; + arm*) + binfmt_misc_required=0 + ;; + esac + + if [[ "${binfmt_misc_required}" == "1" ]]; then + if ! grep -q "/proc/sys/fs/binfmt_misc" /proc/mounts; then + echo "Module binfmt_misc not loaded in host" + echo "Please run:" + echo " sudo modprobe binfmt_misc" + exit 1 + fi fi } From 2f12cf985a88977957e2743601246d2a07aeb660 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 20 Sep 2021 15:16:15 +0100 Subject: [PATCH 030/222] Remove IMG_DATE from work directories --- build.sh | 2 +- export-noobs/00-release/00-run.sh | 2 +- export-noobs/prerun.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index ff008be167..f70c6fbc79 100755 --- a/build.sh +++ b/build.sh @@ -198,7 +198,7 @@ export IMG_FILENAME="${IMG_FILENAME:-"${IMG_DATE}-${IMG_NAME}"}" export ZIP_FILENAME="${ZIP_FILENAME:-"image_${IMG_DATE}-${IMG_NAME}"}" export SCRIPT_DIR="${BASE_DIR}/scripts" -export WORK_DIR="${WORK_DIR:-"${BASE_DIR}/work/${IMG_DATE}-${IMG_NAME}"}" +export WORK_DIR="${WORK_DIR:-"${BASE_DIR}/work/${IMG_NAME}"}" export DEPLOY_DIR=${DEPLOY_DIR:-"${BASE_DIR}/deploy"} export DEPLOY_ZIP="${DEPLOY_ZIP:-1}" export LOG_FILE="${WORK_DIR}/build.log" diff --git a/export-noobs/00-release/00-run.sh b/export-noobs/00-release/00-run.sh index 8524730c0f..bfaea9f812 100755 --- a/export-noobs/00-release/00-run.sh +++ b/export-noobs/00-release/00-run.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}" +NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_NAME}${IMG_SUFFIX}" install -v -m 744 files/partition_setup.sh "${NOOBS_DIR}/" install -v files/partitions.json "${NOOBS_DIR}/" diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index bb9450a3a8..d97508a658 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}" +NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_NAME}${IMG_SUFFIX}" mkdir -p "${STAGE_WORK_DIR}" if [ "${DEPLOY_ZIP}" == "1" ]; then From a8596e0d894315396d62d3546ce96cd1e7ceaf75 Mon Sep 17 00:00:00 2001 From: Josh Ackley <jackley798@gmail.com> Date: Tue, 5 Oct 2021 04:56:04 -0700 Subject: [PATCH 031/222] Use sync mode when running 'kpartx -a' --- scripts/qcow2_handling | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling index f722937b7b..6b2a56bc4f 100644 --- a/scripts/qcow2_handling +++ b/scripts/qcow2_handling @@ -36,7 +36,7 @@ connect_blkdev() { init_nbd qemu-nbd --discard=unmap -c $NBD_DEV "$1" sync - kpartx -a $NBD_DEV + kpartx -as $NBD_DEV sync CURRENT_IMAGE="$1" } @@ -108,7 +108,7 @@ load_qimage() { 254MiB,,83; EOF sync - kpartx -a $NBD_DEV + kpartx -as $NBD_DEV mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV sync @@ -123,7 +123,7 @@ EOF sync qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 sync - kpartx -a $NBD_DEV + kpartx -as $NBD_DEV fi mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}" From d9b4a4a985fe5a015c0a774c9c75f21a2491c0c3 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 4 Oct 2021 15:40:35 +0100 Subject: [PATCH 032/222] Switch to bullseye --- Dockerfile | 2 +- README.md | 2 +- build-docker.sh | 4 +-- build.sh | 2 +- stage2/01-sys-tweaks/00-packages | 6 ++-- stage3/00-install-packages/00-packages | 1 - stage4/00-install-packages/00-packages | 20 ++++++------ stage4/00-install-packages/01-packages | 44 +++++++++++++------------- stage5/00-install-extras/00-packages | 4 +-- 9 files changed, 40 insertions(+), 45 deletions(-) diff --git a/Dockerfile b/Dockerfile index 837e430a61..e7fc561b79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=debian:buster +ARG BASE_IMAGE=debian:bullseye FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND noninteractive diff --git a/README.md b/README.md index 62938e9e31..b4b40bd2df 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ The following environment variables are supported: that the network block device is not disconnected correctly after the Docker process has ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong) -* `RELEASE` (Default: buster) +* `RELEASE` (Default: bullseye) The release version to build images against. Valid values are jessie, stretch buster, bullseye, and testing. diff --git a/build-docker.sh b/build-docker.sh index 01fd5173fd..85200e4c1a 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -78,10 +78,10 @@ BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')" # Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead case "$(uname -m)" in x86_64|aarch64) - BASE_IMAGE=i386/debian:buster + BASE_IMAGE=i386/debian:bullseye ;; *) - BASE_IMAGE=debian:buster + BASE_IMAGE=debian:bullseye ;; esac ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" diff --git a/build.sh b/build.sh index f70c6fbc79..423d62efae 100755 --- a/build.sh +++ b/build.sh @@ -207,7 +207,7 @@ export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi} export FIRST_USER_NAME=${FIRST_USER_NAME:-pi} export FIRST_USER_PASS=${FIRST_USER_PASS:-raspberry} -export RELEASE=${RELEASE:-buster} +export RELEASE=${RELEASE:-bullseye} export WPA_ESSID export WPA_PASSWORD export WPA_COUNTRY diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index a1f4b59573..4af7c43b45 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -1,8 +1,7 @@ ssh less fbset sudo psmisc strace ed ncdu crda console-setup keyboard-configuration debconf-utils parted unzip -build-essential manpages-dev python bash-completion gdb pkg-config -python-rpi.gpio v4l-utils -python-gpiozero +build-essential manpages-dev python3 bash-completion gdb pkg-config +python3-rpi.gpio v4l-utils python3-gpiozero avahi-daemon lua5.1 @@ -26,7 +25,6 @@ policykit-1 ssh-import-id rng-tools ethtool -vl805fw ntfs-3g pciutils rpi-eeprom diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index eac74ea545..460aedd2ce 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -3,7 +3,6 @@ qpdfview gtk2-engines alsa-utils desktop-base git omxplayer -raspberrypi-artwork policykit-1 gvfs rfkill diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index 85a9f806f8..e76d4110d0 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -1,11 +1,11 @@ -python python3-pygame python-pygame python-tk -python3 python3-tk thonny +python3-pygame +python3-tk thonny python3-pgzero -python-serial python3-serial -python-picamera python3-picamera +python3-serial +python3-picamera debian-reference-en dillo raspberrypi-net-mods raspberrypi-ui-mods -python-pip python3-pip +python3-pip python3-numpy pypy alacarte rc-gui sense-hat @@ -13,11 +13,11 @@ tree libgl1-mesa-dri libgles1 libgles2-mesa xcompmgr geany piclone -wiringpi pigpio python-pigpio python3-pigpio raspi-gpio python3-rpi.gpio -python-spidev python3-spidev -python-twython python3-twython -python-smbus python3-smbus -python-flask python3-flask +pigpio python3-pigpio raspi-gpio python3-rpi.gpio +python3-spidev +python3-twython +python3-smbus +python3-flask pprompt piwiz rp-prefapps diff --git a/stage4/00-install-packages/01-packages b/stage4/00-install-packages/01-packages index fdf5e7346e..5441d9dd16 100644 --- a/stage4/00-install-packages/01-packages +++ b/stage4/00-install-packages/01-packages @@ -1,22 +1,22 @@ -python-automationhat python3-automationhat -python-blinkt python3-blinkt -python-cap1xxx python3-cap1xxx -python-drumhat python3-drumhat -python-envirophat python3-envirophat -python-explorerhat python3-explorerhat -python-fourletterphat python3-fourletterphat -python-microdotphat python3-microdotphat -python-mote python3-mote -python-motephat python3-motephat -python-phatbeat python3-phatbeat -python-pianohat python3-pianohat -python-piglow python3-piglow -python-rainbowhat python3-rainbowhat -python-scrollphat python3-scrollphat -python-scrollphathd python3-scrollphathd -python-sn3218 python3-sn3218 -python-skywriter python3-skywriter -python-touchphat python3-touchphat -python-buttonshim python3-buttonshim -python-unicornhathd python3-unicornhathd -python-pantilthat python3-pantilthat +python3-automationhat +python3-blinkt +python3-cap1xxx +python3-drumhat +python3-envirophat +python3-explorerhat +python3-fourletterphat +python3-microdotphat +python3-mote +python3-motephat +python3-phatbeat +python3-pianohat +python3-piglow +python3-rainbowhat +python3-scrollphat +python3-scrollphathd +python3-sn3218 +python3-skywriter +python3-touchphat +python3-buttonshim +python3-unicornhathd +python3-pantilthat diff --git a/stage5/00-install-extras/00-packages b/stage5/00-install-extras/00-packages index 38c5c8d57d..1b9d4884d8 100644 --- a/stage5/00-install-extras/00-packages +++ b/stage5/00-install-extras/00-packages @@ -3,8 +3,7 @@ sonic-pi scratch nuscratch scratch3 smartsim -minecraft-pi python-minecraftpi python-picraft python3-picraft -python-sense-emu sense-emu-tools python-sense-emu-doc +python3-sense-emu sense-emu-tools python-sense-emu-doc wolfram-engine claws-mail @@ -12,5 +11,4 @@ greenfoot-unbundled bluej nodered realvnc-vnc-viewer -python-games code-the-classics From b23d0b144df1aeb87814b7ea84f27eaa30392955 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 5 Oct 2021 15:22:16 +0100 Subject: [PATCH 033/222] Use raspi-config to disable predictable network interface names --- stage1/01-sys-tweaks/00-packages | 1 + stage1/02-net-tweaks/00-run.sh | 4 +++- stage1/03-install-packages/00-packages | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 stage1/01-sys-tweaks/00-packages diff --git a/stage1/01-sys-tweaks/00-packages b/stage1/01-sys-tweaks/00-packages new file mode 100644 index 0000000000..9e91ea1ae3 --- /dev/null +++ b/stage1/01-sys-tweaks/00-packages @@ -0,0 +1 @@ +raspi-config diff --git a/stage1/02-net-tweaks/00-run.sh b/stage1/02-net-tweaks/00-run.sh index 4943cabf0d..95dfbdc5b7 100755 --- a/stage1/02-net-tweaks/00-run.sh +++ b/stage1/02-net-tweaks/00-run.sh @@ -3,4 +3,6 @@ echo "${TARGET_HOSTNAME}" > "${ROOTFS_DIR}/etc/hostname" echo "127.0.1.1 ${TARGET_HOSTNAME}" >> "${ROOTFS_DIR}/etc/hosts" -ln -sf /dev/null "${ROOTFS_DIR}/etc/systemd/network/99-default.link" +on_chroot << EOF + SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_net_names 1 +EOF diff --git a/stage1/03-install-packages/00-packages b/stage1/03-install-packages/00-packages index e8c148c0b5..73edcc901c 100644 --- a/stage1/03-install-packages/00-packages +++ b/stage1/03-install-packages/00-packages @@ -1 +1 @@ -libraspberrypi-bin libraspberrypi0 raspi-config +libraspberrypi-bin libraspberrypi0 From 7c45f4e11f882495856abee521b3624cc5e1e004 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 5 Oct 2021 15:25:17 +0100 Subject: [PATCH 034/222] stage3: install libcamera-tools and libcamera-apps --- stage3/00-install-packages/00-packages | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index 460aedd2ce..25d2542faa 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -12,3 +12,5 @@ fonts-droid-fallback fonts-liberation2 obconf arandr +libcamera-tools +libcamera-apps From 456aa2e50b08ff71e96e139aafcb6f52f57b729f Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 5 Oct 2021 15:26:16 +0100 Subject: [PATCH 035/222] bootstrap: do not install the info package --- scripts/common | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/common b/scripts/common index 300d30b1e7..5b0df12beb 100644 --- a/scripts/common +++ b/scripts/common @@ -16,6 +16,7 @@ bootstrap(){ BOOTSTRAP_ARGS+=(--arch armhf) BOOTSTRAP_ARGS+=(--components "main,contrib,non-free") BOOTSTRAP_ARGS+=(--keyring "${STAGE_DIR}/files/raspberrypi.gpg") + BOOTSTRAP_ARGS+=(--exclude=info) BOOTSTRAP_ARGS+=("$@") printf -v BOOTSTRAP_STR '%q ' "${BOOTSTRAP_ARGS[@]}" From d0eed3b76f0983dea8e9333007dd5e46caf3ba1e Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 5 Oct 2021 15:51:43 +0100 Subject: [PATCH 036/222] Update config.txt Disable overscan Enable camera_auto_detect Switch all models to KMS Enable arm_boost on pi 4 Enable host mode on built-in USB controller on CM4 --- stage1/00-boot-files/files/config.txt | 28 +++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/stage1/00-boot-files/files/config.txt b/stage1/00-boot-files/files/config.txt index 548f4accc2..0e8f1b0bd0 100644 --- a/stage1/00-boot-files/files/config.txt +++ b/stage1/00-boot-files/files/config.txt @@ -5,10 +5,6 @@ # uncomment if you get no picture on HDMI for a default "safe" mode #hdmi_safe=1 -# uncomment this if your display has a black border of unused pixels visible -# and your display can output without overscan -#disable_overscan=1 - # uncomment the following to adjust overscan. Use positive numbers if console # goes off screen, and negative if there is too much border #overscan_left=16 @@ -56,10 +52,26 @@ # Enable audio (loads snd_bcm2835) dtparam=audio=on -[pi4] -# Enable DRM VC4 V3D driver on top of the dispmanx display stack -dtoverlay=vc4-fkms-v3d +# Automatically load overlays for detected cameras +camera_auto_detect=1 + +# Enable DRM VC4 V3D driver +dtoverlay=vc4-kms-v3d max_framebuffers=2 +# Disable compensation for displays with overscan +disable_overscan=1 + +[cm4] +# Enable host mode on the 2711 built-in XHCI USB controller. +# This line should be removed if the legacy DWC2 controller is required +# (e.g. for USB device mode) or if USB support is not required. +otg_mode=1 + +[all] + +[pi4] +# Run as fast as firmware / board allows +arm_boost=1 + [all] -#dtoverlay=vc4-fkms-v3d From ac246e300bdd55005ab811bcd817f1221b328fd7 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 5 Oct 2021 15:54:05 +0100 Subject: [PATCH 037/222] Use raspi-config to toggle boot_wait --- stage2/02-net-tweaks/01-run.sh | 7 ++++--- stage2/02-net-tweaks/files/wait.conf | 3 --- stage3/01-tweaks/00-run.sh | 4 +++- 3 files changed, 7 insertions(+), 7 deletions(-) delete mode 100644 stage2/02-net-tweaks/files/wait.conf diff --git a/stage2/02-net-tweaks/01-run.sh b/stage2/02-net-tweaks/01-run.sh index d82381b1b7..837e86135e 100755 --- a/stage2/02-net-tweaks/01-run.sh +++ b/stage2/02-net-tweaks/01-run.sh @@ -1,11 +1,12 @@ #!/bin/bash -e -install -v -d "${ROOTFS_DIR}/etc/systemd/system/dhcpcd.service.d" -install -v -m 644 files/wait.conf "${ROOTFS_DIR}/etc/systemd/system/dhcpcd.service.d/" - install -v -d "${ROOTFS_DIR}/etc/wpa_supplicant" install -v -m 600 files/wpa_supplicant.conf "${ROOTFS_DIR}/etc/wpa_supplicant/" +on_chroot << EOF + SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_boot_wait 0 +EOF + if [ -v WPA_COUNTRY ]; then echo "country=${WPA_COUNTRY}" >> "${ROOTFS_DIR}/etc/wpa_supplicant/wpa_supplicant.conf" fi diff --git a/stage2/02-net-tweaks/files/wait.conf b/stage2/02-net-tweaks/files/wait.conf deleted file mode 100644 index 595cc2d9e1..0000000000 --- a/stage2/02-net-tweaks/files/wait.conf +++ /dev/null @@ -1,3 +0,0 @@ -[Service] -ExecStart= -ExecStart=/usr/lib/dhcpcd5/dhcpcd -q -w diff --git a/stage3/01-tweaks/00-run.sh b/stage3/01-tweaks/00-run.sh index 5da7c1a197..79b749e397 100755 --- a/stage3/01-tweaks/00-run.sh +++ b/stage3/01-tweaks/00-run.sh @@ -1,3 +1,5 @@ #!/bin/bash -e -rm -f "${ROOTFS_DIR}/etc/systemd/system/dhcpcd.service.d/wait.conf" +on_chroot << EOF + SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_boot_wait 1 +EOF From 5a4b7ec0679dc85efc5abfa65a9a33a3e3cb1273 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 5 Oct 2021 15:54:27 +0100 Subject: [PATCH 038/222] Update release notes --- .../00-release/files/release_notes.txt | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 6e9db375a4..d883488661 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,52 @@ UNRELEASED: + * Based on Debian version 11 (bullseye) + * Desktop components (lxpanel and all plugins, libfm, pcmanfm) now built against GTK+3 + * Applications (piwiz, pipanel, rc_gui, lxinput) now built against GTK+3 + * PiXflat GTK+3 theme updated with numerous changes to support the above + * GTK+3 : toolbar icon size setting added + * GTK+3 : ability to request client-side decoration on windows added + * GTK+3 : setting for indent for frame labels in custom style added + * mutter window manager used instead of openbox on devices with 2GB or more of RAM + * mutter : title bar icon behaviour and appearance modified to match openbox + * mutter : additional keyboard shortcuts added + * mutter : various performance enhancements + * mutter compatibility added to screen magnifier + * Numerous changes to Appearance Settings application to support GTK+3 and mutter + * Updater plugin added to lxpanel to detect and install software updates + * File manager view options simplified to either list or icons, with separate menu option for thumbnails + * New file manager toolbar icons + * KMS used as default display driver + * Modifications to HDMI audio output selection to support the above + * xcompmgr enabled when openbox is running under KMS + * New default camera subsystem based on libcamera + * New camera demo applications (libcamera-still and libcamera-vid) have replaced raspistill and raspivid + * Legacy camera subsystem removed from 64-bit RPi OS (still available on 32-bit) + * Chromium upgraded to version 92.0.4515.98 + * VLC media player upgraded to version 3.0.16 + * Spurious drive removal warning after use of SD card copier removed + * Bookshelf application now includes Custom PC magazine + * Various translation updates - Italian, Korean, Polish, German, Armenian + * Startup wizard now installs Japanese fonts if needed + * Progress and information dialog boxes for lxpanel plugins now common to lxpanel, rather than in individual plugins + * Icon handling code for lxpanel plugins now common to lxpanel + * Package with 4K version of Raspberry Pi wallpaper added to Recommended Software + * Python Games and Minecraft removed from Recommended Software - neither is compatible with bullseye + * Bluetooth pairing and connection dialogs updated for compatibility with more devices + * Bluetooth devices always disconnected before removal to speed up removal process + * Bluetooth pairing dialog now only shows devices which offer services which are usable by Pi + * Separate Bluetooth unpair dialog removed - unpair now an option for each individual device + * Bug fix - mutter : header bar colours not updating when theme is changed + * Bug fix - GTK+3 : tooltips being displayed incorrectly at bottom of screen + * Bug fix - lxpanel : crash when using keyboard shortcut to enable magnifier when magnifier not installed + * Bug fix - lxpanel : lockup in Bluetooth plugin when connecting to certain devices + * Bug fix - lxpanel : discoverable mode icon could get out of sync with underlying Bluetooth system state + * Bug fix - piwiz : missing cities in timezone list + * Bug fix - piwiz : country-specific language packages not being installed + * Bug fix - bookshelf : now waits for longer between packets before timing out + * Bug fix - accented characters now displayed correctly in localisation dialogs + * Raspberry Pi firmware 46f356dcecc5204ca9a8cef7e6894e083b231bc7 + * Linux kernel 5.10.60 +2021-05-07: * Chromium upgraded to version 88.0.4324.187 * NuScratch upgraded to version 20210507 * Node-RED upgraded to version 1.3.4 From b542f68d68155006d63d4bc3d9a41fd5107e8b6d Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Oct 2021 10:08:16 +0100 Subject: [PATCH 039/222] stage2: install python-is-python3 --- stage2/01-sys-tweaks/00-packages | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 4af7c43b45..55f845140c 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -1,6 +1,7 @@ ssh less fbset sudo psmisc strace ed ncdu crda console-setup keyboard-configuration debconf-utils parted unzip -build-essential manpages-dev python3 bash-completion gdb pkg-config +build-essential manpages-dev bash-completion gdb pkg-config +python-is-python3 python3-rpi.gpio v4l-utils python3-gpiozero avahi-daemon From 93beb9fb743dbc2a956c7d49111623759fe7906a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 15 Oct 2021 16:04:41 +0100 Subject: [PATCH 040/222] stage2: install libcamera-apps-lite --- stage2/01-sys-tweaks/00-packages-nr | 1 + 1 file changed, 1 insertion(+) diff --git a/stage2/01-sys-tweaks/00-packages-nr b/stage2/01-sys-tweaks/00-packages-nr index 5acef540d0..83a1e0ceea 100644 --- a/stage2/01-sys-tweaks/00-packages-nr +++ b/stage2/01-sys-tweaks/00-packages-nr @@ -1 +1,2 @@ cifs-utils +libcamera-apps-lite From 6d363779bae70cf027ef5eead3d6fa4d6f603bee Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 29 Oct 2021 10:21:08 +0100 Subject: [PATCH 041/222] Automatically load overlays for detected DSI displays --- stage1/00-boot-files/files/config.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stage1/00-boot-files/files/config.txt b/stage1/00-boot-files/files/config.txt index 0e8f1b0bd0..6e072cb999 100644 --- a/stage1/00-boot-files/files/config.txt +++ b/stage1/00-boot-files/files/config.txt @@ -55,6 +55,9 @@ dtparam=audio=on # Automatically load overlays for detected cameras camera_auto_detect=1 +# Automatically load overlays for detected DSI displays +display_auto_detect=1 + # Enable DRM VC4 V3D driver dtoverlay=vc4-kms-v3d max_framebuffers=2 From 85bc296d16abc116c51e8fc84fcb712189efe4e5 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 29 Oct 2021 10:24:13 +0100 Subject: [PATCH 042/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index d883488661..9de6b2424d 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -44,8 +44,8 @@ UNRELEASED: * Bug fix - piwiz : country-specific language packages not being installed * Bug fix - bookshelf : now waits for longer between packets before timing out * Bug fix - accented characters now displayed correctly in localisation dialogs - * Raspberry Pi firmware 46f356dcecc5204ca9a8cef7e6894e083b231bc7 - * Linux kernel 5.10.60 + * Raspberry Pi firmware 0c4fc71befd228419a225c84d316cabbda8633a7 + * Linux kernel 5.10.63 2021-05-07: * Chromium upgraded to version 88.0.4324.187 * NuScratch upgraded to version 20210507 From 288b21fc27e128ea6b330777aca68e0061ebf4fe Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 29 Oct 2021 20:37:45 +0100 Subject: [PATCH 043/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 9de6b2424d..516f7f7180 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -44,7 +44,7 @@ UNRELEASED: * Bug fix - piwiz : country-specific language packages not being installed * Bug fix - bookshelf : now waits for longer between packets before timing out * Bug fix - accented characters now displayed correctly in localisation dialogs - * Raspberry Pi firmware 0c4fc71befd228419a225c84d316cabbda8633a7 + * Raspberry Pi firmware e2bab29767e51c683a312df20014e3277275b8a6 * Linux kernel 5.10.63 2021-05-07: * Chromium upgraded to version 88.0.4324.187 From 68fee340f95eb630a706e2dbf75615c6be4e980b Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 8 Nov 2021 08:44:36 +0000 Subject: [PATCH 044/222] Do not use qcow2 images by default --- README.md | 2 +- build.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b4b40bd2df..b85930965a 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ The following environment variables are supported: but you should use something else for a customized version. Export files in stages may add suffixes to `IMG_NAME`. -* `USE_QCOW2`(Default: `1` ) +* `USE_QCOW2` **EXPERIMENTAL** (Default: `0` ) Instead of using traditional way of building the rootfs of every stage in single subdirectories and copying over the previous one to the next one, diff --git a/build.sh b/build.sh index 423d62efae..bcfdeb69ff 100755 --- a/build.sh +++ b/build.sh @@ -99,7 +99,7 @@ run_stage(){ STAGE_WORK_DIR="${WORK_DIR}/${STAGE}" ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs - if [ "${USE_QCOW2}" = "1" ]; then + if [ "${USE_QCOW2}" = "1" ]; then if [ ! -f SKIP ]; then load_qimage fi @@ -109,7 +109,7 @@ run_stage(){ unmount "${WORK_DIR}/${STAGE}" fi fi - + if [ ! -f SKIP_IMAGES ]; then if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}" @@ -253,7 +253,7 @@ source "${SCRIPT_DIR}/common" source "${SCRIPT_DIR}/dependencies_check" export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" -export USE_QCOW2="${USE_QCOW2:-1}" +export USE_QCOW2="${USE_QCOW2:-0}" export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G} source "${SCRIPT_DIR}/qcow2_handling" if [ "${USE_QCOW2}" = "1" ]; then From 1e0c20690cb6915c85ebb0e598614198be9d253f Mon Sep 17 00:00:00 2001 From: marcone <48169102+marcone@users.noreply.github.com> Date: Sat, 6 Nov 2021 18:23:40 -0700 Subject: [PATCH 045/222] Fix imagetool example The long option name is "--image-name", not "--image" --- imagetool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagetool.sh b/imagetool.sh index e8467e1873..002b50bc34 100755 --- a/imagetool.sh +++ b/imagetool.sh @@ -25,7 +25,7 @@ Usage: -p, --mount-point mount point for image This tool will use /dev/nbd1 as default for mounting an image. If you want to use another device, execute like this: - NBD_DEV=/dev/nbd2 ./$progname --mount --image <your image> --mount-point <your path> + NBD_DEV=/dev/nbd2 ./$progname --mount --image-name <your image> --mount-point <your path> HEREDOC } From 27983073c97bbf2851a8c36ff6450ec9b33ee4a0 Mon Sep 17 00:00:00 2001 From: myxxmikeyxx <jimmy98719871@gmail.com> Date: Wed, 10 Nov 2021 08:13:28 -0500 Subject: [PATCH 046/222] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b85930965a..08b2c13f29 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ The following environment variables are supported: * `RELEASE` (Default: bullseye) - The release version to build images against. Valid values are jessie, stretch + The release version to build images against. Valid values are jessie, stretch, buster, bullseye, and testing. * `APT_PROXY` (Default: unset) From 9e5228553f7433d22f9fead87198e777a5dec7c4 Mon Sep 17 00:00:00 2001 From: XECDesign <XECDesign@users.noreply.github.com> Date: Wed, 10 Nov 2021 18:20:11 +0000 Subject: [PATCH 047/222] Remove omxplayer --- stage3/00-install-packages/00-packages | 1 - 1 file changed, 1 deletion(-) diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index 25d2542faa..97828ba75c 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -2,7 +2,6 @@ gstreamer1.0-x gstreamer1.0-omx gstreamer1.0-plugins-base gstreamer1.0-plugins-g qpdfview gtk2-engines alsa-utils desktop-base git -omxplayer policykit-1 gvfs rfkill From 6117b9bb6193e377d6bf99d5d94c889525bc1a34 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 12 Nov 2021 10:16:27 +0000 Subject: [PATCH 048/222] Add user to render group --- stage2/01-sys-tweaks/01-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index c1836f9ca6..bb3e11ac08 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -52,7 +52,7 @@ on_chroot <<EOF for GRP in input spi i2c gpio; do groupadd -f -r "\$GRP" done -for GRP in adm dialout cdrom audio users sudo video games plugdev input gpio spi i2c netdev; do +for GRP in adm dialout cdrom audio users sudo video games plugdev input gpio spi i2c netdev render; do adduser $FIRST_USER_NAME \$GRP done EOF From 69cd27944c9d81b8170e8ade5ccbb23895599ed1 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 2 Dec 2021 12:37:19 +0000 Subject: [PATCH 049/222] Update chromium install --- stage3/00-install-packages/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index 97828ba75c..e04dbd705d 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -5,7 +5,7 @@ git policykit-1 gvfs rfkill -chromium-browser rpi-chromium-mods +chromium-browser rpi-chromium-mods libwidevinecdm0 gldriver-test fonts-droid-fallback fonts-liberation2 From 3fb5442494dc28e66c92567a58850d56c03a495a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 6 Dec 2021 10:06:28 +0000 Subject: [PATCH 050/222] Update release notes --- .../00-release/files/release_notes.txt | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 516f7f7180..aaaa185999 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,38 @@ UNRELEASED: + * rc_gui - add combo box to allow resolution to be set for VNC connections + * rc_gui - camera interface switch removed + * lxpanel - remove appearance settings from preferences dialog; instead add menu option to open general Appearance Settings application + * lxpanel - add ellipses to menu items which open dialogs + * lxinput - read current mouse acceleration directly from xinput + * lxinput - use device IDs rather than names to cope with devices changing when powered-down + * lxinput - remove redundant changes to openbox config file + * plymouth - set KillMode to mixed to suppress warning message + * raspi-config - add option to switch composite video + * raspi-config - add option to switch to legacy camera mode + * raspi-config - add option to set resolution for headless connections + * raspberrypi-ui-mods - disable mutter when VNC server is running and fall back to openbox + * pipanel - add command-line option to open on arbitrary tab + * lxplug-network - suppress ’scan received’ logging message + * raspberrypi-ui-mods - set hover colour for taskbar items based on taskbar colour, not system highlight colour + * Legacy camera applications and libraries reinstalled + * Bug fix - lxinput - lxsession config file not being written on first attempt + * Bug fix - lxinput - set timer for file write to prevent slider slowing down + * Bug fix - lxinput - write values to gsettings as well as xinput and xsettings to take effect within mutter + * Bug fix - lxinput - fix failure to parse and write non-English numeric formats + * Bug fix - arandr - various fixes to parsing of non-standard EDID blocks to enable model and serial to be correctly extracted + * Bug fix - arandr - refresh rate calculated to 3 decimal places for monitors which require it + * Bug fix - arandr - enable setting of left and right orientation + * Bug fix - arandr - add compatibility with new touchscreen driver + * Bug fix - arandr - apply settings correctly to DSI and composite displays + * Bug fix - lxplug-magnifier - fix crash when opening preferences without required magnifier package installed + * Bug fix - piwiz - launch screen reader install prompt as a new process to prevent audio lockups crashing wizard + * Bug fix - lxpanel - not loading some plugins (cpufreq, minimise all windows) due to icon loading code not compatible with GTK+3 + * Bug fix - gtk+3 - disabled new GDK touch events to enable double-clicks to be detected on touchscreen + * Bug fix - xrdp - included backports from bookworm version of xrdp and xorgxrdp to restore window frames with mutter over xrdp connections + * Update various translations + * Raspberry Pi firmware + * Linux kernel +2021-10-30: * Based on Debian version 11 (bullseye) * Desktop components (lxpanel and all plugins, libfm, pcmanfm) now built against GTK+3 * Applications (piwiz, pipanel, rc_gui, lxinput) now built against GTK+3 From 351024d3c363979011832076e9524593bba32aa0 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 6 Dec 2021 16:03:29 +0000 Subject: [PATCH 051/222] stage2: Install udisks2 --- export-noobs/00-release/files/release_notes.txt | 1 + stage2/01-sys-tweaks/00-packages | 1 + 2 files changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index aaaa185999..bdc05e2abe 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -30,6 +30,7 @@ UNRELEASED: * Bug fix - gtk+3 - disabled new GDK touch events to enable double-clicks to be detected on touchscreen * Bug fix - xrdp - included backports from bookworm version of xrdp and xorgxrdp to restore window frames with mutter over xrdp connections * Update various translations + * udisks2 added to lite image * Raspberry Pi firmware * Linux kernel 2021-10-30: diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 55f845140c..d22a961f6a 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -30,3 +30,4 @@ ntfs-3g pciutils rpi-eeprom raspinfo +udisks2 From f04a4143e92d8e4997d4fcbc7485f99459f0bb31 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 22 Dec 2021 15:45:12 +0000 Subject: [PATCH 052/222] Install mkvtoolnix for mkvmerge --- export-noobs/00-release/files/release_notes.txt | 1 + stage2/01-sys-tweaks/00-packages-nr | 1 + 2 files changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index bdc05e2abe..914e7cfca8 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -31,6 +31,7 @@ UNRELEASED: * Bug fix - xrdp - included backports from bookworm version of xrdp and xorgxrdp to restore window frames with mutter over xrdp connections * Update various translations * udisks2 added to lite image + * mkvtoolnix added to lite image * Raspberry Pi firmware * Linux kernel 2021-10-30: diff --git a/stage2/01-sys-tweaks/00-packages-nr b/stage2/01-sys-tweaks/00-packages-nr index 83a1e0ceea..6bdb60069e 100644 --- a/stage2/01-sys-tweaks/00-packages-nr +++ b/stage2/01-sys-tweaks/00-packages-nr @@ -1,2 +1,3 @@ cifs-utils libcamera-apps-lite +mkvtoolnix From 786af7c5a4a7e4cc29b45596d9aa9dc9ba830453 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 22 Dec 2021 15:47:54 +0000 Subject: [PATCH 053/222] stage3: install gnome-keyring --- export-noobs/00-release/files/release_notes.txt | 1 + stage3/00-install-packages/00-packages-nr | 1 + 2 files changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 914e7cfca8..b912d1c673 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -32,6 +32,7 @@ UNRELEASED: * Update various translations * udisks2 added to lite image * mkvtoolnix added to lite image + * gnome-keyring added to desktop images * Raspberry Pi firmware * Linux kernel 2021-10-30: diff --git a/stage3/00-install-packages/00-packages-nr b/stage3/00-install-packages/00-packages-nr index ffc324bf4a..75fa2a1c5f 100644 --- a/stage3/00-install-packages/00-packages-nr +++ b/stage3/00-install-packages/00-packages-nr @@ -4,3 +4,4 @@ lxde lxtask menu-xdg zenity xdg-utils gvfs-backends gvfs-fuse lightdm gnome-themes-standard-data gnome-icon-theme +gnome-keyring From 23f75d665521e8ac2e9a5fcbde405dce80820715 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 10 Jan 2022 12:41:48 +0000 Subject: [PATCH 054/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index b912d1c673..b681a064f7 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -33,8 +33,8 @@ UNRELEASED: * udisks2 added to lite image * mkvtoolnix added to lite image * gnome-keyring added to desktop images - * Raspberry Pi firmware - * Linux kernel + * Raspberry Pi firmware cfdbadea5f74c16b7ed5d3b4866092a054e3c3bf + * Linux kernel 5.10.89 2021-10-30: * Based on Debian version 11 (bullseye) * Desktop components (lxpanel and all plugins, libfm, pcmanfm) now built against GTK+3 From cc8f695b9a935717bc96a0d1c74d5d28c3dae92a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 12 Jan 2022 10:21:47 +0000 Subject: [PATCH 055/222] Add 7z and zip support https://github.com/RPi-Distro/repo/issues/266#issuecomment-1010713450 --- export-noobs/00-release/files/release_notes.txt | 1 + stage2/01-sys-tweaks/00-packages | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index b681a064f7..88d9000e15 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -32,6 +32,7 @@ UNRELEASED: * Update various translations * udisks2 added to lite image * mkvtoolnix added to lite image + * 7z and zip support added to lite image * gnome-keyring added to desktop images * Raspberry Pi firmware cfdbadea5f74c16b7ed5d3b4866092a054e3c3bf * Linux kernel 5.10.89 diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index d22a961f6a..7656f03d1a 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -1,5 +1,5 @@ ssh less fbset sudo psmisc strace ed ncdu crda -console-setup keyboard-configuration debconf-utils parted unzip +console-setup keyboard-configuration debconf-utils parted build-essential manpages-dev bash-completion gdb pkg-config python-is-python3 python3-rpi.gpio v4l-utils @@ -31,3 +31,4 @@ pciutils rpi-eeprom raspinfo udisks2 +unzip zip p7zip-full From 231487d9d602771b081e8fd819732dd22f5bac3a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 13 Jan 2022 15:39:22 +0000 Subject: [PATCH 056/222] Use trusted.gpg.d instead of apt-key --- stage0/00-configure-apt/00-run.sh | 2 +- stage0/00-configure-apt/01-packages | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 stage0/00-configure-apt/01-packages diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index d10a49f174..4d0ac92121 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -12,7 +12,7 @@ else rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache" fi -on_chroot apt-key add - < files/raspberrypi.gpg.key +cat files/raspberrypi.gpg.key | gpg --dearmor > "${ROOTFS_DIR}/etc/apt/trusted.gpg.d/raspberrypi-archive-stable.gpg" on_chroot << EOF apt-get update apt-get dist-upgrade -y diff --git a/stage0/00-configure-apt/01-packages b/stage0/00-configure-apt/01-packages new file mode 100644 index 0000000000..f5e37894ed --- /dev/null +++ b/stage0/00-configure-apt/01-packages @@ -0,0 +1 @@ +raspberrypi-archive-keyring From 6d2d36e7f2ce37abe77b58cf883dbd0ed124e8b7 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 14 Jan 2022 09:11:46 +0000 Subject: [PATCH 057/222] Don't try to use qemu-debootstrap Fixes #563 --- scripts/common | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/common b/scripts/common index 5b0df12beb..af4c1f3df2 100644 --- a/scripts/common +++ b/scripts/common @@ -9,10 +9,6 @@ bootstrap(){ export http_proxy=${APT_PROXY} - if [ "$(dpkg --print-architecture)" != "armhf" ] && [ "$(dpkg --print-architecture)" != "aarch64" ]; then - BOOTSTRAP_CMD=qemu-debootstrap - fi - BOOTSTRAP_ARGS+=(--arch armhf) BOOTSTRAP_ARGS+=(--components "main,contrib,non-free") BOOTSTRAP_ARGS+=(--keyring "${STAGE_DIR}/files/raspberrypi.gpg") From 40f67ce4bac65cb8f92a4a9ac72b05bc36ccedeb Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 14 Jan 2022 09:12:38 +0000 Subject: [PATCH 058/222] Install ca-certificates during bootstrap Fixes #424 --- scripts/common | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/common b/scripts/common index af4c1f3df2..e476f0ffa6 100644 --- a/scripts/common +++ b/scripts/common @@ -13,6 +13,7 @@ bootstrap(){ BOOTSTRAP_ARGS+=(--components "main,contrib,non-free") BOOTSTRAP_ARGS+=(--keyring "${STAGE_DIR}/files/raspberrypi.gpg") BOOTSTRAP_ARGS+=(--exclude=info) + BOOTSTRAP_ARGS+=(--include=ca-certificates) BOOTSTRAP_ARGS+=("$@") printf -v BOOTSTRAP_STR '%q ' "${BOOTSTRAP_ARGS[@]}" From f01430c9d8f67a4b9719cc00e74a2079d3834d5d Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 20 Jan 2022 15:31:16 +0000 Subject: [PATCH 059/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 88d9000e15..039cb226dd 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -34,8 +34,8 @@ UNRELEASED: * mkvtoolnix added to lite image * 7z and zip support added to lite image * gnome-keyring added to desktop images - * Raspberry Pi firmware cfdbadea5f74c16b7ed5d3b4866092a054e3c3bf - * Linux kernel 5.10.89 + * Raspberry Pi firmware c6d56567ff6ef17fd85159770f22abcf2c5953ed + * Linux kernel 5.10.92 2021-10-30: * Based on Debian version 11 (bullseye) * Desktop components (lxpanel and all plugins, libfm, pcmanfm) now built against GTK+3 From 0b4874f75ae0a7907b334358403c87fd5750fe54 Mon Sep 17 00:00:00 2001 From: simonliedl <simonliedl@users.noreply.github.com> Date: Wed, 19 Jan 2022 16:16:51 +0100 Subject: [PATCH 060/222] Fix docker build with debian bullseye image - Fix missing sfdisk which is not included anymore in bullseye container - Fix problem with binfmt_misc not being mounted inside docker although present on host --- Dockerfile | 2 +- build-docker.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e7fc561b79..479e708514 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get -y update && \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ - binfmt-support ca-certificates qemu-utils kpartx \ + binfmt-support ca-certificates qemu-utils kpartx fdisk \ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ diff --git a/build-docker.sh b/build-docker.sh index 85200e4c1a..2dc55c164f 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -98,6 +98,8 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ pi-gen \ bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && + # binfmt_misc is sometimes not mounted with debian bullseye image + (mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) && cd /pi-gen; ./build.sh ${BUILD_OPTS} && rsync -av work/*/build.log deploy/" & wait "$!" @@ -112,6 +114,8 @@ else -e "GIT_HASH=${GIT_HASH}" \ pi-gen \ bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && + # binfmt_misc is sometimes not mounted with debian bullseye image + (mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) && cd /pi-gen; ./build.sh ${BUILD_OPTS} && rsync -av work/*/build.log deploy/" & wait "$!" From c393ef105eae82448d31194c28f54b71e1f83da7 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 2 Feb 2022 14:01:30 +0000 Subject: [PATCH 061/222] Strip trailing spaces --- README.md | 12 ++++++------ build-docker.sh | 2 +- build.sh | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 08b2c13f29..e1676bb1fc 100644 --- a/README.md +++ b/README.md @@ -328,7 +328,7 @@ maintenance and allows for more easy customization. - **Stage 5** - The Raspbian Full image. More development tools, an email client, learning tools like Scratch, specialized packages - like sonic-pi, office productivity, etc. + like sonic-pi, office productivity, etc. ### Stage specification @@ -380,8 +380,8 @@ Example: ```bash root@build-machine:~/$ lsblk | grep nbd -nbd1 43:32 0 10G 0 disk -├─nbd1p1 43:33 0 10G 0 part +nbd1 43:32 0 10G 0 disk +├─nbd1p1 43:33 0 10G 0 part └─nbd1p1 253:0 0 10G 0 part root@build-machine:~/$ ps xa | grep qemu-nbd @@ -405,7 +405,7 @@ It can happen, that your build stops in case of an error. Normally `./build.sh` A typical message indicating that there are some orphaned device mapper entries is this: ``` -Failed to set NBD socket +Failed to set NBD socket Disconnect client, due to: Unexpected end-of-file before all bytes were read ``` @@ -428,10 +428,10 @@ If that happens go through the following steps: or sudo ./imagetool.sh --cleanup ``` - + Note: The `imagetool.sh` command will cleanup any /dev/nbdX that is not connected to a running `qemu-nbd` daemon. Be careful if you use network block devices for other tasks utilizing NBDs on your build machine as well. -Now you should be able to start a new build without running into troubles again. Most of the time, especially when using Docker build, you will only need no. 3 to get everything up and running again. +Now you should be able to start a new build without running into troubles again. Most of the time, especially when using Docker build, you will only need no. 3 to get everything up and running again. # Troubleshooting diff --git a/build-docker.sh b/build-docker.sh index 2dc55c164f..a7f18f136f 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -48,7 +48,7 @@ fi CONTAINER_NAME=${CONTAINER_NAME:-pigen_work} CONTINUE=${CONTINUE:-0} PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-0} -PIGEN_DOCKER_OPTS=${PIGEN_DOCKER_OPTS:-""} +PIGEN_DOCKER_OPTS=${PIGEN_DOCKER_OPTS:-""} if [ -z "${IMG_NAME}" ]; then echo "IMG_NAME not set in 'config'" 1>&2 diff --git a/build.sh b/build.sh index bcfdeb69ff..6f6a0d23c4 100755 --- a/build.sh +++ b/build.sh @@ -133,7 +133,7 @@ run_stage(){ done fi - if [ "${USE_QCOW2}" = "1" ]; then + if [ "${USE_QCOW2}" = "1" ]; then unload_qimage else # make sure we are not umounting during export-image stage @@ -369,7 +369,7 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do else run_stage - fi + fi if [ "${USE_QEMU}" != "1" ]; then if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then # shellcheck source=/dev/null From 8862e4d9ff15b44daa8563872fc928a3317e33dc Mon Sep 17 00:00:00 2001 From: Romain Bazile <romain@bazile.io> Date: Wed, 2 Feb 2022 14:10:24 +0100 Subject: [PATCH 062/222] Add check and warning for a space presence in the base path --- README.md | 18 ++++++++++++++++++ build.sh | 8 ++++++++ 2 files changed, 26 insertions(+) diff --git a/README.md b/README.md index e1676bb1fc..893a5cc225 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,24 @@ qemu-utils kpartx The file `depends` contains a list of tools needed. The format of this package is `<tool>[:<debian-package>]`. +## Getting started with building your images + +Getting started is as simple as cloning this repository on your build machine. You +can do so with: + +```bash +git clone -–depth 1 https://github.com/RPI-Distro/pi-gen.git +``` + +Using `--depth 1` with `git clone` will create a shallow clone, only containing +the latest revision of the repository. Do not do this on your development machine. + +Also, be careful to clone the repository to a base path **NOT** containing spaces. +This configuration is not supported by debootstrap and will lead to `pi-gen` not +running. + +After cloning the repository, you can move to the next step and start configuring +your build. ## Config diff --git a/build.sh b/build.sh index 6f6a0d23c4..f743e2a851 100755 --- a/build.sh +++ b/build.sh @@ -155,6 +155,14 @@ if [ "$(id -u)" != "0" ]; then fi BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [[ $BASE_DIR = *" "* ]]; then + echo "There is a space in the base path of pi-gen" + echo "This is not a valid setup supported by debootstrap." + echo "Please remove the spaces, or move pi-gen directory to a base path without spaces" 1>&2 + exit 1 +fi + export BASE_DIR if [ -f config ]; then From 724820051c400f9554d7fa311e7fc378bf598282 Mon Sep 17 00:00:00 2001 From: bwisn <bwisn_dev@outlook.com> Date: Tue, 1 Feb 2022 14:42:00 +0100 Subject: [PATCH 063/222] Add gpg dependency --- Dockerfile | 2 +- README.md | 2 +- depends | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 479e708514..69fd86c7db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get -y update && \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ - binfmt-support ca-certificates qemu-utils kpartx fdisk \ + binfmt-support ca-certificates qemu-utils kpartx fdisk gpg \ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ diff --git a/README.md b/README.md index 893a5cc225..cda02557d5 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ To install the required dependencies for `pi-gen` you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \ -qemu-utils kpartx +qemu-utils kpartx gpg ``` The file `depends` contains a list of tools needed. The format of this diff --git a/depends b/depends index eeb24902d9..654531af85 100644 --- a/depends +++ b/depends @@ -19,3 +19,4 @@ lsmod:kmod bc qemu-nbd:qemu-utils kpartx +gpg From b3f6852c3efed9805a3e8a5eeb3454f0fd1a397d Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 3 Feb 2022 15:50:48 +0000 Subject: [PATCH 064/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 039cb226dd..c580c2d643 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,7 @@ UNRELEASED: + * +2022-01-28: + * Policykit CVE-2021-4034 fix * rc_gui - add combo box to allow resolution to be set for VNC connections * rc_gui - camera interface switch removed * lxpanel - remove appearance settings from preferences dialog; instead add menu option to open general Appearance Settings application @@ -14,7 +17,7 @@ UNRELEASED: * pipanel - add command-line option to open on arbitrary tab * lxplug-network - suppress ’scan received’ logging message * raspberrypi-ui-mods - set hover colour for taskbar items based on taskbar colour, not system highlight colour - * Legacy camera applications and libraries reinstalled + * Legacy camera applications and libraries reinstalled (32-bit only) * Bug fix - lxinput - lxsession config file not being written on first attempt * Bug fix - lxinput - set timer for file write to prevent slider slowing down * Bug fix - lxinput - write values to gsettings as well as xinput and xsettings to take effect within mutter From c63a3aedf95988b25156794894e512d05b3dc92d Mon Sep 17 00:00:00 2001 From: jannik <jannik.schleicher@stud.h-da.de> Date: Wed, 16 Feb 2022 11:01:14 +0100 Subject: [PATCH 065/222] fix dash in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cda02557d5..c19e02e176 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Getting started is as simple as cloning this repository on your build machine. Y can do so with: ```bash -git clone -–depth 1 https://github.com/RPI-Distro/pi-gen.git +git clone --depth 1 https://github.com/RPI-Distro/pi-gen.git ``` Using `--depth 1` with `git clone` will create a shallow clone, only containing From b1d54c82fec27414ee5200bdc2069a56ede8a1af Mon Sep 17 00:00:00 2001 From: Romain Bazile <romain@bazile.io> Date: Tue, 22 Feb 2022 11:30:46 +0100 Subject: [PATCH 066/222] Provide information to build the 64 bit image in the README (#596) Fixes #595 --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c19e02e176..beba5c20ac 100644 --- a/README.md +++ b/README.md @@ -454,7 +454,22 @@ Now you should be able to start a new build without running into troubles again. # Troubleshooting ## `64 Bit Systems` -Please note there is currently an issue when compiling with a 64 Bit OS. See https://github.com/RPi-Distro/pi-gen/issues/271 +Please note there is currently an issue when compiling with a 64 Bit OS. See +https://github.com/RPi-Distro/pi-gen/issues/271 + +A 64 bit image can be generated from the `arm64` branch in this repository. Just +replace the command from [this section](#getting-started-with-building-your-images) +by the one below, and follow the rest of the documentation: +```bash +git clone --depth 1 --branch arm64 https://github.com/RPI-Distro/pi-gen.git +``` + +If you want to generate a 64 bits image from a Raspberry Pi running a 32 bits +version, you need to add `arm_64bit=1` to your `config.txt` file and reboot your +machine. This will restart your machine with a 64 bits kernel. This will only +work from a Raspberry Pi with a 64-bit capable processor (i.e. Raspberry Pi Zero +2, Raspberry Pi 3 or Raspberry Pi 4). + ## `binfmt_misc` From b2bd87c512e7b73bb00b55bf11f914d8b44c4f7b Mon Sep 17 00:00:00 2001 From: jannik <jannik.schleicher@stud.h-da.de> Date: Tue, 22 Feb 2022 19:01:25 +0100 Subject: [PATCH 067/222] Add more binfmt errors and hints to README --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index beba5c20ac..44d8413a66 100644 --- a/README.md +++ b/README.md @@ -478,11 +478,16 @@ possible to make use of `pi-gen` on an x86_64 system, even though it will be run ARM binaries. This requires support from the [`binfmt_misc`](https://en.wikipedia.org/wiki/Binfmt_misc) kernel module. -You may see the following error: +You may see one of the following errors: ``` update-binfmts: warning: Couldn't load the binfmt_misc module. ``` +``` +W: Failure trying to run: chroot "/pi-gen/work/test/stage0/rootfs" /bin/true +and/or +chroot: failed to run command '/bin/true': Exec format error +``` To resolve this, ensure that the following files are available (install them if necessary): @@ -492,3 +497,5 @@ To resolve this, ensure that the following files are available (install them if ``` You may also need to load the module by hand - run `modprobe binfmt_misc`. + +If you are using WSL to build you may have to enable the service `sudo update-binfmts --enable` \ No newline at end of file From 1ad53be314dbc73d7117dbd610a982c90e115f37 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 23 Feb 2022 18:32:32 +0000 Subject: [PATCH 068/222] Add flush option to boot partition https://github.com/raspberrypi/linux/issues/4844 --- stage1/01-sys-tweaks/files/fstab | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage1/01-sys-tweaks/files/fstab b/stage1/01-sys-tweaks/files/fstab index f16e3fb8aa..89f7bcb9c7 100644 --- a/stage1/01-sys-tweaks/files/fstab +++ b/stage1/01-sys-tweaks/files/fstab @@ -1,3 +1,3 @@ proc /proc proc defaults 0 0 -BOOTDEV /boot vfat defaults 0 2 +BOOTDEV /boot vfat defaults.flush 0 2 ROOTDEV / ext4 defaults,noatime 0 1 From c5a906b676647c2334ddb595435ebbbe869bff15 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 23 Feb 2022 18:36:02 +0000 Subject: [PATCH 069/222] Fix whitespace --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 44d8413a66..a8431fad27 100644 --- a/README.md +++ b/README.md @@ -498,4 +498,4 @@ To resolve this, ensure that the following files are available (install them if You may also need to load the module by hand - run `modprobe binfmt_misc`. -If you are using WSL to build you may have to enable the service `sudo update-binfmts --enable` \ No newline at end of file +If you are using WSL to build you may have to enable the service `sudo update-binfmts --enable` From a120c4128ece237754af9e8bef6679d6a51f73ec Mon Sep 17 00:00:00 2001 From: Phil Elwell <8911409+pelwell@users.noreply.github.com> Date: Mon, 28 Feb 2022 09:58:08 +0000 Subject: [PATCH 070/222] Fix fstab flush option raspberrypi/linux#4844 --- stage1/01-sys-tweaks/files/fstab | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage1/01-sys-tweaks/files/fstab b/stage1/01-sys-tweaks/files/fstab index 89f7bcb9c7..2159c1ca67 100644 --- a/stage1/01-sys-tweaks/files/fstab +++ b/stage1/01-sys-tweaks/files/fstab @@ -1,3 +1,3 @@ proc /proc proc defaults 0 0 -BOOTDEV /boot vfat defaults.flush 0 2 +BOOTDEV /boot vfat defaults,flush 0 2 ROOTDEV / ext4 defaults,noatime 0 1 From 5d044ba71b7eba33860fdbf222a9a29ae42691e9 Mon Sep 17 00:00:00 2001 From: Romain Bazile <romain@bazile.io> Date: Tue, 1 Mar 2022 17:23:42 +0100 Subject: [PATCH 071/222] Addition of a `DEPLOY_COMPRESSION` option to the configuration (#592) --- Dockerfile | 2 +- README.md | 25 ++++++++++++++++++++++--- build.sh | 14 ++++++++++++-- depends | 1 + export-image/04-finalise/01-run.sh | 25 ++++++++++++++++++------- export-noobs/prerun.sh | 6 +----- 6 files changed, 55 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 69fd86c7db..2a5d8fe5f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get -y update && \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ - binfmt-support ca-certificates qemu-utils kpartx fdisk gpg \ + binfmt-support ca-certificates qemu-utils kpartx fdisk gpg pigz\ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ diff --git a/README.md b/README.md index a8431fad27..3cab6ec2bd 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ To install the required dependencies for `pi-gen` you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \ -qemu-utils kpartx gpg +qemu-utils kpartx gpg pigz ``` The file `depends` contains a list of tools needed. The format of this @@ -116,9 +116,28 @@ The following environment variables are supported: Output directory for target system images and NOOBS bundles. - * `DEPLOY_ZIP` (Default: `1`) + * `DEPLOY_COMPRESSION` (Default: `zip`) - Setting to `0` will deploy the actual image (`.img`) instead of a zipped image (`.zip`). + Set to: + * `none` to deploy the actual image (`.img`). + * `zip` to deploy a zipped image (`.zip`). + * `gz` to deploy a gzipped image (`.img.gz`). + * `xz` to deploy a xzipped image (`.img.xz`). + + + * `DEPLOY_ZIP` (Deprecated) + + This option has been deprecated in favor of `DEPLOY_COMPRESSION`. + + If `DEPLOY_ZIP=0` is still present in your config file, the behavior is the + same as with `DEPLOY_COMPRESSION=none`. + + * `COMPRESSION_LEVEL` (Default: `6`) + + Compression level to be used when using `zip`, `gz` or `xz` for + `DEPLOY_COMPRESSION`. From 0 to 9 (refer to the tool man page for more + information on this. Usually 0 is no compression but very fast, up to 9 with + the best compression but very slow ). * `USE_QEMU` (Default: `"0"`) diff --git a/build.sh b/build.sh index f743e2a851..4b75e79232 100755 --- a/build.sh +++ b/build.sh @@ -203,12 +203,22 @@ fi export USE_QEMU="${USE_QEMU:-0}" export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}" export IMG_FILENAME="${IMG_FILENAME:-"${IMG_DATE}-${IMG_NAME}"}" -export ZIP_FILENAME="${ZIP_FILENAME:-"image_${IMG_DATE}-${IMG_NAME}"}" +export ARCHIVE_FILENAME="${ARCHIVE_FILENAME:-"image_${IMG_DATE}-${IMG_NAME}"}" export SCRIPT_DIR="${BASE_DIR}/scripts" export WORK_DIR="${WORK_DIR:-"${BASE_DIR}/work/${IMG_NAME}"}" export DEPLOY_DIR=${DEPLOY_DIR:-"${BASE_DIR}/deploy"} -export DEPLOY_ZIP="${DEPLOY_ZIP:-1}" + +# DEPLOY_ZIP was deprecated in favor of DEPLOY_COMPRESSION +# This preserve the old behavior with DEPLOY_ZIP=0 where no archive was created +if [ -z "${DEPLOY_COMPRESSION}" ] && [ "${DEPLOY_ZIP:-1}" = "0" ]; then + echo "DEPLOY_ZIP has been deprecated in favor of DEPLOY_COMPRESSION" + echo "Similar behavior to DEPLOY_ZIP=0 can be obtained with DEPLOY_COMPRESSION=none" + echo "Please update your config file" + DEPLOY_COMPRESSION=none +fi +export DEPLOY_COMPRESSION=${DEPLOY_COMPRESSION:-zip} +export COMPRESSION_LEVEL=${COMPRESSION_LEVEL:-6} export LOG_FILE="${WORK_DIR}/build.log" export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi} diff --git a/depends b/depends index 654531af85..db881712eb 100644 --- a/depends +++ b/depends @@ -20,3 +20,4 @@ bc qemu-nbd:qemu-utils kpartx gpg +pigz diff --git a/export-image/04-finalise/01-run.sh b/export-image/04-finalise/01-run.sh index d7ace58d85..c104366f6e 100755 --- a/export-image/04-finalise/01-run.sh +++ b/export-image/04-finalise/01-run.sh @@ -78,7 +78,7 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" mkdir -p "${DEPLOY_DIR}" -rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" +rm -f "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.*" rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" mv "$INFO_FILE" "$DEPLOY_DIR/" @@ -95,11 +95,22 @@ else make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE" fi -if [ "${DEPLOY_ZIP}" == "1" ]; then +case "${DEPLOY_COMPRESSION}" in +zip) pushd "${STAGE_WORK_DIR}" > /dev/null - zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \ - "$(basename "${IMG_FILE}")" + zip -"${COMPRESSION_LEVEL}" \ + "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.zip" "$(basename "${IMG_FILE}")" popd > /dev/null -else - mv "$IMG_FILE" "$DEPLOY_DIR/" -fi + ;; +gz) + pigz --force -"${COMPRESSION_LEVEL}" "$IMG_FILE" --stdout > \ + "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.img.gz" + ;; +xz) + xz --compress --force --threads 0 --memlimit-compress=50% -"${COMPRESSION_LEVEL}" \ + --stdout "$IMG_FILE" > "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.img.xz" + ;; +none | *) + cp "$IMG_FILE" "$DEPLOY_DIR/" +;; +esac diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index d97508a658..97dd09490e 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -3,11 +3,7 @@ NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_NAME}${IMG_SUFFIX}" mkdir -p "${STAGE_WORK_DIR}" -if [ "${DEPLOY_ZIP}" == "1" ]; then - IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" -else - IMG_FILE="${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" -fi +IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" unmount_image "${IMG_FILE}" From 95ac3cfb3b223dad058c1aee1d8b6e3f5e5b5935 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 18 Mar 2022 15:03:23 +0000 Subject: [PATCH 072/222] Remove default password --- README.md | 4 ++-- build.sh | 2 +- stage1/01-sys-tweaks/00-run.sh | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3cab6ec2bd..2ef35e8b1b 100644 --- a/README.md +++ b/README.md @@ -179,9 +179,9 @@ The following environment variables are supported: Username for the first user - * `FIRST_USER_PASS` (Default: "raspberry") + * `FIRST_USER_PASS` (Default: unset) - Password for the first user + Password for the first user. If unset, the account is locked. * `WPA_ESSID`, `WPA_PASSWORD` and `WPA_COUNTRY` (Default: unset) diff --git a/build.sh b/build.sh index 4b75e79232..014035c7b0 100755 --- a/build.sh +++ b/build.sh @@ -224,7 +224,7 @@ export LOG_FILE="${WORK_DIR}/build.log" export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi} export FIRST_USER_NAME=${FIRST_USER_NAME:-pi} -export FIRST_USER_PASS=${FIRST_USER_PASS:-raspberry} +export FIRST_USER_PASS export RELEASE=${RELEASE:-bullseye} export WPA_ESSID export WPA_PASSWORD diff --git a/stage1/01-sys-tweaks/00-run.sh b/stage1/01-sys-tweaks/00-run.sh index 9c380ec558..0f761bd53e 100755 --- a/stage1/01-sys-tweaks/00-run.sh +++ b/stage1/01-sys-tweaks/00-run.sh @@ -8,7 +8,10 @@ on_chroot << EOF if ! id -u ${FIRST_USER_NAME} >/dev/null 2>&1; then adduser --disabled-password --gecos "" ${FIRST_USER_NAME} fi -echo "${FIRST_USER_NAME}:${FIRST_USER_PASS}" | chpasswd + +if [ -n "${FIRST_USER_PASS}" ]; then + echo "${FIRST_USER_NAME}:${FIRST_USER_PASS}" | chpasswd +fi echo "root:root" | chpasswd EOF From b0316ae8706e3abe0d4bd2bdf1cda976a6e2e6a7 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 18 Mar 2022 17:03:26 +0000 Subject: [PATCH 073/222] Enable userconf-pi on first boot --- export-image/01-user-rename/00-packages | 1 + export-image/01-user-rename/01-run.sh | 5 +++++ export-image/{01-set-sources => 02-set-sources}/01-run.sh | 0 export-image/{02-network => 03-network}/01-run.sh | 0 export-image/{02-network => 03-network}/files/resolv.conf | 0 export-image/{03-set-partuuid => 04-set-partuuid}/00-run.sh | 0 export-image/{04-finalise => 05-finalise}/01-run.sh | 0 7 files changed, 6 insertions(+) create mode 100644 export-image/01-user-rename/00-packages create mode 100755 export-image/01-user-rename/01-run.sh rename export-image/{01-set-sources => 02-set-sources}/01-run.sh (100%) rename export-image/{02-network => 03-network}/01-run.sh (100%) rename export-image/{02-network => 03-network}/files/resolv.conf (100%) rename export-image/{03-set-partuuid => 04-set-partuuid}/00-run.sh (100%) rename export-image/{04-finalise => 05-finalise}/01-run.sh (100%) diff --git a/export-image/01-user-rename/00-packages b/export-image/01-user-rename/00-packages new file mode 100644 index 0000000000..e8b69fc4e2 --- /dev/null +++ b/export-image/01-user-rename/00-packages @@ -0,0 +1 @@ +userconf-pi diff --git a/export-image/01-user-rename/01-run.sh b/export-image/01-user-rename/01-run.sh new file mode 100755 index 0000000000..8a5617bb1f --- /dev/null +++ b/export-image/01-user-rename/01-run.sh @@ -0,0 +1,5 @@ +#!/bin/bash -e + +on_chroot << EOF + SUDO_USER="${FIRST_USER_NAME}" rename-user -f -s +EOF diff --git a/export-image/01-set-sources/01-run.sh b/export-image/02-set-sources/01-run.sh similarity index 100% rename from export-image/01-set-sources/01-run.sh rename to export-image/02-set-sources/01-run.sh diff --git a/export-image/02-network/01-run.sh b/export-image/03-network/01-run.sh similarity index 100% rename from export-image/02-network/01-run.sh rename to export-image/03-network/01-run.sh diff --git a/export-image/02-network/files/resolv.conf b/export-image/03-network/files/resolv.conf similarity index 100% rename from export-image/02-network/files/resolv.conf rename to export-image/03-network/files/resolv.conf diff --git a/export-image/03-set-partuuid/00-run.sh b/export-image/04-set-partuuid/00-run.sh similarity index 100% rename from export-image/03-set-partuuid/00-run.sh rename to export-image/04-set-partuuid/00-run.sh diff --git a/export-image/04-finalise/01-run.sh b/export-image/05-finalise/01-run.sh similarity index 100% rename from export-image/04-finalise/01-run.sh rename to export-image/05-finalise/01-run.sh From 09ebc77e6b6dac3892ab09242fe39eec892abe93 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 21 Mar 2022 11:48:11 +0000 Subject: [PATCH 074/222] stage4: Install rpi-imager --- stage4/00-install-packages/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index e76d4110d0..4e9519fb10 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -23,3 +23,4 @@ piwiz rp-prefapps ffmpeg vlc +rpi-imager From 1b635daf30b70dd43a3b6ba15da222a35b9b1666 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 21 Mar 2022 11:48:20 +0000 Subject: [PATCH 075/222] stage4: Install rpi-wayland --- stage4/00-install-packages/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index 4e9519fb10..7854596a1f 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -24,3 +24,4 @@ rp-prefapps ffmpeg vlc rpi-imager +rpi-wayland From f1f6722ba445ba07d6929b9b4a1f6dbb811cd03f Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 28 Mar 2022 19:44:24 +0100 Subject: [PATCH 076/222] Update release notes --- .../00-release/files/release_notes.txt | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index c580c2d643..ccfe5af539 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,28 @@ UNRELEASED: - * + * Default “pi” user has been removed; the first-boot wizard enforces the creation of a new user account + * rename-user script added to allow existing users to be renamed by temporarily rebooting to cut-down first-boot wizard + * Overscan now managed by xrandr under KMS, can be set independently for both monitors, and takes effect on the fly rather than requiring reboot + * piwiz - first-boot wizard now runs in a separate session as a different user with different graphics + * piwiz - first-boot wizard now has automatic pairing for discoverable Bluetooth mice and keyboards + * lxinput - keyboard delay and repeat settings now persist across reboots under mutter + * raspi-config / rc_gui - removed pixel doubling option when KMS driver in use + * raspi-config - removed composition manager option when legacy driver in use + * arandr - restored support for interlaced displays + * mutter - implemented more intuitive window and application cycling behaviour + * pi-greeter - rebuilt for GTK3 + * Bug fix - graphical corruption in system tray icons + * Bug fix - desktop items vanishing when dragged + * Bug fix - terminal windows not focussed correctly when launched + * Bug fix - crash after multiple update checks in updater plugin + * Bug fix - Raspberry Pi keyboard auto-detect by wizard was broken in previous release + * Bug fix - spurious “connected” dialog box shown when reconnecting to Bluetooth LE devices on boot + * Support for experimental Wayland backend added - can be enabled as an advanced option in raspi-config + * Various small bug fixes and graphical tweaks + * Chromium upgraded to version 98.0.4758.106 + * FFmpeg HW acceleration improved + * OpenJDK 17 now defaults to 'client' JVM for ARMv6 compatibility + * Raspberry Pi firmware 69277bc713133a54a1d20554d79544da1ae2b6ca + * Linux kernel 5.15.30 2022-01-28: * Policykit CVE-2021-4034 fix * rc_gui - add combo box to allow resolution to be set for VNC connections From f651a43337fd203a4a29f1898e8814324eb2545d Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 29 Mar 2022 10:59:14 +0100 Subject: [PATCH 077/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index ccfe5af539..5a0696e5ca 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -2,6 +2,7 @@ UNRELEASED: * Default “pi” user has been removed; the first-boot wizard enforces the creation of a new user account * rename-user script added to allow existing users to be renamed by temporarily rebooting to cut-down first-boot wizard * Overscan now managed by xrandr under KMS, can be set independently for both monitors, and takes effect on the fly rather than requiring reboot + * GTK3 switch control now used in place of paired radio buttons throughout * piwiz - first-boot wizard now runs in a separate session as a different user with different graphics * piwiz - first-boot wizard now has automatic pairing for discoverable Bluetooth mice and keyboards * lxinput - keyboard delay and repeat settings now persist across reboots under mutter From 226b479f8d32919c9fe36dd5b4c20c02682f8180 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 4 Apr 2022 12:40:32 +0100 Subject: [PATCH 078/222] stage1: install systemd-timesyncd systemd no longer depends on systemd-timesyncd, so we need to include it here --- stage1/03-install-packages/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage1/03-install-packages/00-packages b/stage1/03-install-packages/00-packages index 73edcc901c..a3d2fb66fd 100644 --- a/stage1/03-install-packages/00-packages +++ b/stage1/03-install-packages/00-packages @@ -1 +1,2 @@ libraspberrypi-bin libraspberrypi0 +systemd-timesyncd From 3385618efbd428bc14a13b7195d44978689d343a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 8 Apr 2022 15:26:37 +0100 Subject: [PATCH 079/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 5a0696e5ca..331d8d9a6b 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,7 @@ UNRELEASED: - * Default “pi” user has been removed; the first-boot wizard enforces the creation of a new user account + * +2022-04-04: + * Default "pi” user has been removed; the first-boot wizard enforces the creation of a new user account * rename-user script added to allow existing users to be renamed by temporarily rebooting to cut-down first-boot wizard * Overscan now managed by xrandr under KMS, can be set independently for both monitors, and takes effect on the fly rather than requiring reboot * GTK3 switch control now used in place of paired radio buttons throughout @@ -16,7 +18,7 @@ UNRELEASED: * Bug fix - terminal windows not focussed correctly when launched * Bug fix - crash after multiple update checks in updater plugin * Bug fix - Raspberry Pi keyboard auto-detect by wizard was broken in previous release - * Bug fix - spurious “connected” dialog box shown when reconnecting to Bluetooth LE devices on boot + * Bug fix - spurious "connected” dialog box shown when reconnecting to Bluetooth LE devices on boot * Support for experimental Wayland backend added - can be enabled as an advanced option in raspi-config * Various small bug fixes and graphical tweaks * Chromium upgraded to version 98.0.4758.106 From 01b2432007766a6a1acc942f62d4ece7b25e560d Mon Sep 17 00:00:00 2001 From: Romain Bazile <romain@bazile.io> Date: Fri, 17 Jun 2022 16:45:08 +0200 Subject: [PATCH 080/222] Create a `DISABLE_FIRST_BOOT_USER_RENAME` flag to be set in config (#618) Closes #614 --- README.md | 13 +++++++++++-- build.sh | 12 ++++++++++++ export-image/01-user-rename/01-run.sh | 8 +++++--- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2ef35e8b1b..a86f95155b 100644 --- a/README.md +++ b/README.md @@ -175,14 +175,23 @@ The following environment variables are supported: To get the current value from a running system, look in `/etc/timezone`. - * `FIRST_USER_NAME` (Default: "pi" ) + * `FIRST_USER_NAME` (Default: `pi`) - Username for the first user + Username for the first user. This user only exists during the image creation process. Unless + `DISABLE_FIRST_BOOT_USER_RENAME` is set to `1`, this user will be renamed on the first boot with + a name chosen by the final user. This security feature is designed to prevent shipping images + with a default username and help prevent malicious actors from taking over your devices. * `FIRST_USER_PASS` (Default: unset) Password for the first user. If unset, the account is locked. + * `DISABLE_FIRST_BOOT_USER_RENAME` (Default: `0`) + + Disable the renaming of the first user during the first boot. This make it so `FIRST_USER_NAME` + stays activated. `FIRST_USER_PASS` must be set for this to work. Please be aware of the implied + security risk of defining a default username and password for your devices. + * `WPA_ESSID`, `WPA_PASSWORD` and `WPA_COUNTRY` (Default: unset) If these are set, they are use to configure `wpa_supplicant.conf`, so that the Raspberry Pi can automatically connect to a wireless network on first boot. If `WPA_ESSID` is set and `WPA_PASSWORD` is unset an unprotected wireless network will be configured. If set, `WPA_PASSWORD` must be between 8 and 63 characters. diff --git a/build.sh b/build.sh index 014035c7b0..135da35b90 100755 --- a/build.sh +++ b/build.sh @@ -225,6 +225,7 @@ export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi} export FIRST_USER_NAME=${FIRST_USER_NAME:-pi} export FIRST_USER_PASS +export DISABLE_FIRST_BOOT_USER_RENAME=${DISABLE_FIRST_BOOT_USER_RENAME:-0} export RELEASE=${RELEASE:-bullseye} export WPA_ESSID export WPA_PASSWORD @@ -290,6 +291,17 @@ if [[ ! "$FIRST_USER_NAME" =~ ^[a-z][-a-z0-9_]*$ ]]; then exit 1 fi +if [[ "$DISABLE_FIRST_BOOT_USER_RENAME" == "1" ]] && [ -z "${FIRST_USER_PASS}" ]; then + echo "To disable user rename on first boot, FIRST_USER_PASS needs to be set" + echo "Not setting FIRST_USER_PASS makes your system vulnerable and open to cyberattacks" + exit 1 +fi + +if [[ "$DISABLE_FIRST_BOOT_USER_RENAME" == "1" ]]; then + echo "User rename on the first boot is disabled" + echo "Be advised of the security risks linked to shipping a device with default username/password set." +fi + if [[ -n "${APT_PROXY}" ]] && ! curl --silent "${APT_PROXY}" >/dev/null ; then echo "Could not reach APT_PROXY server: ${APT_PROXY}" exit 1 diff --git a/export-image/01-user-rename/01-run.sh b/export-image/01-user-rename/01-run.sh index 8a5617bb1f..f3f900f793 100755 --- a/export-image/01-user-rename/01-run.sh +++ b/export-image/01-user-rename/01-run.sh @@ -1,5 +1,7 @@ #!/bin/bash -e -on_chroot << EOF - SUDO_USER="${FIRST_USER_NAME}" rename-user -f -s -EOF +if [[ "${DISABLE_FIRST_BOOT_USER_RENAME}" == "0" ]]; then + on_chroot <<- EOF + SUDO_USER="${FIRST_USER_NAME}" rename-user -f -s + EOF +fi From eb823214ef1857dca91f48aed06b4abcf1b7f7df Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 20 Apr 2022 14:14:14 +0100 Subject: [PATCH 081/222] Prevent install of OpenJDK 11 OpenJDK 11 does not support ARMv6. Switching to OpenJDK 17 with client JVM. --- stage5/00-install-libreoffice/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage5/00-install-libreoffice/00-packages b/stage5/00-install-libreoffice/00-packages index 148e7b941f..dc1b8d770b 100644 --- a/stage5/00-install-libreoffice/00-packages +++ b/stage5/00-install-libreoffice/00-packages @@ -1,3 +1,3 @@ -libreoffice-pi +libreoffice-pi openjdk-11-jre- libreoffice-help-en-gb libreoffice-l10n-en-gb From d3105c68ab08f6f9017f6a8d479e4eb018692180 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 20 Apr 2022 15:16:10 +0100 Subject: [PATCH 082/222] Use unbundled BlueJ variant --- stage5/00-install-extras/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage5/00-install-extras/00-packages b/stage5/00-install-extras/00-packages index 1b9d4884d8..d839f2b98b 100644 --- a/stage5/00-install-extras/00-packages +++ b/stage5/00-install-extras/00-packages @@ -7,7 +7,7 @@ python3-sense-emu sense-emu-tools python-sense-emu-doc wolfram-engine claws-mail -greenfoot-unbundled bluej +greenfoot-unbundled bluej-unbundled nodered realvnc-vnc-viewer From 00da85177cdd6a1446381fb45fd5439c9bba860a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 4 May 2022 15:21:12 +0100 Subject: [PATCH 083/222] Adjust partition formatting and mount options Format the root partition with the metadata_csum option Format the boot partition with 4 sectors per cluster for a slight performance boost Remove 'flush' mount option from the boot partition --- export-image/prerun.sh | 4 ++-- export-noobs/00-release/files/partitions.json | 2 +- export-noobs/00-release/files/release_notes.txt | 4 +++- scripts/qcow2_handling | 4 ++-- stage1/01-sys-tweaks/files/fstab | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/export-image/prerun.sh b/export-image/prerun.sh index fad7f80e79..145d61eee0 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -70,12 +70,12 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" ROOT_FEATURES="^huge_file" - for FEATURE in metadata_csum 64bit; do + for FEATURE in 64bit; do if grep -q "$FEATURE" /etc/mke2fs.conf; then ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" fi done - mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null + mkdosfs -n boot -F 32 -s 4 -v "$BOOT_DEV" > /dev/null mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 diff --git a/export-noobs/00-release/files/partitions.json b/export-noobs/00-release/files/partitions.json index 3d060173a3..8ca103d545 100644 --- a/export-noobs/00-release/files/partitions.json +++ b/export-noobs/00-release/files/partitions.json @@ -3,7 +3,7 @@ { "filesystem_type": "FAT", "label": "boot", - "mkfs_options": "-F 32", + "mkfs_options": "-F 32 -s 4", "partition_size_nominal": BOOT_NOM, "uncompressed_tarball_size": BOOT_SIZE, "want_maximised": false, diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 331d8d9a6b..4240372d3a 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,7 @@ UNRELEASED: - * + * Format the root partition with the metadata_csum option + * Format the boot partition with 4 sectors per cluster for a slight performance boost + * Remove 'flush' mount option from the boot partition 2022-04-04: * Default "pi” user has been removed; the first-boot wizard enforces the creation of a new user account * rename-user script added to allow existing users to be renamed by temporarily rebooting to cut-down first-boot wizard diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling index 6b2a56bc4f..66708e710e 100644 --- a/scripts/qcow2_handling +++ b/scripts/qcow2_handling @@ -109,8 +109,8 @@ load_qimage() { EOF sync kpartx -as $NBD_DEV - mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV - mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV + mkdosfs -n boot -F 32 -s 4 -v $MAP_BOOT_DEV + mkfs.ext4 -L rootfs -O "^huge_file,^64bit" $MAP_ROOT_DEV sync else if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then diff --git a/stage1/01-sys-tweaks/files/fstab b/stage1/01-sys-tweaks/files/fstab index 2159c1ca67..f16e3fb8aa 100644 --- a/stage1/01-sys-tweaks/files/fstab +++ b/stage1/01-sys-tweaks/files/fstab @@ -1,3 +1,3 @@ proc /proc proc defaults 0 0 -BOOTDEV /boot vfat defaults,flush 0 2 +BOOTDEV /boot vfat defaults 0 2 ROOTDEV / ext4 defaults,noatime 0 1 From 288a0a615abbc5f1024d8ee351e5e251d92aa3e2 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 4 May 2022 15:24:01 +0100 Subject: [PATCH 084/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 4240372d3a..aff9383117 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,5 @@ UNRELEASED: + * Install OpenJDK 17 rather than OpenJDK 11 on full images * Format the root partition with the metadata_csum option * Format the boot partition with 4 sectors per cluster for a slight performance boost * Remove 'flush' mount option from the boot partition From 4de913c1fff28adde5681a5258978f6bfaa880b6 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 14 Jun 2022 10:47:34 +0100 Subject: [PATCH 085/222] Modify 010_pi-nopasswd to match FIRST_USER_NAME https://github.com/RPi-Distro/raspberrypi-sys-mods/issues/62#issuecomment-1148443217= --- stage2/01-sys-tweaks/01-run.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index bb3e11ac08..294a0ec36b 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -57,6 +57,10 @@ for GRP in adm dialout cdrom audio users sudo video games plugdev input gpio spi done EOF +if [ -f "${ROOTFS_DIR}/etc/sudoers.d/010_pi-nopasswd" ]; then + sed -i "s/^pi /$FIRST_USER_NAME /" "${ROOTFS_DIR}/etc/sudoers.d/010_pi-nopasswd" +fi + on_chroot << EOF setupcon --force --save-only -v EOF From db3cd433fb27a6b22744b3120655e4e80460d7dc Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 9 Aug 2022 12:18:13 +0100 Subject: [PATCH 086/222] stage2: Install NetworkManager --- export-noobs/00-release/files/release_notes.txt | 1 + stage2/02-net-tweaks/00-packages | 1 + 2 files changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index aff9383117..ed3ab10fb1 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,5 @@ UNRELEASED: + * Install NetworkManager * Install OpenJDK 17 rather than OpenJDK 11 on full images * Format the root partition with the metadata_csum option * Format the boot partition with 4 sectors per cluster for a slight performance boost diff --git a/stage2/02-net-tweaks/00-packages b/stage2/02-net-tweaks/00-packages index cc4a68e670..f0125f6a6d 100644 --- a/stage2/02-net-tweaks/00-packages +++ b/stage2/02-net-tweaks/00-packages @@ -1,4 +1,5 @@ wpasupplicant wireless-tools firmware-atheros firmware-brcm80211 firmware-libertas firmware-misc-nonfree firmware-realtek raspberrypi-net-mods dhcpcd5 +network-manager net-tools From 68ddb44960c671d76f70ce6e5a26ec281401dd23 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 19 Aug 2022 12:35:48 +0100 Subject: [PATCH 087/222] Use raspi-config to set WPA_COUNTRY --- stage2/02-net-tweaks/01-run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stage2/02-net-tweaks/01-run.sh b/stage2/02-net-tweaks/01-run.sh index 837e86135e..058547d339 100755 --- a/stage2/02-net-tweaks/01-run.sh +++ b/stage2/02-net-tweaks/01-run.sh @@ -8,7 +8,9 @@ on_chroot << EOF EOF if [ -v WPA_COUNTRY ]; then - echo "country=${WPA_COUNTRY}" >> "${ROOTFS_DIR}/etc/wpa_supplicant/wpa_supplicant.conf" + on_chroot <<- EOF + SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_wifi_country "${WPA_COUNTRY}" + EOF fi if [ -v WPA_ESSID ] && [ -v WPA_PASSWORD ]; then From d53c4e58fb34d84f8be67a94912f09b8c635b157 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 19 Aug 2022 12:38:33 +0100 Subject: [PATCH 088/222] Default to dhcpcd for now --- stage2/02-net-tweaks/01-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/stage2/02-net-tweaks/01-run.sh b/stage2/02-net-tweaks/01-run.sh index 058547d339..3b06c0577e 100755 --- a/stage2/02-net-tweaks/01-run.sh +++ b/stage2/02-net-tweaks/01-run.sh @@ -5,6 +5,7 @@ install -v -m 600 files/wpa_supplicant.conf "${ROOTFS_DIR}/etc/wpa_supplicant/" on_chroot << EOF SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_boot_wait 0 + SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_netconf 1 EOF if [ -v WPA_COUNTRY ]; then From 69cd4101622c206718f25d2d39e2a2be4e4c7aba Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 30 Aug 2022 16:54:09 +0100 Subject: [PATCH 089/222] Install picamera2 --- export-noobs/00-release/files/release_notes.txt | 1 + stage2/01-sys-tweaks/00-packages-nr | 1 + stage3/00-install-packages/00-packages | 2 ++ stage3/00-install-packages/01-run.sh | 1 + 4 files changed, 5 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index ed3ab10fb1..f4e1810efc 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,6 +1,7 @@ UNRELEASED: * Install NetworkManager * Install OpenJDK 17 rather than OpenJDK 11 on full images + * Install picamera2 * Format the root partition with the metadata_csum option * Format the boot partition with 4 sectors per cluster for a slight performance boost * Remove 'flush' mount option from the boot partition diff --git a/stage2/01-sys-tweaks/00-packages-nr b/stage2/01-sys-tweaks/00-packages-nr index 6bdb60069e..453ada8be2 100644 --- a/stage2/01-sys-tweaks/00-packages-nr +++ b/stage2/01-sys-tweaks/00-packages-nr @@ -1,3 +1,4 @@ cifs-utils libcamera-apps-lite mkvtoolnix +python3-picamera2 diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index e04dbd705d..d8a272f69b 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -13,3 +13,5 @@ obconf arandr libcamera-tools libcamera-apps +python3-pyqt5 +python3-opengl diff --git a/stage3/00-install-packages/01-run.sh b/stage3/00-install-packages/01-run.sh index d76874722a..0e8248dc10 100755 --- a/stage3/00-install-packages/01-run.sh +++ b/stage3/00-install-packages/01-run.sh @@ -5,4 +5,5 @@ update-alternatives --install /usr/bin/x-www-browser \ x-www-browser /usr/bin/chromium-browser 86 update-alternatives --install /usr/bin/gnome-www-browser \ gnome-www-browser /usr/bin/chromium-browser 86 + apt-mark auto python3-pyqt5 python3-opengl EOF From ec643eb66a229f21ab7e2cd520716c1a00d0564a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 31 Aug 2022 17:03:45 +0100 Subject: [PATCH 090/222] Update release notes --- .../00-release/files/release_notes.txt | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index f4e1810efc..ba3c466401 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,10 +1,31 @@ UNRELEASED: - * Install NetworkManager + * lxpanel - new main menu plugin with text search + * lxpanel - new separate audio input plugin with microphone volume and input select + * lxpanel - keyboard shortcuts to open wifi and bluetooth plugins added + * lxpanel - now shown with short delay after startup and between each + * rc_gui - only allows valid hostnames to be set + * piwiz - no longer allows "root" as a user name + * gtk3 - menus can now be resized after being drawn + * raspi-config - option to switch between dhcpcd and Network Manager added + * lxpanel - new network plugin compatible with Network Manager added + * piwiz - compatibility with Network Manager added + * Bug fix - 100% CPU usage in file manager when desktop item unmounted + * Bug fix - window manager was preventing switching between international keyboard layouts + * Bug fix - system tray redrawing made more robust + * Bug fix - translations not being shown in various lxpanel plugins + * Bug fix - updater plugin failing on x86 images + * Bug fix - force power on for Bluetooth hardware when unblocked with rfkill + * Bug fix - message boxes in rc_gui not centred correctly + * Bug fix - switching sessions between Wayland and X11 made more robust + * Bug fix - switching of ALSA devices in raspi-config made compatible with third-party devices + * Install NetworkManager (disabled) * Install OpenJDK 17 rather than OpenJDK 11 on full images * Install picamera2 * Format the root partition with the metadata_csum option * Format the boot partition with 4 sectors per cluster for a slight performance boost * Remove 'flush' mount option from the boot partition + * Raspberry Pi firmware 48cd70fe84432c5d050637b61e4b7b9c831c98bf + * Linux kernel 5.15.61 2022-04-04: * Default "pi” user has been removed; the first-boot wizard enforces the creation of a new user account * rename-user script added to allow existing users to be renamed by temporarily rebooting to cut-down first-boot wizard From 67491f15a8cf73a917fd3b9a988c16ff3ba6bb69 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 1 Sep 2022 11:21:09 +0100 Subject: [PATCH 091/222] Install 'file' --- stage2/01-sys-tweaks/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 7656f03d1a..a2c3207ade 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -32,3 +32,4 @@ rpi-eeprom raspinfo udisks2 unzip zip p7zip-full +file From 7206ab1c222425addfdf206f515a5952ee1ffc78 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 1 Sep 2022 16:27:15 +0100 Subject: [PATCH 092/222] Switch to sys-mods' firstboot script --- stage2/01-sys-tweaks/00-patches/07-resize-init.diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/01-sys-tweaks/00-patches/07-resize-init.diff b/stage2/01-sys-tweaks/00-patches/07-resize-init.diff index cb160ae3ed..dfc01d4950 100644 --- a/stage2/01-sys-tweaks/00-patches/07-resize-init.diff +++ b/stage2/01-sys-tweaks/00-patches/07-resize-init.diff @@ -2,4 +2,4 @@ +++ stage2/rootfs/boot/cmdline.txt @@ -1 +1 @@ -console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 fsck.repair=yes rootwait -+console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 fsck.repair=yes rootwait quiet init=/usr/lib/raspi-config/init_resize.sh ++console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 fsck.repair=yes rootwait quiet init=/usr/lib/raspberrypi-sys-mods/firstboot From 66255495f29be5d09b765d081aff6fc0f11e59b4 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 6 Sep 2022 11:28:50 +0100 Subject: [PATCH 093/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index ba3c466401..07f92a4673 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -2,7 +2,7 @@ UNRELEASED: * lxpanel - new main menu plugin with text search * lxpanel - new separate audio input plugin with microphone volume and input select * lxpanel - keyboard shortcuts to open wifi and bluetooth plugins added - * lxpanel - now shown with short delay after startup and between each + * lxpanel - notifications now shown with short delay after startup and between each * rc_gui - only allows valid hostnames to be set * piwiz - no longer allows "root" as a user name * gtk3 - menus can now be resized after being drawn From c0cddb7c834e6a0e55c11246bee40188ea50aab8 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 7 Sep 2022 13:00:14 +0100 Subject: [PATCH 094/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 07f92a4673..259ba102dc 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,6 @@ UNRELEASED: + * +2022-09-06: * lxpanel - new main menu plugin with text search * lxpanel - new separate audio input plugin with microphone volume and input select * lxpanel - keyboard shortcuts to open wifi and bluetooth plugins added From cb26ff3c1dca37dfa234131c0b01be4a5d8544b3 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 14 Sep 2022 16:50:18 +0100 Subject: [PATCH 095/222] nodered is no longer included in full images --- stage5/00-install-extras/00-packages | 1 - 1 file changed, 1 deletion(-) diff --git a/stage5/00-install-extras/00-packages b/stage5/00-install-extras/00-packages index d839f2b98b..909480b296 100644 --- a/stage5/00-install-extras/00-packages +++ b/stage5/00-install-extras/00-packages @@ -8,7 +8,6 @@ python3-sense-emu sense-emu-tools python-sense-emu-doc wolfram-engine claws-mail greenfoot-unbundled bluej-unbundled -nodered realvnc-vnc-viewer code-the-classics From 59886d8cdf92045e0d7b66c8f5caccdb5a9afda2 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 15 Sep 2022 15:41:26 +0100 Subject: [PATCH 096/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 259ba102dc..09ce50ddff 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,14 @@ UNRELEASED: - * + * NodeRED removed from Recommended Software and full image - should only be installed via apt + * Bug fix - splash screen version number and date incorrect + * Bug fix - text entry in searchable main menu ignored while caps or num lock active + * Bug fix - keyboard shortcuts to open Wi-fi and Bluetooth plugin menus not working in 64-bit builds + * Bug fix - typo in Bluetooth device menu + * Bug fix - firstboot script skipped further steps if rootfs resize failed + * Bug fix - firstboot script inadvertently wrote /test.log + * Bug fix - typo in raspi-config resulted in empty file /2 being created + * Raspberry Pi firmware 48cd70fe84432c5d050637b61e4b7b9c831c98bf + * Linux kernel 5.15.61 2022-09-06: * lxpanel - new main menu plugin with text search * lxpanel - new separate audio input plugin with microphone volume and input select @@ -29,7 +38,7 @@ UNRELEASED: * Raspberry Pi firmware 48cd70fe84432c5d050637b61e4b7b9c831c98bf * Linux kernel 5.15.61 2022-04-04: - * Default "pi” user has been removed; the first-boot wizard enforces the creation of a new user account + * Default "pi" user has been removed; the first-boot wizard enforces the creation of a new user account * rename-user script added to allow existing users to be renamed by temporarily rebooting to cut-down first-boot wizard * Overscan now managed by xrandr under KMS, can be set independently for both monitors, and takes effect on the fly rather than requiring reboot * GTK3 switch control now used in place of paired radio buttons throughout @@ -46,7 +55,7 @@ UNRELEASED: * Bug fix - terminal windows not focussed correctly when launched * Bug fix - crash after multiple update checks in updater plugin * Bug fix - Raspberry Pi keyboard auto-detect by wizard was broken in previous release - * Bug fix - spurious "connected” dialog box shown when reconnecting to Bluetooth LE devices on boot + * Bug fix - spurious "connected" dialog box shown when reconnecting to Bluetooth LE devices on boot * Support for experimental Wayland backend added - can be enabled as an advanced option in raspi-config * Various small bug fixes and graphical tweaks * Chromium upgraded to version 98.0.4758.106 From 8a42abcd1dbd8c9c1fdfca4e0c3778255b2f9cc4 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 21 Sep 2022 16:15:01 +0100 Subject: [PATCH 097/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 09ce50ddff..9316161d95 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,9 +1,13 @@ UNRELEASED: * NodeRED removed from Recommended Software and full image - should only be installed via apt + * Improved speed of startup of lxpanel network controller plugins + * Improved detection of Bluetooth HID devices in first-boot wizard * Bug fix - splash screen version number and date incorrect * Bug fix - text entry in searchable main menu ignored while caps or num lock active * Bug fix - keyboard shortcuts to open Wi-fi and Bluetooth plugin menus not working in 64-bit builds * Bug fix - typo in Bluetooth device menu + * Bug fix - crash when cycling windows in mutter + * Bug fix - spurious text output in raspi-config network configuration selection * Bug fix - firstboot script skipped further steps if rootfs resize failed * Bug fix - firstboot script inadvertently wrote /test.log * Bug fix - typo in raspi-config resulted in empty file /2 being created From 6a61be92953ab6a281d36898e4fd8a02d448e043 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 22 Sep 2022 13:06:38 +0100 Subject: [PATCH 098/222] Disable first run wizard if DISABLE_FIRST_BOOT_USER_RENAME is set --- export-image/01-user-rename/01-run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/export-image/01-user-rename/01-run.sh b/export-image/01-user-rename/01-run.sh index f3f900f793..aa5dd94102 100755 --- a/export-image/01-user-rename/01-run.sh +++ b/export-image/01-user-rename/01-run.sh @@ -4,4 +4,6 @@ if [[ "${DISABLE_FIRST_BOOT_USER_RENAME}" == "0" ]]; then on_chroot <<- EOF SUDO_USER="${FIRST_USER_NAME}" rename-user -f -s EOF +else + rm -f "${ROOTFS_DIR}/etc/xdg/autostart/piwiz.desktop" fi From 6b8e41274f429b704933d70f8bd42a3eec8fb961 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 26 Sep 2022 12:14:28 +0100 Subject: [PATCH 099/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 9316161d95..4b6edff110 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,6 @@ UNRELEASED: + * +2022-09-22: * NodeRED removed from Recommended Software and full image - should only be installed via apt * Improved speed of startup of lxpanel network controller plugins * Improved detection of Bluetooth HID devices in first-boot wizard From 4da039215b162cf4a6a670340728377f5ddb5a18 Mon Sep 17 00:00:00 2001 From: kralo <kralo@users.noreply.github.com> Date: Mon, 3 Oct 2022 17:39:39 +0200 Subject: [PATCH 100/222] Update README.md document WPA_COUNTRY Code format (#636) raspi-config will silently fail, if the argument to `do_wifi_country` is not an uppercase 2-letter Code from the list in /usr/share/zoneinfo/iso3166.tab --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a86f95155b..05230cf09c 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ The following environment variables are supported: * `WPA_ESSID`, `WPA_PASSWORD` and `WPA_COUNTRY` (Default: unset) - If these are set, they are use to configure `wpa_supplicant.conf`, so that the Raspberry Pi can automatically connect to a wireless network on first boot. If `WPA_ESSID` is set and `WPA_PASSWORD` is unset an unprotected wireless network will be configured. If set, `WPA_PASSWORD` must be between 8 and 63 characters. + If these are set, they are use to configure `wpa_supplicant.conf`, so that the Raspberry Pi can automatically connect to a wireless network on first boot. If `WPA_ESSID` is set and `WPA_PASSWORD` is unset an unprotected wireless network will be configured. If set, `WPA_PASSWORD` must be between 8 and 63 characters. `WPA_COUNTRY` is a 2-letter ISO/IEC 3166 country Code, i.e. `GB` * `ENABLE_SSH` (Default: `0`) From f00b72208cf73a6ed0ab20c16e28c5dbad04a3d7 Mon Sep 17 00:00:00 2001 From: kralo <kralo@users.noreply.github.com> Date: Fri, 14 Oct 2022 15:48:30 +0200 Subject: [PATCH 101/222] apt/retries: specify option from the Acquire group (#638) per man apt.conf, Acquire group is not part of the apt group --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 135da35b90..ea96633129 100755 --- a/build.sh +++ b/build.sh @@ -21,7 +21,7 @@ EOF PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < "${i}-packages-nr")" if [ -n "$PACKAGES" ]; then on_chroot << EOF -apt-get -o APT::Acquire::Retries=3 install --no-install-recommends -y $PACKAGES +apt-get -o Acquire::Retries=3 install --no-install-recommends -y $PACKAGES EOF if [ "${USE_QCOW2}" = "1" ]; then on_chroot << EOF @@ -36,7 +36,7 @@ EOF PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < "${i}-packages")" if [ -n "$PACKAGES" ]; then on_chroot << EOF -apt-get -o APT::Acquire::Retries=3 install -y $PACKAGES +apt-get -o Acquire::Retries=3 install -y $PACKAGES EOF if [ "${USE_QCOW2}" = "1" ]; then on_chroot << EOF From 47e34efb89fb76178d0b7f8690d1b24a222c077b Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 1 Nov 2022 14:09:24 +0000 Subject: [PATCH 102/222] stage2: pre-install kms++-utils --- export-noobs/00-release/files/release_notes.txt | 2 +- stage2/01-sys-tweaks/00-packages | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 4b6edff110..3e6650dbeb 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,5 @@ UNRELEASED: - * + * Install kms++-utils 2022-09-22: * NodeRED removed from Recommended Software and full image - should only be installed via apt * Improved speed of startup of lxpanel network controller plugins diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index a2c3207ade..6138c6d5a4 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -33,3 +33,4 @@ raspinfo udisks2 unzip zip p7zip-full file +kms++-utils From 549681f6e217d6e713afb943d571afda83542a2f Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 8 Nov 2022 11:28:18 +0000 Subject: [PATCH 103/222] Updates to avoid common pitfalls when changing the RELEASE variable Fixes #635, fixes #645, fixes #632 --- README.md | 13 ++++++++----- build.sh | 2 +- stage0/prerun.sh | 5 +++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 05230cf09c..35f2c981cb 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ Getting started is as simple as cloning this repository on your build machine. Y can do so with: ```bash -git clone --depth 1 https://github.com/RPI-Distro/pi-gen.git +git clone https://github.com/RPI-Distro/pi-gen.git ``` -Using `--depth 1` with `git clone` will create a shallow clone, only containing +`--depth 1` can be added afer `git clone` to create a shallow clone, only containing the latest revision of the repository. Do not do this on your development machine. Also, be careful to clone the repository to a base path **NOT** containing spaces. @@ -80,8 +80,11 @@ The following environment variables are supported: * `RELEASE` (Default: bullseye) - The release version to build images against. Valid values are jessie, stretch, - buster, bullseye, and testing. + The release version to build images against. Valid values are any supported + Debian release. However, since different releases will have different sets of + packages available, you'll need to either modify your stages accordingly, or + checkout the appropriate branch. For example, if you'd like to build a + `buster` image, you should do so from the `buster` branch. * `APT_PROXY` (Default: unset) @@ -489,7 +492,7 @@ A 64 bit image can be generated from the `arm64` branch in this repository. Just replace the command from [this section](#getting-started-with-building-your-images) by the one below, and follow the rest of the documentation: ```bash -git clone --depth 1 --branch arm64 https://github.com/RPI-Distro/pi-gen.git +git clone --branch arm64 https://github.com/RPI-Distro/pi-gen.git ``` If you want to generate a 64 bits image from a Raspberry Pi running a 32 bits diff --git a/build.sh b/build.sh index ea96633129..8966bc735c 100755 --- a/build.sh +++ b/build.sh @@ -226,7 +226,7 @@ export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi} export FIRST_USER_NAME=${FIRST_USER_NAME:-pi} export FIRST_USER_PASS export DISABLE_FIRST_BOOT_USER_RENAME=${DISABLE_FIRST_BOOT_USER_RENAME:-0} -export RELEASE=${RELEASE:-bullseye} +export RELEASE=${RELEASE:-bullseye} # Don't forget to update stage0/prerun.sh export WPA_ESSID export WPA_PASSWORD export WPA_COUNTRY diff --git a/stage0/prerun.sh b/stage0/prerun.sh index d4dd0a1fb6..024d369f36 100755 --- a/stage0/prerun.sh +++ b/stage0/prerun.sh @@ -1,5 +1,10 @@ #!/bin/bash -e +if [ "$RELEASE" != "bullseye" ]; then + echo "WARNING: RELEASE does not match the intended option for this branch." + echo " Please check the relevant README.md section." +fi + if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then bootstrap ${RELEASE} "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/ fi From c3083ecd503629eac5184ec692f65bbbd28ac317 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 8 Nov 2022 12:04:09 +0000 Subject: [PATCH 104/222] Add parameter to keep cap_setfcap Fixes #643 --- README.md | 7 +++++++ build.sh | 4 ++++ scripts/common | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 35f2c981cb..d7beb1f473 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,13 @@ The following environment variables are supported: public key authentication. Note that if SSH is not enabled this will take effect when SSH becomes enabled. + * `SETFCAP` (Default: unset) + + * Setting to `1` will prevent pi-gen from dropping the "capabilities" + feature. Generating the root filesystem with capabilities enabled and running + it from a filesystem that does not support capabilities (like NFS) can cause + issues. Only enable this if you understand what it is. + * `STAGE_LIST` (Default: `stage*`) If set, then instead of working through the numeric stages in order, this list will be followed. For example setting to `"stage0 stage1 mystage stage2"` will run the contents of `mystage` before stage2. Note that quotes are needed around the list. An absolute or relative path can be given for stages outside the pi-gen directory. diff --git a/build.sh b/build.sh index 8966bc735c..986f81cc3b 100755 --- a/build.sh +++ b/build.sh @@ -283,6 +283,10 @@ fi export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" +if [ "$SETFCAP" != "1" ]; then + export CAPSH_ARG="--drop=cap_setfcap" +fi + dependencies_check "${BASE_DIR}/depends" #check username is valid diff --git a/scripts/common b/scripts/common index e476f0ffa6..14be1c2013 100644 --- a/scripts/common +++ b/scripts/common @@ -17,7 +17,7 @@ bootstrap(){ BOOTSTRAP_ARGS+=("$@") printf -v BOOTSTRAP_STR '%q ' "${BOOTSTRAP_ARGS[@]}" - setarch linux32 capsh --drop=cap_setfcap -- -c "'${BOOTSTRAP_CMD}' $BOOTSTRAP_STR" || true + setarch linux32 capsh $CAPSH_ARG -- -c "'${BOOTSTRAP_CMD}' $BOOTSTRAP_STR" || true if [ -d "$2/debootstrap" ] && ! rmdir "$2/debootstrap"; then cp "$2/debootstrap/debootstrap.log" "${STAGE_WORK_DIR}" @@ -90,7 +90,7 @@ on_chroot() { mount --bind /sys "${ROOTFS_DIR}/sys" fi - setarch linux32 capsh --drop=cap_setfcap "--chroot=${ROOTFS_DIR}/" -- -e "$@" + setarch linux32 capsh $CAPSH_ARG "--chroot=${ROOTFS_DIR}/" -- -e "$@" } export -f on_chroot From 1d13a292a79f151b06ff84823b9e41781120ce6f Mon Sep 17 00:00:00 2001 From: rfc6919 <rfc6919@users.noreply.github.com> Date: Fri, 11 Nov 2022 19:56:04 +1100 Subject: [PATCH 105/222] use a single partitioned loop device (#641) --- export-image/prerun.sh | 32 ++++++-------------------------- export-noobs/prerun.sh | 32 ++++++-------------------------- scripts/common | 20 ++++++++------------ 3 files changed, 20 insertions(+), 64 deletions(-) diff --git a/export-image/prerun.sh b/export-image/prerun.sh index 145d61eee0..c3412c255f 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -33,41 +33,21 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" - PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) - BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) - BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) - - ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) - ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) - - echo "Mounting BOOT_DEV..." - cnt=0 - until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do - if [ $cnt -lt 5 ]; then - cnt=$((cnt + 1)) - echo "Error in losetup for BOOT_DEV. Retrying..." - sleep 5 - else - echo "ERROR: losetup for BOOT_DEV failed; exiting" - exit 1 - fi - done - - echo "Mounting ROOT_DEV..." + echo "Creating loop device..." cnt=0 - until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do + until LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do if [ $cnt -lt 5 ]; then cnt=$((cnt + 1)) - echo "Error in losetup for ROOT_DEV. Retrying..." + echo "Error in losetup. Retrying..." sleep 5 else - echo "ERROR: losetup for ROOT_DEV failed; exiting" + echo "ERROR: losetup failed; exiting" exit 1 fi done - echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" - echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" + BOOT_DEV="${LOOP_DEV}p1" + ROOT_DEV="${LOOP_DEV}p2" ROOT_FEATURES="^huge_file" for FEATURE in 64bit; do diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 97dd09490e..3858c66193 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -9,41 +9,21 @@ unmount_image "${IMG_FILE}" rm -rf "${NOOBS_DIR}" -PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) -BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) -BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) - -ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) -ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) - -echo "Mounting BOOT_DEV..." -cnt=0 -until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do - if [ $cnt -lt 5 ]; then - cnt=$((cnt + 1)) - echo "Error in losetup for BOOT_DEV. Retrying..." - sleep 5 - else - echo "ERROR: losetup for BOOT_DEV failed; exiting" - exit 1 - fi -done - -echo "Mounting ROOT_DEV..." +echo "Creating loop device..." cnt=0 -until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do +until LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do if [ $cnt -lt 5 ]; then cnt=$((cnt + 1)) - echo "Error in losetup for ROOT_DEV. Retrying..." + echo "Error in losetup. Retrying..." sleep 5 else - echo "ERROR: losetup for ROOT_DEV failed; exiting" + echo "ERROR: losetup failed; exiting" exit 1 fi done -echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" -echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" +BOOT_DEV="${LOOP_DEV}p1" +ROOT_DEV="${LOOP_DEV}p2" mkdir -p "${STAGE_WORK_DIR}/rootfs" mkdir -p "${NOOBS_DIR}" diff --git a/scripts/common b/scripts/common index 14be1c2013..82bd726d6a 100644 --- a/scripts/common +++ b/scripts/common @@ -57,19 +57,15 @@ export -f unmount unmount_image(){ sync sleep 1 - local LOOP_DEVICES - LOOP_DEVICES=$(losetup --list | grep "$(basename "${1}")" | cut -f1 -d' ') - for LOOP_DEV in ${LOOP_DEVICES}; do - if [ -n "${LOOP_DEV}" ]; then - local MOUNTED_DIR - MOUNTED_DIR=$(mount | grep "$(basename "${LOOP_DEV}")" | head -n 1 | cut -f 3 -d ' ') - if [ -n "${MOUNTED_DIR}" ] && [ "${MOUNTED_DIR}" != "/" ]; then - unmount "$(dirname "${MOUNTED_DIR}")" + LOOP_DEVICE=$(losetup --list | grep "$1" | cut -f1 -d' ') + if [ -n "$LOOP_DEVICE" ]; then + for part in "$LOOP_DEVICE"p*; do + if DIR=$(findmnt -n -o target -S "$part"); then + unmount "$DIR" fi - sleep 1 - losetup -d "${LOOP_DEV}" - fi - done + done + losetup -d "$LOOP_DEVICE" + fi } export -f unmount_image From 05081aa0c4d3faf482cc22a88c5dd11364907aa5 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 8 Feb 2023 18:46:32 +0000 Subject: [PATCH 106/222] Fix apt gpg key install --- stage0/00-configure-apt/00-run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 4d0ac92121..6494a03d05 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -12,7 +12,8 @@ else rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache" fi -cat files/raspberrypi.gpg.key | gpg --dearmor > "${ROOTFS_DIR}/etc/apt/trusted.gpg.d/raspberrypi-archive-stable.gpg" +cat files/raspberrypi.gpg.key | gpg --dearmor > "${STAGE_WORK_DIR}/raspberrypi-archive-stable.gpg" +install -m 644 "${STAGE_WORK_DIR}/raspberrypi-archive-stable.gpg" "${ROOTFS_DIR}/etc/apt/trusted.gpg.d/" on_chroot << EOF apt-get update apt-get dist-upgrade -y From 5d2c6f31cefc7710e3bbc44012b9ffb843294e34 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 14 Feb 2023 15:09:53 +0000 Subject: [PATCH 107/222] export-image: change boot partition lable to 'bootfs' Fixes #669 --- export-image/prerun.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export-image/prerun.sh b/export-image/prerun.sh index c3412c255f..2d8e1bbe27 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -55,7 +55,7 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" fi done - mkdosfs -n boot -F 32 -s 4 -v "$BOOT_DEV" > /dev/null + mkdosfs -n bootfs -F 32 -s 4 -v "$BOOT_DEV" > /dev/null mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 From f2d385517c9631f2ded876deb1115725d0c75995 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 14 Feb 2023 15:26:12 +0000 Subject: [PATCH 108/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 3e6650dbeb..c6948280c0 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,16 @@ UNRELEASED: + * glamor now disabled on all platforms other than Raspberry Pi 4 with legacy video driver + * msdri3 video driver support added + * KiCad added to Recommended Software + * Support for new touchscreen driver added to Screen Resolution tool; minor UI tweaks + * GTK message dialogs shown with right-justified buttons + * Bug fix - updater plugin now does not clear icon when an update has failed + * Bug fix - keyboard highlight now shown on GTK switch control + * Some Korean and Brazilian translations added + * Fix rpi-imager hidden ssid configuration * Install kms++-utils + * Raspberry Pi firmware 78852e166b4cf3ebb31d051e996d54792f0994b0 + * Linux kernel 5.15.84 2022-09-22: * NodeRED removed from Recommended Software and full image - should only be installed via apt * Improved speed of startup of lxpanel network controller plugins From cf40e4b299ca4023f842eed1ecc5b254eb3eb688 Mon Sep 17 00:00:00 2001 From: Stefan Becker <stefanb2@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:30:34 +0300 Subject: [PATCH 109/222] Check that docker is not set up in rootless mode (#679) Mounting binfmt_misc inside the container requires root rights. If "docker info" indicates rootless mode then fall back to "sudo docker". Allow the user to override the docker command with the DOCKER by setting the environment variable. --- build-docker.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index a7f18f136f..b3995d53b0 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -4,10 +4,15 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" BUILD_OPTS="$*" -DOCKER="docker" - -if ! ${DOCKER} ps >/dev/null 2>&1; then - DOCKER="sudo docker" +# Allow user to override docker command +DOCKER=${DOCKER:-docker} + +# Ensure that default docker command is not set up in rootless mode +if \ + ! ${DOCKER} ps >/dev/null 2>&1 || \ + ${DOCKER} info 2>/dev/null | grep -q rootless \ +; then + DOCKER="sudo ${DOCKER}" fi if ! ${DOCKER} ps >/dev/null; then echo "error connecting to docker:" From 0db41e35efc3ebee7c334e11ae8d91607ef5931a Mon Sep 17 00:00:00 2001 From: Stefan Becker <stefanb2@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:31:02 +0300 Subject: [PATCH 110/222] Ensure that deploy is always owned by user (#680) Switch "docker cp" to tar streaming mode. As the receiving "tar" command is executed outside Docker the resulting directories & files are owned by the calling user. --- build-docker.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-docker.sh b/build-docker.sh index b3995d53b0..413c12a69f 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -126,8 +126,9 @@ else wait "$!" fi +# Ensure that deploy/ is always owned by calling user echo "copying results from deploy/" -${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy . +${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy - | tar -xf - ls -lah deploy # cleanup From b8437a77ba9e52ae23ee80414198d5d92f2babf5 Mon Sep 17 00:00:00 2001 From: Stefan Becker <stefanb2@users.noreply.github.com> Date: Tue, 28 Mar 2023 13:53:54 +0300 Subject: [PATCH 111/222] DRY removal for docker run command line (#686) Factor out the differences into variables so that there is only one implementation of the "docker run" command line to maintain. --- build-docker.sh | 65 ++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index 413c12a69f..01b9f8e98c 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -92,40 +92,43 @@ esac ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" if [ "${CONTAINER_EXISTS}" != "" ]; then - trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM - time ${DOCKER} run --rm --privileged \ - --cap-add=ALL \ - -v /dev:/dev \ - -v /lib/modules:/lib/modules \ - ${PIGEN_DOCKER_OPTS} \ - --volume "${CONFIG_FILE}":/config:ro \ - -e "GIT_HASH=${GIT_HASH}" \ - --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ - pi-gen \ - bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && - # binfmt_misc is sometimes not mounted with debian bullseye image - (mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) && - cd /pi-gen; ./build.sh ${BUILD_OPTS} && - rsync -av work/*/build.log deploy/" & - wait "$!" + DOCKER_CMDLINE_NAME="${CONTAINER_NAME}_cont" + DOCKER_CMDLINE_PRE=( \ + --rm \ + ) + DOCKER_CMDLINE_POST=( \ + --volumes-from="${CONTAINER_NAME}" \ + ) else - trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM - time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \ - --cap-add=ALL \ - -v /dev:/dev \ - -v /lib/modules:/lib/modules \ - ${PIGEN_DOCKER_OPTS} \ - --volume "${CONFIG_FILE}":/config:ro \ - -e "GIT_HASH=${GIT_HASH}" \ - pi-gen \ - bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && - # binfmt_misc is sometimes not mounted with debian bullseye image - (mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) && - cd /pi-gen; ./build.sh ${BUILD_OPTS} && - rsync -av work/*/build.log deploy/" & - wait "$!" + DOCKER_CMDLINE_NAME="${CONTAINER_NAME}" + DOCKER_CMDLINE_PRE=( \ + ) + DOCKER_CMDLINE_POST=( \ + ) fi +trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${DOCKER_CMDLINE_NAME}' SIGINT SIGTERM +time ${DOCKER} run \ + "${DOCKER_CMDLINE_PRE[@]}" \ + --name "${DOCKER_CMDLINE_NAME}" \ + --privileged \ + --cap-add=ALL \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules \ + ${PIGEN_DOCKER_OPTS} \ + --volume "${CONFIG_FILE}":/config:ro \ + -e "GIT_HASH=${GIT_HASH}" \ + "${DOCKER_CMDLINE_POST[@]}" \ + pi-gen \ + bash -e -o pipefail -c " + dpkg-reconfigure qemu-user-static && + # binfmt_misc is sometimes not mounted with debian bullseye image + (mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) && + cd /pi-gen; ./build.sh ${BUILD_OPTS} && + rsync -av work/*/build.log deploy/ + " & + wait "$!" + # Ensure that deploy/ is always owned by calling user echo "copying results from deploy/" ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy - | tar -xf - From 4e9275398278c4cab8ba68f3d9ca54ab97b73922 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 29 Mar 2023 07:55:37 +0100 Subject: [PATCH 112/222] Use tmpfs for chroot /run and /tmp --- scripts/common | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/common b/scripts/common index 82bd726d6a..5731b72fa6 100644 --- a/scripts/common +++ b/scripts/common @@ -86,6 +86,14 @@ on_chroot() { mount --bind /sys "${ROOTFS_DIR}/sys" fi + if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/run)"; then + mount -t tmpfs tmpfs "${ROOTFS_DIR}/run" + fi + + if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/tmp)"; then + mount -t tmpfs tmpfs "${ROOTFS_DIR}/tmp" + fi + setarch linux32 capsh $CAPSH_ARG "--chroot=${ROOTFS_DIR}/" -- -e "$@" } export -f on_chroot From 6dc45a80e764eacd6e311c32164b623bbe7f8e08 Mon Sep 17 00:00:00 2001 From: Mohsen Tahmasebi <mhntah@gmail.com> Date: Mon, 3 Apr 2023 11:57:59 +0330 Subject: [PATCH 113/222] Register binfmt_misc entry for docker build (#685) --- build-docker.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/build-docker.sh b/build-docker.sh index 01b9f8e98c..5495dfaaea 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -107,6 +107,40 @@ else ) fi +# Check if binfmt_misc is required +binfmt_misc_required=1 +case $(uname -m) in + aarch64) + binfmt_misc_required=0 + ;; + arm*) + binfmt_misc_required=0 + ;; +esac + +# Check if qemu-arm-static and /proc/sys/fs/binfmt_misc are present +if [[ "${binfmt_misc_required}" == "1" ]]; then + if ! qemu_arm=$(which qemu-arm-static) ; then + echo "qemu-arm-static not found (please install qemu-user-static)" + exit 1 + fi + if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then + echo "binfmt_misc required but not mounted, trying to mount it..." + if ! mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; then + echo "mounting binfmt_misc failed" + exit 1 + fi + echo "binfmt_misc mounted" + fi + # Register qemu-arm for binfmt_misc (binfmt_misc won't care duplicate entries unless they have common names) + reg="echo ':qemu-arm-rpi:M::"\ +"\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:"\ +"\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:"\ +"$qemu_arm:F' > /proc/sys/fs/binfmt_misc/register" + echo "Registering qemu-arm for binfmt_misc..." + sudo bash -c "$reg" 2>/dev/null || true +fi + trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${DOCKER_CMDLINE_NAME}' SIGINT SIGTERM time ${DOCKER} run \ "${DOCKER_CMDLINE_PRE[@]}" \ From 9249e146d2e3987adf2ea0e5e19e295d37f3f886 Mon Sep 17 00:00:00 2001 From: Stefan Becker <stefanb2@users.noreply.github.com> Date: Mon, 3 Apr 2023 12:08:50 +0300 Subject: [PATCH 114/222] Copy container log to deploy (#688) The container log is much more detailed than the default build.log, which makes life much easier for a build maintainer. Before deleting the container copy the log out to the deploy/ directory. --- build-docker.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-docker.sh b/build-docker.sh index 5495dfaaea..7d0b5ad9c7 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -166,6 +166,10 @@ time ${DOCKER} run \ # Ensure that deploy/ is always owned by calling user echo "copying results from deploy/" ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy - | tar -xf - + +echo "copying log from container ${CONTAINER_NAME} to depoy/" +${DOCKER} logs --timestamps "${CONTAINER_NAME}" &>deploy/build-docker.log + ls -lah deploy # cleanup From 47eee1f0ddcf8811559d51eea1c1bb48335e3e88 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 25 Apr 2023 15:44:14 +0100 Subject: [PATCH 115/222] Update release notes --- .../00-release/files/release_notes.txt | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index c6948280c0..7ec34fde2c 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,37 @@ UNRELEASED: + * 64-bit Mathematica added to rp-prefapps + * Bug fix - occasional segfault in CPU temperature plugin + * Bug fix - X server crash when changing screen orientation + * Bug fix - X server DPMS not working + * Mathematica updated to 13.2.1 + * Matlab updated to 23.1.0 + * Chromium updated to 113.0.5672.59 + * Raspberry Pi Imager updated to 1.7.4 + * RealVNC server updated to 7.0.1.49073 + * RealVNC viewer updated to 7.0.1.48981 + * Updated VLC HW acceleration patch + * libcamera + - Add generalised statistics handling. + - Fix overflow that would cause incorrect calculations in the AGC algorithm. + - Improve IMX296 sensor tuning. + * libcamera-apps + - Improve handling of audio resampling and encoding using libav + - Improve performance of QT preview window rendering + - Add support for 16-bit Bayer in the DNG writer + - Fix for encoder lockup when framerate is set to 0 + - Improved thumbnail rendering + * picamera2 + - MJPEG server example that uses the hardware MJPEG encoder. + - Example showing preview from two cameras in a single Qt app. + - H264 encoder accepts frame time interval for SPS headers. + - H264 encoder should advertise correct profile/level. + - H264 encoder supports constant quality parameter. + - Exif DateTime and DateTimeOriginal tags are now added. + - Various bug fixes (check Picamera2 release notes for more details). + * Some translations added + * Raspberry Pi firmware 055e044d5359ded1aacc5a17a8e35365373d0b8b + * Linux kernel 6.1.21 +2023-02-21: * glamor now disabled on all platforms other than Raspberry Pi 4 with legacy video driver * msdri3 video driver support added * KiCad added to Recommended Software From 01d24ef22778337ed04cf9d6444b1be57b6a1e1a Mon Sep 17 00:00:00 2001 From: Stefan Becker <stefanb2@users.noreply.github.com> Date: Tue, 9 May 2023 16:24:49 +0300 Subject: [PATCH 116/222] Skip binfmt_misc register when unnecessary (#693) This improves commit 6dc45a80e764eacd6e311c32164b623bbe7f8e08 Check if interpreter is already registered before running the register step. This avoids unnecessary "sudo" execution which may require user interaction for the password. Improves on #685 --- build-docker.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index 7d0b5ad9c7..4fdd89c090 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -132,13 +132,15 @@ if [[ "${binfmt_misc_required}" == "1" ]]; then fi echo "binfmt_misc mounted" fi - # Register qemu-arm for binfmt_misc (binfmt_misc won't care duplicate entries unless they have common names) - reg="echo ':qemu-arm-rpi:M::"\ + if ! grep -q "^interpreter ${qemu_arm}" /proc/sys/fs/binfmt_misc/qemu-arm* ; then + # Register qemu-arm for binfmt_misc + reg="echo ':qemu-arm-rpi:M::"\ "\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:"\ "\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:"\ -"$qemu_arm:F' > /proc/sys/fs/binfmt_misc/register" - echo "Registering qemu-arm for binfmt_misc..." - sudo bash -c "$reg" 2>/dev/null || true +"${qemu_arm}:F' > /proc/sys/fs/binfmt_misc/register" + echo "Registering qemu-arm for binfmt_misc..." + sudo bash -c "${reg}" 2>/dev/null || true + fi fi trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${DOCKER_CMDLINE_NAME}' SIGINT SIGTERM From 446f3e34c8ceb33dfe7bd38578f1cd6b6afe19da Mon Sep 17 00:00:00 2001 From: bamartin125 <bamartin125@gmail.com> Date: Thu, 17 Aug 2023 10:24:47 -0500 Subject: [PATCH 117/222] Adds call to mknod if loop dev does not exist after call to `losetup -f` (#483) --- export-image/prerun.sh | 2 +- export-noobs/prerun.sh | 2 +- scripts/common | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/export-image/prerun.sh b/export-image/prerun.sh index 2d8e1bbe27..267bbe0322 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -35,7 +35,7 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then echo "Creating loop device..." cnt=0 - until LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do + until ensure_next_loopdev && LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do if [ $cnt -lt 5 ]; then cnt=$((cnt + 1)) echo "Error in losetup. Retrying..." diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 3858c66193..62828368cb 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -11,7 +11,7 @@ rm -rf "${NOOBS_DIR}" echo "Creating loop device..." cnt=0 -until LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do +until ensure_next_loopdev && LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do if [ $cnt -lt 5 ]; then cnt=$((cnt + 1)) echo "Error in losetup. Retrying..." diff --git a/scripts/common b/scripts/common index 5731b72fa6..c94b0f0abb 100644 --- a/scripts/common +++ b/scripts/common @@ -102,3 +102,11 @@ update_issue() { echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using ${PI_GEN}, ${PI_GEN_REPO}, ${GIT_HASH}, ${1}" > "${ROOTFS_DIR}/etc/rpi-issue" } export -f update_issue + +ensure_next_loopdev() { + local loopdev + loopdev="$(losetup -f)" + loopmaj="$(echo "$loopdev" | sed -E 's/.*[^0-9]*?([0-9]+)$/\1/')" + [[ -b "$loopdev" ]] || mknod "$loopdev" b 7 "$loopmaj" +} +export -f ensure_next_loopdev From ed68013abbe46c0f665a157a9bdfd06603811101 Mon Sep 17 00:00:00 2001 From: Reuben Miller <reuben.d.miller@gmail.com> Date: Mon, 25 Sep 2023 15:59:25 +0200 Subject: [PATCH 118/222] fix: support bash v3 by removing array usage (#705) --- build-docker.sh | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index 4fdd89c090..3445a4069a 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,6 +1,8 @@ -#!/bin/bash -eu +#!/usr/bin/env bash +# Note: Avoid usage of arrays as MacOS users have an older version of bash (v3.x) which does not supports arrays +set -eu -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)" BUILD_OPTS="$*" @@ -93,18 +95,12 @@ ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" if [ "${CONTAINER_EXISTS}" != "" ]; then DOCKER_CMDLINE_NAME="${CONTAINER_NAME}_cont" - DOCKER_CMDLINE_PRE=( \ - --rm \ - ) - DOCKER_CMDLINE_POST=( \ - --volumes-from="${CONTAINER_NAME}" \ - ) + DOCKER_CMDLINE_PRE="--rm" + DOCKER_CMDLINE_POST="--volumes-from=\"${CONTAINER_NAME}\"" else DOCKER_CMDLINE_NAME="${CONTAINER_NAME}" - DOCKER_CMDLINE_PRE=( \ - ) - DOCKER_CMDLINE_POST=( \ - ) + DOCKER_CMDLINE_PRE="" + DOCKER_CMDLINE_POST="" fi # Check if binfmt_misc is required @@ -145,7 +141,7 @@ fi trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${DOCKER_CMDLINE_NAME}' SIGINT SIGTERM time ${DOCKER} run \ - "${DOCKER_CMDLINE_PRE[@]}" \ + $DOCKER_CMDLINE_PRE \ --name "${DOCKER_CMDLINE_NAME}" \ --privileged \ --cap-add=ALL \ @@ -154,7 +150,7 @@ time ${DOCKER} run \ ${PIGEN_DOCKER_OPTS} \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ - "${DOCKER_CMDLINE_POST[@]}" \ + $DOCKER_CMDLINE_POST \ pi-gen \ bash -e -o pipefail -c " dpkg-reconfigure qemu-user-static && From 6aca615037355f3f0e51f984d196989167424e50 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 12:50:34 +0100 Subject: [PATCH 119/222] Switch to Bookworm --- build.sh | 2 +- stage0/prerun.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 986f81cc3b..ca0fb576fc 100755 --- a/build.sh +++ b/build.sh @@ -226,7 +226,7 @@ export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi} export FIRST_USER_NAME=${FIRST_USER_NAME:-pi} export FIRST_USER_PASS export DISABLE_FIRST_BOOT_USER_RENAME=${DISABLE_FIRST_BOOT_USER_RENAME:-0} -export RELEASE=${RELEASE:-bullseye} # Don't forget to update stage0/prerun.sh +export RELEASE=${RELEASE:-bookworm} # Don't forget to update stage0/prerun.sh export WPA_ESSID export WPA_PASSWORD export WPA_COUNTRY diff --git a/stage0/prerun.sh b/stage0/prerun.sh index 024d369f36..c8e51a426e 100755 --- a/stage0/prerun.sh +++ b/stage0/prerun.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -if [ "$RELEASE" != "bullseye" ]; then +if [ "$RELEASE" != "bookworm" ]; then echo "WARNING: RELEASE does not match the intended option for this branch." echo " Please check the relevant README.md section." fi From 9d9d0637909a506c82cb76cea90a94a958b1732c Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 12:59:39 +0100 Subject: [PATCH 120/222] Remove/replace obsolete packages --- stage2/01-sys-tweaks/00-packages | 6 +++--- stage3/00-install-packages/00-packages-nr | 2 +- stage4/00-install-packages/00-packages | 3 --- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 6138c6d5a4..86bdd9d351 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -1,4 +1,4 @@ -ssh less fbset sudo psmisc strace ed ncdu crda +ssh less fbset sudo psmisc strace ed ncdu console-setup keyboard-configuration debconf-utils parted build-essential manpages-dev bash-completion gdb pkg-config python-is-python3 @@ -7,7 +7,7 @@ python3-gpiozero avahi-daemon lua5.1 luajit -hardlink ca-certificates curl +ca-certificates curl fake-hwclock nfs-common usbutils libraspberrypi-dev libraspberrypi-doc libfreetype6-dev dosfstools @@ -29,7 +29,7 @@ ethtool ntfs-3g pciutils rpi-eeprom -raspinfo +raspi-utils udisks2 unzip zip p7zip-full file diff --git a/stage3/00-install-packages/00-packages-nr b/stage3/00-install-packages/00-packages-nr index 75fa2a1c5f..91131983de 100644 --- a/stage3/00-install-packages/00-packages-nr +++ b/stage3/00-install-packages/00-packages-nr @@ -3,5 +3,5 @@ mousepad lxde lxtask menu-xdg zenity xdg-utils gvfs-backends gvfs-fuse -lightdm gnome-themes-standard-data gnome-icon-theme +lightdm gnome-themes-extra-data gnome-icon-theme gnome-keyring diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index 7854596a1f..bfb3f0d587 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -2,12 +2,10 @@ python3-pygame python3-tk thonny python3-pgzero python3-serial -python3-picamera debian-reference-en dillo raspberrypi-net-mods raspberrypi-ui-mods python3-pip python3-numpy -pypy alacarte rc-gui sense-hat tree libgl1-mesa-dri libgles1 libgles2-mesa xcompmgr @@ -24,4 +22,3 @@ rp-prefapps ffmpeg vlc rpi-imager -rpi-wayland From a524fe44ce88b54869cb6f83dffb403f45543a9e Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 13:01:04 +0100 Subject: [PATCH 121/222] stage3: replace qpdfview with evince --- stage3/00-install-packages/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index d8a272f69b..a9e44e8252 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -1,5 +1,5 @@ gstreamer1.0-x gstreamer1.0-omx gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-alsa gstreamer1.0-libav -qpdfview gtk2-engines alsa-utils +evince gtk2-engines alsa-utils desktop-base git policykit-1 From 43c49d84d28bfafb05386ff53557d61dbbb48743 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 13:01:54 +0100 Subject: [PATCH 122/222] stage3: install eom --- stage3/00-install-packages/00-packages-nr | 1 + 1 file changed, 1 insertion(+) diff --git a/stage3/00-install-packages/00-packages-nr b/stage3/00-install-packages/00-packages-nr index 91131983de..66f3004819 100644 --- a/stage3/00-install-packages/00-packages-nr +++ b/stage3/00-install-packages/00-packages-nr @@ -1,5 +1,6 @@ xserver-xorg-video-fbdev xserver-xorg xinit xserver-xorg-video-fbturbo mousepad +eom lxde lxtask menu-xdg zenity xdg-utils gvfs-backends gvfs-fuse From 9fc1385eb7c666b0105e7eb9e6acc86a853a88df Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 13:02:50 +0100 Subject: [PATCH 123/222] stage3: install Firefox --- stage3/00-install-packages/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index a9e44e8252..2d206cd17e 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -6,6 +6,7 @@ policykit-1 gvfs rfkill chromium-browser rpi-chromium-mods libwidevinecdm0 +firefox rpi-firefox-mods gldriver-test fonts-droid-fallback fonts-liberation2 From 8dd23cbc1d01b2a89f560c98f5e5b8ef2e71fbf3 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 13:34:21 +0100 Subject: [PATCH 124/222] Remove dhcpcd and associated options --- README.md | 4 ++-- build.sh | 2 -- stage2/02-net-tweaks/00-packages | 1 - stage2/02-net-tweaks/01-run.sh | 20 -------------------- stage3/01-tweaks/00-run.sh | 5 ----- 5 files changed, 2 insertions(+), 30 deletions(-) delete mode 100755 stage3/01-tweaks/00-run.sh diff --git a/README.md b/README.md index d7beb1f473..ea276abc22 100644 --- a/README.md +++ b/README.md @@ -195,9 +195,9 @@ The following environment variables are supported: stays activated. `FIRST_USER_PASS` must be set for this to work. Please be aware of the implied security risk of defining a default username and password for your devices. - * `WPA_ESSID`, `WPA_PASSWORD` and `WPA_COUNTRY` (Default: unset) + * `WPA_COUNTRY` (Default: unset) - If these are set, they are use to configure `wpa_supplicant.conf`, so that the Raspberry Pi can automatically connect to a wireless network on first boot. If `WPA_ESSID` is set and `WPA_PASSWORD` is unset an unprotected wireless network will be configured. If set, `WPA_PASSWORD` must be between 8 and 63 characters. `WPA_COUNTRY` is a 2-letter ISO/IEC 3166 country Code, i.e. `GB` + Sets the default WLAN regulatory domain and unblocks WLAN interfaces. This should be a 2-letter ISO/IEC 3166 country Code, i.e. `GB` * `ENABLE_SSH` (Default: `0`) diff --git a/build.sh b/build.sh index ca0fb576fc..22b605d822 100755 --- a/build.sh +++ b/build.sh @@ -227,8 +227,6 @@ export FIRST_USER_NAME=${FIRST_USER_NAME:-pi} export FIRST_USER_PASS export DISABLE_FIRST_BOOT_USER_RENAME=${DISABLE_FIRST_BOOT_USER_RENAME:-0} export RELEASE=${RELEASE:-bookworm} # Don't forget to update stage0/prerun.sh -export WPA_ESSID -export WPA_PASSWORD export WPA_COUNTRY export ENABLE_SSH="${ENABLE_SSH:-0}" export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}" diff --git a/stage2/02-net-tweaks/00-packages b/stage2/02-net-tweaks/00-packages index f0125f6a6d..d446837df7 100644 --- a/stage2/02-net-tweaks/00-packages +++ b/stage2/02-net-tweaks/00-packages @@ -1,5 +1,4 @@ wpasupplicant wireless-tools firmware-atheros firmware-brcm80211 firmware-libertas firmware-misc-nonfree firmware-realtek raspberrypi-net-mods -dhcpcd5 network-manager net-tools diff --git a/stage2/02-net-tweaks/01-run.sh b/stage2/02-net-tweaks/01-run.sh index 3b06c0577e..d49bdebfcd 100755 --- a/stage2/02-net-tweaks/01-run.sh +++ b/stage2/02-net-tweaks/01-run.sh @@ -3,32 +3,12 @@ install -v -d "${ROOTFS_DIR}/etc/wpa_supplicant" install -v -m 600 files/wpa_supplicant.conf "${ROOTFS_DIR}/etc/wpa_supplicant/" -on_chroot << EOF - SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_boot_wait 0 - SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_netconf 1 -EOF - if [ -v WPA_COUNTRY ]; then on_chroot <<- EOF SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_wifi_country "${WPA_COUNTRY}" EOF fi -if [ -v WPA_ESSID ] && [ -v WPA_PASSWORD ]; then -on_chroot <<EOF -set -o pipefail -wpa_passphrase "${WPA_ESSID}" "${WPA_PASSWORD}" | tee -a "/etc/wpa_supplicant/wpa_supplicant.conf" -EOF -elif [ -v WPA_ESSID ]; then -cat >> "${ROOTFS_DIR}/etc/wpa_supplicant/wpa_supplicant.conf" << EOL - -network={ - ssid="${WPA_ESSID}" - key_mgmt=NONE -} -EOL -fi - # Disable wifi on 5GHz models if WPA_COUNTRY is not set mkdir -p "${ROOTFS_DIR}/var/lib/systemd/rfkill/" if [ -n "$WPA_COUNTRY" ]; then diff --git a/stage3/01-tweaks/00-run.sh b/stage3/01-tweaks/00-run.sh deleted file mode 100755 index 79b749e397..0000000000 --- a/stage3/01-tweaks/00-run.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -e - -on_chroot << EOF - SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_boot_wait 1 -EOF From 4a4a66ed3661a241558beda050b1bf4d6ce49128 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 13:40:09 +0100 Subject: [PATCH 125/222] export-image: auto-remove and purge packages on dist-upgrade --- export-image/02-set-sources/01-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export-image/02-set-sources/01-run.sh b/export-image/02-set-sources/01-run.sh index 5f512099b3..6d5049613c 100755 --- a/export-image/02-set-sources/01-run.sh +++ b/export-image/02-set-sources/01-run.sh @@ -4,6 +4,6 @@ rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache" find "${ROOTFS_DIR}/var/lib/apt/lists/" -type f -delete on_chroot << EOF apt-get update -apt-get -y dist-upgrade +apt-get -y dist-upgrade --auto-remove --purge apt-get clean EOF From 88ac5235d62135806157183c466fdea202ce38f8 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 13:42:31 +0100 Subject: [PATCH 126/222] Enable Wayland by default --- stage4/06-enable-wayland/00-run.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 stage4/06-enable-wayland/00-run.sh diff --git a/stage4/06-enable-wayland/00-run.sh b/stage4/06-enable-wayland/00-run.sh new file mode 100755 index 0000000000..bd8bbc3bcc --- /dev/null +++ b/stage4/06-enable-wayland/00-run.sh @@ -0,0 +1,5 @@ +#!/bin/bash -e + +on_chroot << EOF + SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_wayland W2 +EOF From c856800c72c8a58e17d7ee20295605d479b00dff Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 14:08:13 +0100 Subject: [PATCH 127/222] Switch to new kernel and firmware packages --- export-image/04-set-partuuid/00-run.sh | 3 +- export-image/05-finalise/01-run.sh | 5 +- export-image/prerun.sh | 12 ++-- stage0/00-configure-apt/00-run.sh | 1 + stage0/02-firmware/01-packages | 8 ++- stage0/02-firmware/02-run.sh | 3 + stage1/00-boot-files/00-run.sh | 17 ++++- stage1/00-boot-files/files/config.txt | 62 +++++-------------- stage1/01-sys-tweaks/files/fstab | 2 +- .../00-patches/07-resize-init.diff | 4 +- 10 files changed, 53 insertions(+), 64 deletions(-) create mode 100755 stage0/02-firmware/02-run.sh diff --git a/export-image/04-set-partuuid/00-run.sh b/export-image/04-set-partuuid/00-run.sh index 16e1b158ef..2694295c3c 100755 --- a/export-image/04-set-partuuid/00-run.sh +++ b/export-image/04-set-partuuid/00-run.sh @@ -12,7 +12,6 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" - sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" - + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/firmware/cmdline.txt" fi diff --git a/export-image/05-finalise/01-run.sh b/export-image/05-finalise/01-run.sh index c104366f6e..eae7d6e901 100755 --- a/export-image/05-finalise/01-run.sh +++ b/export-image/05-finalise/01-run.sh @@ -3,7 +3,10 @@ IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" INFO_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.info" +sed -i 's/^update_initramfs=.*/update_initramfs=all/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" + on_chroot << EOF +update-initramfs -u if [ -x /etc/init.d/fake-hwclock ]; then /etc/init.d/fake-hwclock stop fi @@ -53,7 +56,7 @@ rm -f "${ROOTFS_DIR}/root/.vnc/private.key" rm -f "${ROOTFS_DIR}/etc/vnc/updateid" update_issue "$(basename "${EXPORT_DIR}")" -install -m 644 "${ROOTFS_DIR}/etc/rpi-issue" "${ROOTFS_DIR}/boot/issue.txt" +install -m 644 "${ROOTFS_DIR}/etc/rpi-issue" "${ROOTFS_DIR}/boot/firmware/issue.txt" cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" diff --git a/export-image/prerun.sh b/export-image/prerun.sh index 267bbe0322..a5f94e93ac 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -10,8 +10,8 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then rm -rf "${ROOTFS_DIR}" mkdir -p "${ROOTFS_DIR}" - BOOT_SIZE="$((256 * 1024 * 1024))" - ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) + BOOT_SIZE="$((512 * 1024 * 1024))" + ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot/firmware --block-size=1 | cut -f 1) # All partition sizes and starts will be aligned to this size ALIGN="$((4 * 1024 * 1024))" @@ -59,9 +59,9 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 - mkdir -p "${ROOTFS_DIR}/boot" - mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat + mkdir -p "${ROOTFS_DIR}/boot/firmware" + mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot/firmware" -t vfat - rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" - rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" + rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot/firmware "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" + rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/firmware/" "${ROOTFS_DIR}/boot/firmware/" fi diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 6494a03d05..fa4f59f435 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -15,6 +15,7 @@ fi cat files/raspberrypi.gpg.key | gpg --dearmor > "${STAGE_WORK_DIR}/raspberrypi-archive-stable.gpg" install -m 644 "${STAGE_WORK_DIR}/raspberrypi-archive-stable.gpg" "${ROOTFS_DIR}/etc/apt/trusted.gpg.d/" on_chroot << EOF +dpkg --add-architecture arm64 apt-get update apt-get dist-upgrade -y EOF diff --git a/stage0/02-firmware/01-packages b/stage0/02-firmware/01-packages index b166aa8dff..150958a287 100644 --- a/stage0/02-firmware/01-packages +++ b/stage0/02-firmware/01-packages @@ -1,2 +1,6 @@ -raspberrypi-bootloader -raspberrypi-kernel +initramfs-tools +raspi-firmware +linux-image-rpi-v6 +linux-image-rpi-v7 +linux-image-rpi-v7l +linux-image-rpi-v8 diff --git a/stage0/02-firmware/02-run.sh b/stage0/02-firmware/02-run.sh new file mode 100755 index 0000000000..0b2bca9625 --- /dev/null +++ b/stage0/02-firmware/02-run.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e + +sed -i 's/^update_initramfs=.*/update_initramfs=no/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" diff --git a/stage1/00-boot-files/00-run.sh b/stage1/00-boot-files/00-run.sh index bc61397d9d..ddb0f5b22b 100755 --- a/stage1/00-boot-files/00-run.sh +++ b/stage1/00-boot-files/00-run.sh @@ -1,4 +1,17 @@ #!/bin/bash -e -install -m 644 files/cmdline.txt "${ROOTFS_DIR}/boot/" -install -m 644 files/config.txt "${ROOTFS_DIR}/boot/" +mkdir -p "${ROOTFS_DIR}/boot/firmware" + +if ! [ -L "${ROOTFS_DIR}/boot/overlays" ]; then + ln -s firmware/overlays "${ROOTFS_DIR}/boot/overlays" +fi + +install -m 644 files/cmdline.txt "${ROOTFS_DIR}/boot/firmware/" +if ! [ -L "${ROOTFS_DIR}/boot/cmdline.txt" ]; then + ln -s firmware/cmdline.txt "${ROOTFS_DIR}/boot/cmdline.txt" +fi + +install -m 644 files/config.txt "${ROOTFS_DIR}/boot/firmware/" +if ! [ -L "${ROOTFS_DIR}/boot/config.txt" ]; then + ln -s firmware/config.txt "${ROOTFS_DIR}/boot/config.txt" +fi diff --git a/stage1/00-boot-files/files/config.txt b/stage1/00-boot-files/files/config.txt index 6e072cb999..621637afda 100644 --- a/stage1/00-boot-files/files/config.txt +++ b/stage1/00-boot-files/files/config.txt @@ -1,70 +1,41 @@ # For more options and information see -# http://rpf.io/configtxt +# http://rptl.io/configtxt # Some settings may impact device functionality. See link above for details -# uncomment if you get no picture on HDMI for a default "safe" mode -#hdmi_safe=1 - -# uncomment the following to adjust overscan. Use positive numbers if console -# goes off screen, and negative if there is too much border -#overscan_left=16 -#overscan_right=16 -#overscan_top=16 -#overscan_bottom=16 - -# uncomment to force a console size. By default it will be display's size minus -# overscan. -#framebuffer_width=1280 -#framebuffer_height=720 - -# uncomment if hdmi display is not detected and composite is being output -#hdmi_force_hotplug=1 - -# uncomment to force a specific HDMI mode (this will force VGA) -#hdmi_group=1 -#hdmi_mode=1 - -# uncomment to force a HDMI mode rather than DVI. This can make audio work in -# DMT (computer monitor) modes -#hdmi_drive=2 - -# uncomment to increase signal to HDMI, if you have interference, blanking, or -# no display -#config_hdmi_boost=4 - -# uncomment for composite PAL -#sdtv_mode=2 - -#uncomment to overclock the arm. 700 MHz is the default. -#arm_freq=800 - # Uncomment some or all of these to enable the optional hardware interfaces #dtparam=i2c_arm=on #dtparam=i2s=on #dtparam=spi=on -# Uncomment this to enable infrared communication. -#dtoverlay=gpio-ir,gpio_pin=17 -#dtoverlay=gpio-ir-tx,gpio_pin=18 - -# Additional overlays and parameters are documented /boot/overlays/README - # Enable audio (loads snd_bcm2835) dtparam=audio=on +# Additional overlays and parameters are documented +# /boot/firmware/overlays/README + # Automatically load overlays for detected cameras camera_auto_detect=1 # Automatically load overlays for detected DSI displays display_auto_detect=1 +# Automatically load initramfs files, if found +auto_initramfs=1 + # Enable DRM VC4 V3D driver dtoverlay=vc4-kms-v3d max_framebuffers=2 +# Don't have the firmware create an initial video= setting in cmdline.txt. +# Use the kernel's default instead. +disable_fw_kms_setup=1 + # Disable compensation for displays with overscan disable_overscan=1 +# Run as fast as firmware / board allows +arm_boost=1 + [cm4] # Enable host mode on the 2711 built-in XHCI USB controller. # This line should be removed if the legacy DWC2 controller is required @@ -73,8 +44,3 @@ otg_mode=1 [all] -[pi4] -# Run as fast as firmware / board allows -arm_boost=1 - -[all] diff --git a/stage1/01-sys-tweaks/files/fstab b/stage1/01-sys-tweaks/files/fstab index f16e3fb8aa..525c3bc84c 100644 --- a/stage1/01-sys-tweaks/files/fstab +++ b/stage1/01-sys-tweaks/files/fstab @@ -1,3 +1,3 @@ proc /proc proc defaults 0 0 -BOOTDEV /boot vfat defaults 0 2 +BOOTDEV /boot/firmware vfat defaults 0 2 ROOTDEV / ext4 defaults,noatime 0 1 diff --git a/stage2/01-sys-tweaks/00-patches/07-resize-init.diff b/stage2/01-sys-tweaks/00-patches/07-resize-init.diff index dfc01d4950..e508a9f525 100644 --- a/stage2/01-sys-tweaks/00-patches/07-resize-init.diff +++ b/stage2/01-sys-tweaks/00-patches/07-resize-init.diff @@ -1,5 +1,5 @@ ---- stage2.orig/rootfs/boot/cmdline.txt -+++ stage2/rootfs/boot/cmdline.txt +--- stage2.orig/rootfs/boot/firmware/cmdline.txt ++++ stage2/rootfs/boot/firmware/cmdline.txt @@ -1 +1 @@ -console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 fsck.repair=yes rootwait +console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 fsck.repair=yes rootwait quiet init=/usr/lib/raspberrypi-sys-mods/firstboot From 64b498b2dd458937cfc930d557513ff36db4dd23 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 14:59:28 +0100 Subject: [PATCH 128/222] Don't generate NOOBS files --- stage2/EXPORT_NOOBS | 2 -- stage4/EXPORT_NOOBS | 2 -- stage5/EXPORT_NOOBS | 2 -- 3 files changed, 6 deletions(-) delete mode 100644 stage2/EXPORT_NOOBS delete mode 100644 stage4/EXPORT_NOOBS delete mode 100644 stage5/EXPORT_NOOBS diff --git a/stage2/EXPORT_NOOBS b/stage2/EXPORT_NOOBS deleted file mode 100644 index 0d78ecfd03..0000000000 --- a/stage2/EXPORT_NOOBS +++ /dev/null @@ -1,2 +0,0 @@ -NOOBS_NAME="Raspberry Pi OS Lite (32-bit)" -NOOBS_DESCRIPTION="A port of Debian with no desktop environment" diff --git a/stage4/EXPORT_NOOBS b/stage4/EXPORT_NOOBS deleted file mode 100644 index f197ee1db7..0000000000 --- a/stage4/EXPORT_NOOBS +++ /dev/null @@ -1,2 +0,0 @@ -NOOBS_NAME="Raspberry Pi OS (32-bit)" -NOOBS_DESCRIPTION="A port of Debian with the Raspberry Pi Desktop" diff --git a/stage5/EXPORT_NOOBS b/stage5/EXPORT_NOOBS deleted file mode 100644 index ac7051f0c5..0000000000 --- a/stage5/EXPORT_NOOBS +++ /dev/null @@ -1,2 +0,0 @@ -NOOBS_NAME="Raspberry Pi OS Full (32-bit)" -NOOBS_DESCRIPTION="A port of Debian with desktop and recommended applications" From 188c155717b68dbd5704f337fd3b2cbd1b80d966 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 8 Sep 2023 11:39:51 +0100 Subject: [PATCH 129/222] Migrate to raspberrypi.com --- stage0/00-configure-apt/files/raspi.list | 4 ++-- stage0/00-configure-apt/files/sources.list | 4 ++-- stage0/prerun.sh | 2 +- stage4/03-bookshelf/00-run.sh | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stage0/00-configure-apt/files/raspi.list b/stage0/00-configure-apt/files/raspi.list index 41c5a70755..5f889b16a8 100644 --- a/stage0/00-configure-apt/files/raspi.list +++ b/stage0/00-configure-apt/files/raspi.list @@ -1,3 +1,3 @@ -deb http://archive.raspberrypi.org/debian/ RELEASE main +deb http://archive.raspberrypi.com/debian/ RELEASE main # Uncomment line below then 'apt-get update' to enable 'apt-get source' -#deb-src http://archive.raspberrypi.org/debian/ RELEASE main +#deb-src http://archive.raspberrypi.com/debian/ RELEASE main diff --git a/stage0/00-configure-apt/files/sources.list b/stage0/00-configure-apt/files/sources.list index 4a5dd8a9fd..cd339c5e10 100644 --- a/stage0/00-configure-apt/files/sources.list +++ b/stage0/00-configure-apt/files/sources.list @@ -1,3 +1,3 @@ -deb http://raspbian.raspberrypi.org/raspbian/ RELEASE main contrib non-free rpi +deb http://raspbian.raspberrypi.com/raspbian/ RELEASE main contrib non-free rpi # Uncomment line below then 'apt-get update' to enable 'apt-get source' -#deb-src http://raspbian.raspberrypi.org/raspbian/ RELEASE main contrib non-free rpi +#deb-src http://raspbian.raspberrypi.com/raspbian/ RELEASE main contrib non-free rpi diff --git a/stage0/prerun.sh b/stage0/prerun.sh index c8e51a426e..0252071470 100755 --- a/stage0/prerun.sh +++ b/stage0/prerun.sh @@ -6,5 +6,5 @@ if [ "$RELEASE" != "bookworm" ]; then fi if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then - bootstrap ${RELEASE} "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/ + bootstrap ${RELEASE} "${ROOTFS_DIR}" http://raspbian.raspberrypi.com/raspbian/ fi diff --git a/stage4/03-bookshelf/00-run.sh b/stage4/03-bookshelf/00-run.sh index 8244568bd6..5bf017b738 100755 --- a/stage4/03-bookshelf/00-run.sh +++ b/stage4/03-bookshelf/00-run.sh @@ -1,6 +1,6 @@ #!/bin/sh -e -BOOKSHELF_URL="https://magpi.raspberrypi.org/bookshelf.xml" +BOOKSHELF_URL="https://magpi.raspberrypi.com/bookshelf.xml" GUIDE_URL="$(curl -s "$BOOKSHELF_URL" | awk -F '[<>]' "/<TITLE>Raspberry Pi Beginner's Guide 4th Edition<\/TITLE>/ {f=1; next} f==1 && /PDF/ {print \$3; exit}")" OUTPUT="$(basename "$GUIDE_URL" | cut -f1 -d'?')" From e6dd7caa2b849cc22db42abb84b1a93c0818db39 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 12 Sep 2023 18:04:27 +0100 Subject: [PATCH 130/222] stage2: remove rng-tools See https://github.com/RPi-Distro/raspberrypi-sys-mods/pull/77 --- stage2/01-sys-tweaks/00-packages | 1 - 1 file changed, 1 deletion(-) diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 86bdd9d351..64529c7f06 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -24,7 +24,6 @@ htop man-db policykit-1 ssh-import-id -rng-tools ethtool ntfs-3g pciutils From 574d3e705535ac4fc47fd3f801d47fcc07184a63 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 13 Sep 2023 13:20:51 +0100 Subject: [PATCH 131/222] stage0: only fetch armhf packages from Raspbian --- stage0/00-configure-apt/files/sources.list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage0/00-configure-apt/files/sources.list b/stage0/00-configure-apt/files/sources.list index cd339c5e10..3ef0699311 100644 --- a/stage0/00-configure-apt/files/sources.list +++ b/stage0/00-configure-apt/files/sources.list @@ -1,3 +1,3 @@ -deb http://raspbian.raspberrypi.com/raspbian/ RELEASE main contrib non-free rpi +deb [ arch=armhf ] http://raspbian.raspberrypi.com/raspbian/ RELEASE main contrib non-free rpi # Uncomment line below then 'apt-get update' to enable 'apt-get source' #deb-src http://raspbian.raspberrypi.com/raspbian/ RELEASE main contrib non-free rpi From d7dde0ca5e02a7db400878f17267fde5f60429d4 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Sep 2023 15:42:20 +0100 Subject: [PATCH 132/222] stage5: disable greenfoot and bluej as they don't seem to work --- stage5/00-install-extras/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage5/00-install-extras/00-packages b/stage5/00-install-extras/00-packages index 909480b296..331f958354 100644 --- a/stage5/00-install-extras/00-packages +++ b/stage5/00-install-extras/00-packages @@ -7,7 +7,7 @@ python3-sense-emu sense-emu-tools python-sense-emu-doc wolfram-engine claws-mail -greenfoot-unbundled bluej-unbundled +#greenfoot-unbundled bluej-unbundled realvnc-vnc-viewer code-the-classics From 6051f1da7669875baf7e3c74daf67cb795c2a16e Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 13 Sep 2023 16:00:39 +0100 Subject: [PATCH 133/222] stage4: remove pimoroni packages for now --- stage4/00-install-packages/01-packages | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 stage4/00-install-packages/01-packages diff --git a/stage4/00-install-packages/01-packages b/stage4/00-install-packages/01-packages deleted file mode 100644 index 5441d9dd16..0000000000 --- a/stage4/00-install-packages/01-packages +++ /dev/null @@ -1,22 +0,0 @@ -python3-automationhat -python3-blinkt -python3-cap1xxx -python3-drumhat -python3-envirophat -python3-explorerhat -python3-fourletterphat -python3-microdotphat -python3-mote -python3-motephat -python3-phatbeat -python3-pianohat -python3-piglow -python3-rainbowhat -python3-scrollphat -python3-scrollphathd -python3-sn3218 -python3-skywriter -python3-touchphat -python3-buttonshim -python3-unicornhathd -python3-pantilthat From 3a6342b63d21d2a3385d99fa1e8c0fa9ca55b94c Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 14 Sep 2023 14:49:03 +0100 Subject: [PATCH 134/222] stage0: install kernel headers --- stage0/02-firmware/01-packages | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stage0/02-firmware/01-packages b/stage0/02-firmware/01-packages index 150958a287..38f2f52262 100644 --- a/stage0/02-firmware/01-packages +++ b/stage0/02-firmware/01-packages @@ -4,3 +4,6 @@ linux-image-rpi-v6 linux-image-rpi-v7 linux-image-rpi-v7l linux-image-rpi-v8 +linux-headers-rpi-v6 +linux-headers-rpi-v7 +linux-headers-rpi-v7l From 7ae1a26ae93991a25d39217c07f0b6656319a5e1 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 14 Sep 2023 14:50:07 +0100 Subject: [PATCH 135/222] stage4: ignore power key and let raspberrypi-ui-mods handle it --- stage4/07-disable-pwr-button/00-run.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 stage4/07-disable-pwr-button/00-run.sh diff --git a/stage4/07-disable-pwr-button/00-run.sh b/stage4/07-disable-pwr-button/00-run.sh new file mode 100755 index 0000000000..44646bd122 --- /dev/null +++ b/stage4/07-disable-pwr-button/00-run.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e + +sed -i 's/^.*HandlePowerKey=.*$/HandlePowerKey=ignore/' "${ROOTFS_DIR}/etc/systemd/logind.conf" From c4f98a873fd622c4ae01a65f38b1b11c8259acf5 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 23 Aug 2023 12:34:23 +0100 Subject: [PATCH 136/222] Move picamera2 to stage3 - PyAV takes up too much space for lite images --- stage2/01-sys-tweaks/00-packages-nr | 1 - stage3/00-install-packages/00-packages | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/01-sys-tweaks/00-packages-nr b/stage2/01-sys-tweaks/00-packages-nr index 453ada8be2..6bdb60069e 100644 --- a/stage2/01-sys-tweaks/00-packages-nr +++ b/stage2/01-sys-tweaks/00-packages-nr @@ -1,4 +1,3 @@ cifs-utils libcamera-apps-lite mkvtoolnix -python3-picamera2 diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index 2d206cd17e..cc18081907 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -14,5 +14,6 @@ obconf arandr libcamera-tools libcamera-apps +python3-picamera2 python3-pyqt5 python3-opengl From 5b15e14ff930c97ce1365d36b7696877d124b878 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 25 Sep 2023 21:33:15 +0100 Subject: [PATCH 137/222] stage2: install GPIO libraries --- stage2/01-sys-tweaks/00-packages | 5 ++++- stage4/00-install-packages/00-packages | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 64529c7f06..6142e2eb1b 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -2,8 +2,11 @@ ssh less fbset sudo psmisc strace ed ncdu console-setup keyboard-configuration debconf-utils parted build-essential manpages-dev bash-completion gdb pkg-config python-is-python3 -python3-rpi.gpio v4l-utils +v4l-utils +python3-libgpiod python3-gpiozero +pigpio python3-pigpio raspi-gpio python3-rpi.gpio +python3-spidev avahi-daemon lua5.1 luajit diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index bfb3f0d587..567f3ce795 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -11,8 +11,6 @@ tree libgl1-mesa-dri libgles1 libgles2-mesa xcompmgr geany piclone -pigpio python3-pigpio raspi-gpio python3-rpi.gpio -python3-spidev python3-twython python3-smbus python3-flask From dc6a3c1dd869f07c5fa7a61d667f09b81c66fb4a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 26 Sep 2023 14:19:01 +0100 Subject: [PATCH 138/222] stage2: install python3-smbus2 --- stage2/01-sys-tweaks/00-packages | 1 + stage4/00-install-packages/00-packages | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 6142e2eb1b..0e6bfb93e8 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -7,6 +7,7 @@ python3-libgpiod python3-gpiozero pigpio python3-pigpio raspi-gpio python3-rpi.gpio python3-spidev +python3-smbus2 avahi-daemon lua5.1 luajit diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index 567f3ce795..23077524c3 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -12,7 +12,6 @@ libgl1-mesa-dri libgles1 libgles2-mesa xcompmgr geany piclone python3-twython -python3-smbus python3-flask pprompt piwiz From a0b8868fbfd04da73d449b84773b7020bbf7c62b Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 27 Sep 2023 15:56:37 +0100 Subject: [PATCH 139/222] Update release notes --- .../00-release/files/release_notes.txt | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 7ec34fde2c..1eeffe9d33 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,26 @@ UNRELEASED: + * Based on Debian bookworm release + * Support for Raspberry Pi 5 + * Desktop now runs on the Wayfire Wayland compositing window manager on Raspberry Pi 4 and 5 platforms; on X11 using the openbox window manager on older platforms + * lxpanel replaced with new wf-panel-pi application when running Wayland; existing lxpanel plugins migrated; gpu performance and power plugins added + * pcmanfm modified to use Wayland backend when running on Wayland + * PipeWire used instead of PulseAudio as audio control subsystem; various changes made to volume control plugin to support this + * NetworkManager used instead of dhcpcd as networking interface; various changes made to networking plugin to support this + * Firefox browser added as alternative to Chromium; selection of default browser added to Raspberry Pi Configuration tool + * WayVNC VNC server used instead of RealVNC when running on Wayland + * All customisation and configuration applications modified to customise Wayfire environment as appropriate + * grim used as screenshot tool instead of scrot when running on Wayland + * eom image viewer used instead of gpicview + * evince document viewer used instead of qpdfview + * Chromium web browser updated to version 116 + * VLC media player updated to version 3.0.18 + * Magnifier program not available when running Wayland; Wayfire includes screen zoom capabilities + * CustomPC and Wireframe removed from Bookshelf + * Numerous small changes and bug fixes + * Switched from raspberrypi-kernel to Debian-based kernel packaging (linux-image-rpi-*) + * Switched from raspberrypi-bootloader to Debian based firmware packaging (raspi-firmware) + * /boot mount point moved to /boot/firmware +2023-05-03: * 64-bit Mathematica added to rp-prefapps * Bug fix - occasional segfault in CPU temperature plugin * Bug fix - X server crash when changing screen orientation From fb56ad562991cf3ae5c96ab50983e1deeaefc7b6 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 4 Oct 2023 00:05:53 +0100 Subject: [PATCH 140/222] stage5: match rp-prefapps package list --- stage5/00-install-extras/00-packages | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/stage5/00-install-extras/00-packages b/stage5/00-install-extras/00-packages index 331f958354..ad5dcad6dc 100644 --- a/stage5/00-install-extras/00-packages +++ b/stage5/00-install-extras/00-packages @@ -1,13 +1,8 @@ mu-editor -sonic-pi scratch nuscratch scratch3 smartsim - -python3-sense-emu sense-emu-tools python-sense-emu-doc - wolfram-engine claws-mail -#greenfoot-unbundled bluej-unbundled realvnc-vnc-viewer - code-the-classics +kicad From 663401de3cbbcf54d393195cb2aa7d06e5127ffd Mon Sep 17 00:00:00 2001 From: marcone <48169102+marcone@users.noreply.github.com> Date: Mon, 16 Oct 2023 04:41:17 -0700 Subject: [PATCH 141/222] README.md: default release is bookworm (#715) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ea276abc22..6369ac5c31 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ The following environment variables are supported: that the network block device is not disconnected correctly after the Docker process has ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong) -* `RELEASE` (Default: bullseye) +* `RELEASE` (Default: bookworm) The release version to build images against. Valid values are any supported Debian release. However, since different releases will have different sets of From 0ef867e5eb949e9340d5d812527578a7d10162a2 Mon Sep 17 00:00:00 2001 From: Stefan Becker <stefanb2@users.noreply.github.com> Date: Tue, 17 Oct 2023 16:42:07 +0300 Subject: [PATCH 142/222] Fix build-docker.sh CONTINUE=1 mode (#716) This fixes commit ed68013abbe46c0f665a157a9bdfd06603811101 Remove extra quotes so that docker sees pigen_work, not "pigen_work". --- build-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-docker.sh b/build-docker.sh index 3445a4069a..9549d90ab5 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -96,7 +96,7 @@ ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" if [ "${CONTAINER_EXISTS}" != "" ]; then DOCKER_CMDLINE_NAME="${CONTAINER_NAME}_cont" DOCKER_CMDLINE_PRE="--rm" - DOCKER_CMDLINE_POST="--volumes-from=\"${CONTAINER_NAME}\"" + DOCKER_CMDLINE_POST="--volumes-from=${CONTAINER_NAME}" else DOCKER_CMDLINE_NAME="${CONTAINER_NAME}" DOCKER_CMDLINE_PRE="" From d9668973950853ce3dc026246cad24fcbb5dbcfa Mon Sep 17 00:00:00 2001 From: Stefan Becker <stefanb2@users.noreply.github.com> Date: Mon, 23 Oct 2023 14:11:43 +0300 Subject: [PATCH 143/222] README.md: add PI_GEN_RELEASE (#720) This is the text written to "/etc/issue.txt" by update_issue(). The default is the name used by official Raspberry Pi builds. Fixes #719 --- README.md | 5 +++++ build.sh | 1 + scripts/common | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6369ac5c31..2fdb65e252 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,11 @@ The following environment variables are supported: but you should use something else for a customized version. Export files in stages may add suffixes to `IMG_NAME`. + * `PI_GEN_RELEASE` (Default: `Raspberry Pi reference`) + + The release name to use in `/etc/issue.txt`. The default should only be used + for official Raspberry Pi builds. + * `USE_QCOW2` **EXPERIMENTAL** (Default: `0` ) Instead of using traditional way of building the rootfs of every stage in diff --git a/build.sh b/build.sh index 22b605d822..7d7bc40368 100755 --- a/build.sh +++ b/build.sh @@ -194,6 +194,7 @@ trap term EXIT INT TERM export PI_GEN=${PI_GEN:-pi-gen} export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen} +export PI_GEN_RELEASE=${PI_GEN_RELEASE:-Raspberry Pi reference} if [ -z "${IMG_NAME}" ]; then echo "IMG_NAME not set" 1>&2 diff --git a/scripts/common b/scripts/common index c94b0f0abb..74c7938d52 100644 --- a/scripts/common +++ b/scripts/common @@ -99,7 +99,7 @@ on_chroot() { export -f on_chroot update_issue() { - echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using ${PI_GEN}, ${PI_GEN_REPO}, ${GIT_HASH}, ${1}" > "${ROOTFS_DIR}/etc/rpi-issue" + echo -e "${PI_GEN_RELEASE} ${IMG_DATE}\nGenerated using ${PI_GEN}, ${PI_GEN_REPO}, ${GIT_HASH}, ${1}" > "${ROOTFS_DIR}/etc/rpi-issue" } export -f update_issue From 19acff02387a56c68259c5315769110d90e3f656 Mon Sep 17 00:00:00 2001 From: Ponj <breval.ferrari@gmail.com> Date: Wed, 22 Nov 2023 14:59:13 +0100 Subject: [PATCH 144/222] Fix typo (#725) depoy > deploy --- build-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-docker.sh b/build-docker.sh index 9549d90ab5..83e76a587c 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -165,7 +165,7 @@ time ${DOCKER} run \ echo "copying results from deploy/" ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy - | tar -xf - -echo "copying log from container ${CONTAINER_NAME} to depoy/" +echo "copying log from container ${CONTAINER_NAME} to deploy/" ${DOCKER} logs --timestamps "${CONTAINER_NAME}" &>deploy/build-docker.log ls -lah deploy From 95c9fa8cb9c1b7c872d550824efa93628a50e733 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 27 Nov 2023 15:24:41 +0000 Subject: [PATCH 145/222] stage2: include gpiod binary tools --- stage2/01-sys-tweaks/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 0e6bfb93e8..55528ad070 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -3,7 +3,7 @@ console-setup keyboard-configuration debconf-utils parted build-essential manpages-dev bash-completion gdb pkg-config python-is-python3 v4l-utils -python3-libgpiod +gpiod python3-libgpiod python3-gpiozero pigpio python3-pigpio raspi-gpio python3-rpi.gpio python3-spidev From 5b22a553619c817d8ce563019d8a7a78eebb4966 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 27 Nov 2023 15:34:24 +0000 Subject: [PATCH 146/222] Update release notes --- .../00-release/files/release_notes.txt | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 1eeffe9d33..44778d126f 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,39 @@ UNRELEASED: + * Serial port switching in rc_gui and raspi-config modified to support Raspberry Pi 5 + * Touch screens now identified with unique per-device strings to enable correct association with display devices + * Compatibility with RP1 displays added + * Theme files monitored by pixdecor plugin to load changes on the fly + * Main menu shortcut to The Magpi website restored + * GTK+2 theme harmonised with GTK+3 theme to enable more uniform theming of Qt applications + * Battery monitor plugin enabled + * Taskbar Preferences menu item added to panel right-click menu + * Better reloading of on-the-fly theme changes + * Various improvements to Wayfire rendering + * Dark GTK theme added + * Bug fix - suppressed warning notifications when removing USB devices without mounted filesystems + * Bug fix - volume keyboard shortcuts locked up on some devices + * Bug fix - correctly handling multiple USB audio devices with same name + * Bug fix - some translations not loading in panel plugins + * Bug fix - window titlebars disappearing when tiled + * Bug fix - local installer service failing to install local deb files + * Bug fix - wizard not correctly setting locale when explicit UTF-8 character set required + * Bug fix - system updates could fail if an update required the removal of an installed package + * Bug fix - prevent file manager windows opening larger than screen size + * Bug fix - GPU plugin displaying load percentage as -1 + * Bug fix - various window manager crashes associated with shadows on certain window types + * Bug fix - allow VNC to be enabled if either RealVNC or WayVNC are installed + * WayVNC - improved support for encrypted connections + * Mathematica now works on Raspberry Pi 5 and 64-bit + * Scratch 3 now works on Raspberry Pi 5 and 64-bit + * Thonny updated to version 4.1.4 + * Chromium updated to 119.0.6045.171 + * Firefox updated to 119.0 + * gpiod binary tools included in lite images + * Japanese translations updated + * German translation added to Appearance Settings + * Raspberry Pi firmware 12af703dd07d7721c1f2f58c6f71d9fe66270838 + * Linux kernel 6.1.63 +2023-10-10: * Based on Debian bookworm release * Support for Raspberry Pi 5 * Desktop now runs on the Wayfire Wayland compositing window manager on Raspberry Pi 4 and 5 platforms; on X11 using the openbox window manager on older platforms From 193bae26590f9b457f72ca122b7d4fbf9ea168f8 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 28 Nov 2023 14:25:26 +0000 Subject: [PATCH 147/222] Remove obsolete packages --- stage1/03-install-packages/00-packages | 1 - stage2/01-sys-tweaks/00-packages | 1 - 2 files changed, 2 deletions(-) diff --git a/stage1/03-install-packages/00-packages b/stage1/03-install-packages/00-packages index a3d2fb66fd..97ca0d2bc6 100644 --- a/stage1/03-install-packages/00-packages +++ b/stage1/03-install-packages/00-packages @@ -1,2 +1 @@ -libraspberrypi-bin libraspberrypi0 systemd-timesyncd diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 55528ad070..abeaa6a4be 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -13,7 +13,6 @@ lua5.1 luajit ca-certificates curl fake-hwclock nfs-common usbutils -libraspberrypi-dev libraspberrypi-doc libfreetype6-dev dosfstools dphys-swapfile raspberrypi-sys-mods From 399a5c2bcd46ff4ee18a9652fceca579d317acc7 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 29 Nov 2023 01:29:33 +0000 Subject: [PATCH 148/222] stage2: install python3-venv --- stage2/01-sys-tweaks/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index abeaa6a4be..93a69be89f 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -36,3 +36,4 @@ udisks2 unzip zip p7zip-full file kms++-utils +python3-venv From ae818349453dea51fa26c66f0d616e3d48aed1e3 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 29 Nov 2023 01:32:10 +0000 Subject: [PATCH 149/222] export-image: Add issue.txt symlink --- export-image/05-finalise/01-run.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/export-image/05-finalise/01-run.sh b/export-image/05-finalise/01-run.sh index eae7d6e901..826198f0df 100755 --- a/export-image/05-finalise/01-run.sh +++ b/export-image/05-finalise/01-run.sh @@ -57,6 +57,10 @@ rm -f "${ROOTFS_DIR}/etc/vnc/updateid" update_issue "$(basename "${EXPORT_DIR}")" install -m 644 "${ROOTFS_DIR}/etc/rpi-issue" "${ROOTFS_DIR}/boot/firmware/issue.txt" +if ! [ -L "${ROOTFS_DIR}/boot/issue.txt" ]; then + ln -s firmware/issue.txt "${ROOTFS_DIR}/boot/issue.txt" +fi + cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" From e07ee9b98812355591de0b7a8c39762a7ca1ceda Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 29 Nov 2023 01:34:04 +0000 Subject: [PATCH 150/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 44778d126f..6507e59f86 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -29,6 +29,7 @@ UNRELEASED: * Chromium updated to 119.0.6045.171 * Firefox updated to 119.0 * gpiod binary tools included in lite images + * python3-venv included in lite images * Japanese translations updated * German translation added to Appearance Settings * Raspberry Pi firmware 12af703dd07d7721c1f2f58c6f71d9fe66270838 From 02b371de1a87bbe70cfcbd498527067a7d2ee6ca Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 29 Nov 2023 16:14:37 +0000 Subject: [PATCH 151/222] stage4: Update beginner's guide URL, assuming the latest version is always first --- stage4/03-bookshelf/00-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage4/03-bookshelf/00-run.sh b/stage4/03-bookshelf/00-run.sh index 5bf017b738..d62e70ead8 100755 --- a/stage4/03-bookshelf/00-run.sh +++ b/stage4/03-bookshelf/00-run.sh @@ -1,7 +1,7 @@ #!/bin/sh -e BOOKSHELF_URL="https://magpi.raspberrypi.com/bookshelf.xml" -GUIDE_URL="$(curl -s "$BOOKSHELF_URL" | awk -F '[<>]' "/<TITLE>Raspberry Pi Beginner's Guide 4th Edition<\/TITLE>/ {f=1; next} f==1 && /PDF/ {print \$3; exit}")" +GUIDE_URL="$(curl -s "$BOOKSHELF_URL" | awk -F '[<>]' "/<TITLE>Raspberry Pi Beginner's Guide .*<\/TITLE>/ {f=1; next} f==1 && /PDF/ {print \$3; exit}")" OUTPUT="$(basename "$GUIDE_URL" | cut -f1 -d'?')" if [ ! -f "files/$OUTPUT" ]; then From 1377e903b864673138908165572a18933400e8eb Mon Sep 17 00:00:00 2001 From: Vasilis Tsiligiannis <acinonyx@openwrt.gr> Date: Thu, 4 Jan 2024 13:05:51 +0200 Subject: [PATCH 152/222] Ensure loop device partition nodes are created (#741) Although the loop block device is created before attaching the image to it, the devices for the partition that the image contains are still not created. This patch creates those devices as well, when they are not already available. Fixes #482 Signed-off-by: Vasilis Tsiligiannis <acinonyx@openwrt.gr> --- export-image/prerun.sh | 1 + export-noobs/prerun.sh | 1 + scripts/common | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/export-image/prerun.sh b/export-image/prerun.sh index a5f94e93ac..d042c6ad5e 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -46,6 +46,7 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then fi done + ensure_loopdev_partitions "$LOOP_DEV" BOOT_DEV="${LOOP_DEV}p1" ROOT_DEV="${LOOP_DEV}p2" diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 62828368cb..80cbb8ee68 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -22,6 +22,7 @@ until ensure_next_loopdev && LOOP_DEV="$(losetup --show --find --partscan "$IMG_ fi done +ensure_loopdev_partitions "$LOOP_DEV" BOOT_DEV="${LOOP_DEV}p1" ROOT_DEV="${LOOP_DEV}p2" diff --git a/scripts/common b/scripts/common index 74c7938d52..cda5551302 100644 --- a/scripts/common +++ b/scripts/common @@ -110,3 +110,17 @@ ensure_next_loopdev() { [[ -b "$loopdev" ]] || mknod "$loopdev" b 7 "$loopmaj" } export -f ensure_next_loopdev + +ensure_loopdev_partitions() { + local line + local partition + local majmin + lsblk -r -n -o "NAME,MAJ:MIN" "$1" | grep -v "^${1#/dev/} " | while read -r line; do + partition="${line%% *}" + majmin="${line#* }" + if [ ! -b "/dev/$partition" ]; then + mknod "/dev/$partition" b "${majmin%:*}" "${majmin#*:}" + fi + done +} +export -f ensure_loopdev_partitions From c5fdb01eac6a802957ad1eed5b6b9d4f06f791c4 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 8 Dec 2023 07:58:19 +0000 Subject: [PATCH 153/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 6507e59f86..d305e33a06 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,6 @@ UNRELEASED: + * Fix Raspberry Pi Imager's WLAN configuration for lite images +2023-12-05: * Serial port switching in rc_gui and raspi-config modified to support Raspberry Pi 5 * Touch screens now identified with unique per-device strings to enable correct association with display devices * Compatibility with RP1 displays added From ec04064a552372b64a2cb7a1feb1bbda0c86ecac Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 18 Dec 2023 11:01:37 +0000 Subject: [PATCH 154/222] stage0: Only disable initramfs updates if update-initramfs.conf exists --- stage0/02-firmware/02-run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stage0/02-firmware/02-run.sh b/stage0/02-firmware/02-run.sh index 0b2bca9625..9cf9154cd4 100755 --- a/stage0/02-firmware/02-run.sh +++ b/stage0/02-firmware/02-run.sh @@ -1,3 +1,5 @@ #!/bin/bash -e -sed -i 's/^update_initramfs=.*/update_initramfs=no/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" +if [ -f "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" ]; then + sed -i 's/^update_initramfs=.*/update_initramfs=no/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" +fi From a143895f0a465ffcd20323264e648cba3f39ccc6 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 18 Dec 2023 11:04:13 +0000 Subject: [PATCH 155/222] stage0: Check whether foreign architecture is needed --- stage0/00-configure-apt/00-run.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index fa4f59f435..926c4c61c6 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -14,8 +14,13 @@ fi cat files/raspberrypi.gpg.key | gpg --dearmor > "${STAGE_WORK_DIR}/raspberrypi-archive-stable.gpg" install -m 644 "${STAGE_WORK_DIR}/raspberrypi-archive-stable.gpg" "${ROOTFS_DIR}/etc/apt/trusted.gpg.d/" -on_chroot << EOF -dpkg --add-architecture arm64 -apt-get update -apt-get dist-upgrade -y +on_chroot <<- \EOF + ARCH="$(dpkg --print-architecture)" + if [ "$ARCH" = "armhf" ]; then + dpkg --add-architecture arm64 + elif [ "$ARCH" = "arm64" ]; then + dpkg --add-architecture armhf + fi + apt-get update + apt-get dist-upgrade -y EOF From 30fa760eb166cfe8a8f1350b1cfd7d356c3753de Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 17 Jan 2024 17:30:10 +0000 Subject: [PATCH 156/222] stage0: prevent kernel packages from creating useless symlinks --- stage0/02-firmware/02-run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stage0/02-firmware/02-run.sh b/stage0/02-firmware/02-run.sh index 9cf9154cd4..677cd81765 100755 --- a/stage0/02-firmware/02-run.sh +++ b/stage0/02-firmware/02-run.sh @@ -3,3 +3,8 @@ if [ -f "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" ]; then sed -i 's/^update_initramfs=.*/update_initramfs=no/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" fi + +if [ ! -f "${ROOTFS_DIR}/etc/kernel-img.conf" ]; then + echo "do_symlinks=0" > "${ROOTFS_DIR}/etc/kernel-img.conf" +fi +rm -f "${ROOTFS_DIR}/"{vmlinuz,initrd.img}* From e682f62aed71be53fd8d49ac94c4d11f01996efe Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 17 Jan 2024 17:34:49 +0000 Subject: [PATCH 157/222] stage3: add vulkan support --- stage3/00-install-packages/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index cc18081907..9175bcca36 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -17,3 +17,4 @@ libcamera-apps python3-picamera2 python3-pyqt5 python3-opengl +vulkan-tools mesa-vulkan-drivers From a12534473182eb234a1751da7348f784b9271f42 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 22 Jan 2024 14:48:55 +0000 Subject: [PATCH 158/222] Remove cmdline.txt and config.txt symlinks --- stage1/00-boot-files/00-run.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/stage1/00-boot-files/00-run.sh b/stage1/00-boot-files/00-run.sh index ddb0f5b22b..8f1f823918 100755 --- a/stage1/00-boot-files/00-run.sh +++ b/stage1/00-boot-files/00-run.sh @@ -7,11 +7,8 @@ if ! [ -L "${ROOTFS_DIR}/boot/overlays" ]; then fi install -m 644 files/cmdline.txt "${ROOTFS_DIR}/boot/firmware/" -if ! [ -L "${ROOTFS_DIR}/boot/cmdline.txt" ]; then - ln -s firmware/cmdline.txt "${ROOTFS_DIR}/boot/cmdline.txt" -fi - install -m 644 files/config.txt "${ROOTFS_DIR}/boot/firmware/" -if ! [ -L "${ROOTFS_DIR}/boot/config.txt" ]; then - ln -s firmware/config.txt "${ROOTFS_DIR}/boot/config.txt" -fi + +for file in cmdline.txt config.txt; do + printf "DO NOT EDIT THIS FILE\n\nThe file you are looking for has moved to %s\n" "/boot/firmware/${file}" > "${ROOTFS_DIR}/boot/${file}" +done From 0425477d19bf6f8fdfff17b57deafff92029f998 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 23 Jan 2024 13:36:24 +0000 Subject: [PATCH 159/222] stage2: rc.local drop-in no longer required, since Debian ships its own version --- stage2/01-sys-tweaks/01-run.sh | 3 --- stage2/01-sys-tweaks/files/ttyoutput.conf | 2 -- 2 files changed, 5 deletions(-) delete mode 100644 stage2/01-sys-tweaks/files/ttyoutput.conf diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index 294a0ec36b..59988b2f89 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -2,9 +2,6 @@ install -m 755 files/resize2fs_once "${ROOTFS_DIR}/etc/init.d/" -install -d "${ROOTFS_DIR}/etc/systemd/system/rc-local.service.d" -install -m 644 files/ttyoutput.conf "${ROOTFS_DIR}/etc/systemd/system/rc-local.service.d/" - install -m 644 files/50raspi "${ROOTFS_DIR}/etc/apt/apt.conf.d/" install -m 644 files/console-setup "${ROOTFS_DIR}/etc/default/" diff --git a/stage2/01-sys-tweaks/files/ttyoutput.conf b/stage2/01-sys-tweaks/files/ttyoutput.conf deleted file mode 100644 index 6a396a48e0..0000000000 --- a/stage2/01-sys-tweaks/files/ttyoutput.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -StandardOutput=tty From c20ce09facaaadcc480a7bbda4fe100fcacdee12 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 30 Jan 2024 14:43:13 +0000 Subject: [PATCH 160/222] export-image: make sure initramfs is created Fixes #749 --- export-image/05-finalise/01-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export-image/05-finalise/01-run.sh b/export-image/05-finalise/01-run.sh index 826198f0df..3b56b20626 100755 --- a/export-image/05-finalise/01-run.sh +++ b/export-image/05-finalise/01-run.sh @@ -6,7 +6,7 @@ INFO_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.info" sed -i 's/^update_initramfs=.*/update_initramfs=all/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" on_chroot << EOF -update-initramfs -u +update-initramfs -k all -c if [ -x /etc/init.d/fake-hwclock ]; then /etc/init.d/fake-hwclock stop fi From 69037ce9fe9abe51cc6510971b3fb307745b1ef9 Mon Sep 17 00:00:00 2001 From: Tom Dewey <146750643+tdewey-rpi@users.noreply.github.com> Date: Mon, 19 Feb 2024 14:32:58 +0000 Subject: [PATCH 161/222] Update README.md (#755) Remove references to 'Raspbian', and instead refer to 'Raspberry Pi OS'. Additionally, a first pass of editing for conciseness. Replace bookworm instructions with bullseye --- README.md | 90 ++++++++++++++++++++++++++----------------------------- 1 file changed, 42 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 2fdb65e252..e19b2b3064 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,18 @@ # pi-gen -Tool used to create Raspberry Pi OS images. (Previously known as Raspbian). +Tool used to create Raspberry Pi OS images, and custom images based on Raspberry Pi OS, +which was in turn derived from the Raspbian project. +**Note**: Raspberry Pi OS 32 bit images are based primarily on Raspbian, while +Raspberry Pi OS 64 bit images are based primarily on Debian. ## Dependencies -pi-gen runs on Debian-based operating systems. Currently it is only supported on -either Debian Buster or Ubuntu Xenial and is known to have issues building on -earlier releases of these systems. On other Linux distributions it may be possible -to use the Docker build described below. +pi-gen runs on Debian-based operating systems released after 2017, and we +always advise you use the latest OS for security reasons. + +On other Linux distributions it may be possible to use the Docker build described +below. To install the required dependencies for `pi-gen` you should run: @@ -50,17 +54,16 @@ The following environment variables are supported: * `IMG_NAME` **required** (Default: unset) - The name of the image to build with the current stage directories. Setting - `IMG_NAME=Raspbian` is logical for an unmodified RPi-Distro/pi-gen build, - but you should use something else for a customized version. Export files - in stages may add suffixes to `IMG_NAME`. + The name of the image to build with the current stage directories. Use this + variable to set the root name of your OS, eg `IMG_NAME=Frobulator`. + Export files in stages may add suffixes to `IMG_NAME`. * `PI_GEN_RELEASE` (Default: `Raspberry Pi reference`) The release name to use in `/etc/issue.txt`. The default should only be used for official Raspberry Pi builds. -* `USE_QCOW2` **EXPERIMENTAL** (Default: `0` ) + * `USE_QCOW2` **EXPERIMENTAL** (Default: `0` ) Instead of using traditional way of building the rootfs of every stage in single subdirectories and copying over the previous one to the next one, @@ -73,14 +76,12 @@ The following environment variables are supported: * `BASE_QCOW2_SIZE` (Default: 12G) Size of the virtual qcow2 disk. - Note: it will not actually use that much of space at once but defines the - maximum size of the virtual disk. If you change the build process by adding - a lot of bigger packages or additional build stages, it can be necessary to - increase the value because the virtual disk can run out of space like a normal - hard drive would. - - **CAUTION:** Although the qcow2 build mechanism will run fine inside Docker, it can happen - that the network block device is not disconnected correctly after the Docker process has + Note: This is a maximum size - not a guaranteed allocation size. You may need + to increase the default size if you are building a particularly full image, containing + many packages or pre-built artefacts. + + **CAUTION:** Although the qcow2 build mechanism will run inside Docker, there is a known issue + where the network block device is not disconnected correctly after the Docker process has ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong) * `RELEASE` (Default: bookworm) @@ -89,7 +90,7 @@ The following environment variables are supported: Debian release. However, since different releases will have different sets of packages available, you'll need to either modify your stages accordingly, or checkout the appropriate branch. For example, if you'd like to build a - `buster` image, you should do so from the `buster` branch. + `bullseye` image, you should do so from the `bullseye` branch. * `APT_PROXY` (Default: unset) @@ -231,10 +232,10 @@ The following environment variables are supported: If set, then instead of working through the numeric stages in order, this list will be followed. For example setting to `"stage0 stage1 mystage stage2"` will run the contents of `mystage` before stage2. Note that quotes are needed around the list. An absolute or relative path can be given for stages outside the pi-gen directory. -A simple example for building Raspbian: +A simple example for building Raspberry Pi OS: ```bash -IMG_NAME='Raspbian' +IMG_NAME='raspios' ``` The config file can also be specified on the command line as an argument the `build.sh` or `build-docker.sh` scripts. @@ -249,17 +250,17 @@ This is parsed after `config` so can be used to override values set there. The following process is followed to build images: - * Loop through all of the stage directories in alphanumeric order + * Interate through all of the stage directories in alphanumeric order - * Move on to the next directory if this stage directory contains a file called + * Bypass a stage directory if it contains a file called "SKIP" * Run the script ```prerun.sh``` which is generally just used to copy the build directory between stages. - * In each stage directory loop through each subdirectory and then run each of the - install scripts it contains, again in alphanumeric order. These need to be named - with a two digit padded number at the beginning. + * In each stage directory iterate through each subdirectory and then run each of the + install scripts it contains, again in alphanumeric order. **These need to be named + with a two digit padded number at the beginning.** There are a number of different files and directories which can be used to control different parts of the build process: @@ -294,7 +295,7 @@ It is recommended to examine build.sh for finer details. Docker can be used to perform the build inside a container. This partially isolates the build from the host system, and allows using the script on non-debian based -systems (e.g. Fedora Linux). The isolate is not complete due to the need to use +systems (e.g. Fedora Linux). The isolation is not complete due to the need to use some kernel level services for arm emulation (binfmt) and loop devices (losetup). To build: @@ -307,7 +308,7 @@ vi config # Edit your config file. See above. If everything goes well, your finished image will be in the `deploy/` folder. You can then remove the build container with `docker rm -v pigen_work` -If something breaks along the line, you can edit the corresponding scripts, and +If you encounter errors during the build, you can edit the corresponding scripts, and continue: ```bash @@ -351,43 +352,36 @@ maintenance and allows for more easy customization. `debootstrap`, which creates a minimal filesystem suitable for use as a base.tgz on Debian systems. This stage also configures apt settings and installs `raspberrypi-bootloader` which is missed by debootstrap. The - minimal core is installed but not configured, and the system will not quite - boot yet. + minimal core is installed but not configured. As a result, this stage will not boot. - **Stage 1** - truly minimal system. This stage makes the system bootable by installing system files like `/etc/fstab`, configures the bootloader, makes the network operable, and installs packages like raspi-config. At this stage the system should boot to a local console from which you have the means to perform basic tasks needed to configure and install the system. - This is as minimal as a system can possibly get, and its arguably not - really usable yet in a traditional sense yet. Still, if you want minimal, - this is minimal and the rest you could reasonably do yourself as sysadmin. - - **Stage 2** - lite system. This stage produces the Raspbian-Lite image. It - installs some optimized memory functions, sets timezone and charmap + - **Stage 2** - lite system. This stage produces the Raspberry Pi OS Lite image. + Stage 2 installs some optimized memory functions, sets timezone and charmap defaults, installs fake-hwclock and ntp, wireless LAN and bluetooth support, dphys-swapfile, and other basics for managing the hardware. It also creates necessary groups and gives the pi user access to sudo and the standard console hardware permission groups. - There are a few tools that may not make a whole lot of sense here for - development purposes on a minimal system such as basic Python and Lua - packages as well as the `build-essential` package. They are lumped right - in with more essential packages presently, though they need not be with - pi-gen. These are understandable for Raspbian's target audience, but if - you were looking for something between truly minimal and Raspbian-Lite, - here's where you start trimming. + Note: Raspberry Pi OS Lite contains a number of tools for development, + including `Python`, `Lua` and the `build-essential` package. If you are + creating an image to deploy in products, be sure to remove extraneous development + tools before deployment. - **Stage 3** - desktop system. Here's where you get the full desktop system - with X11 and LXDE, web browsers, git for development, Raspbian custom UI + with X11 and LXDE, web browsers, git for development, Raspberry Pi OS custom UI enhancements, etc. This is a base desktop system, with some development tools installed. - - **Stage 4** - Normal Raspbian image. System meant to fit on a 4GB card. This is the - stage that installs most things that make Raspbian friendly to new - users like system documentation. + - **Stage 4** - Normal Raspberry Pi OS image. System meant to fit on a 4GB card. + This is the stage that installs most things that make Raspberry Pi OS friendly + to new users - e.g. system documentation. - - **Stage 5** - The Raspbian Full image. More development + - **Stage 5** - The Raspberry Pi OS Full image. More development tools, an email client, learning tools like Scratch, specialized packages like sonic-pi, office productivity, etc. @@ -402,7 +396,7 @@ to `./stage2` (if building a minimal system). ```bash # Example for building a lite system -echo "IMG_NAME='Raspbian'" > config +echo "IMG_NAME='raspios'" > config touch ./stage3/SKIP ./stage4/SKIP ./stage5/SKIP touch ./stage4/SKIP_IMAGES ./stage5/SKIP_IMAGES sudo ./build.sh # or ./build-docker.sh From 502e5fa0e3e1342ac7eedaea964b40ec55d7adb4 Mon Sep 17 00:00:00 2001 From: Mark Spatz <mark.h.spatz@gmail.com> Date: Thu, 22 Feb 2024 23:19:10 -0500 Subject: [PATCH 162/222] Remove QCOW2 build mechanism (#648) --- Dockerfile | 2 +- README.md | 88 +-------- build-docker.sh | 3 - build.sh | 131 +------------ depends | 2 - export-image/04-set-partuuid/00-run.sh | 18 +- export-image/05-finalise/01-run.sh | 23 +-- export-image/prerun.sh | 104 +++++----- export-noobs/00-release/00-run.sh | 6 +- export-noobs/prerun.sh | 4 - imagetool.sh | 114 ----------- scripts/qcow2_handling | 256 ------------------------- stage0/prerun.sh | 2 +- 13 files changed, 78 insertions(+), 675 deletions(-) delete mode 100755 imagetool.sh delete mode 100644 scripts/qcow2_handling diff --git a/Dockerfile b/Dockerfile index 2a5d8fe5f2..673f7564df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get -y update && \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ - binfmt-support ca-certificates qemu-utils kpartx fdisk gpg pigz\ + binfmt-support ca-certificates fdisk gpg pigz\ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ diff --git a/README.md b/README.md index e19b2b3064..3529770836 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ To install the required dependencies for `pi-gen` you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \ -qemu-utils kpartx gpg pigz +gpg pigz ``` The file `depends` contains a list of tools needed. The format of this @@ -63,27 +63,6 @@ The following environment variables are supported: The release name to use in `/etc/issue.txt`. The default should only be used for official Raspberry Pi builds. - * `USE_QCOW2` **EXPERIMENTAL** (Default: `0` ) - - Instead of using traditional way of building the rootfs of every stage in - single subdirectories and copying over the previous one to the next one, - qcow2 based virtual disks with backing images are used in every stage. - This speeds up the build process and reduces overall space consumption - significantly. - - <u>Additional optional parameters regarding qcow2 build:</u> - - * `BASE_QCOW2_SIZE` (Default: 12G) - - Size of the virtual qcow2 disk. - Note: This is a maximum size - not a guaranteed allocation size. You may need - to increase the default size if you are building a particularly full image, containing - many packages or pre-built artefacts. - - **CAUTION:** Although the qcow2 build mechanism will run inside Docker, there is a known issue - where the network block device is not disconnected correctly after the Docker process has - ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong) - * `RELEASE` (Default: bookworm) The release version to build images against. Valid values are any supported @@ -423,71 +402,6 @@ follows: * Once you're happy with the image you can remove the SKIP_IMAGES files and export your image to test -# Regarding Qcow2 image building - -### Get infos about the image in use - -If you issue the two commands shown in the example below in a second command shell while a build -is running you can find out, which network block device is currently being used and which qcow2 image -is bound to it. - -Example: - -```bash -root@build-machine:~/$ lsblk | grep nbd -nbd1 43:32 0 10G 0 disk -├─nbd1p1 43:33 0 10G 0 part -└─nbd1p1 253:0 0 10G 0 part - -root@build-machine:~/$ ps xa | grep qemu-nbd - 2392 pts/6 S+ 0:00 grep --color=auto qemu-nbd -31294 ? Ssl 0:12 qemu-nbd --discard=unmap -c /dev/nbd1 image-stage4.qcow2 -``` - -Here you can see, that the qcow2 image `image-stage4.qcow2` is currently connected to `/dev/nbd1` with -the associated partition map `/dev/mapper/nbd1p1`. Don't worry that `lsblk` shows two entries. It is totally fine, because the device map is accessible via `/dev/mapper/nbd1p1` and also via `/dev/dm-0`. This is all part of the device mapper functionality of the kernel. See `dmsetup` for further information. - -### Mount a qcow2 image - -If you want to examine the content of a a single stage, you can simply mount the qcow2 image found in the `WORK_DIR` directory with the tool `./imagetool.sh`. - -See `./imagetool.sh -h` for further details on how to use it. - -### Disconnect an image if something went wrong - -It can happen, that your build stops in case of an error. Normally `./build.sh` should handle image disconnection appropriately, but in rare cases, especially during a Docker build, this may not work as expected. If that happens, starting a new build will fail and you may have to disconnect the image and/or device yourself. - -A typical message indicating that there are some orphaned device mapper entries is this: - -``` -Failed to set NBD socket -Disconnect client, due to: Unexpected end-of-file before all bytes were read -``` - -If that happens go through the following steps: - -1. First, check if the image is somehow mounted to a directory entry and umount it as you would any other block device, like i.e. a hard disk or USB stick. - -2. Second, to disconnect an image from `qemu-nbd`, the QEMU Disk Network Block Device Server, issue the following command (be sure to change the device name to the one actually used): - - ```bash - sudo qemu-nbd -d /dev/nbd1 - ``` - - Note: if you use Docker build, normally no active `qemu-nbd` process exists anymore as it will be terminated when the Docker container stops. - -3. To disconnect a device partition map from the network block device, execute: - - ```bash - sudo kpartx -d /dev/nbd1 - or - sudo ./imagetool.sh --cleanup - ``` - - Note: The `imagetool.sh` command will cleanup any /dev/nbdX that is not connected to a running `qemu-nbd` daemon. Be careful if you use network block devices for other tasks utilizing NBDs on your build machine as well. - -Now you should be able to start a new build without running into troubles again. Most of the time, especially when using Docker build, you will only need no. 3 to get everything up and running again. - # Troubleshooting ## `64 Bit Systems` diff --git a/build-docker.sh b/build-docker.sh index 83e76a587c..41e341f59e 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -144,9 +144,6 @@ time ${DOCKER} run \ $DOCKER_CMDLINE_PRE \ --name "${DOCKER_CMDLINE_NAME}" \ --privileged \ - --cap-add=ALL \ - -v /dev:/dev \ - -v /lib/modules:/lib/modules \ ${PIGEN_DOCKER_OPTS} \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ diff --git a/build.sh b/build.sh index 7d7bc40368..f34adfce17 100755 --- a/build.sh +++ b/build.sh @@ -23,11 +23,6 @@ EOF on_chroot << EOF apt-get -o Acquire::Retries=3 install --no-install-recommends -y $PACKAGES EOF - if [ "${USE_QCOW2}" = "1" ]; then - on_chroot << EOF -apt-get clean -EOF - fi fi log "End ${SUB_STAGE_DIR}/${i}-packages-nr" fi @@ -38,11 +33,6 @@ EOF on_chroot << EOF apt-get -o Acquire::Retries=3 install -y $PACKAGES EOF - if [ "${USE_QCOW2}" = "1" ]; then - on_chroot << EOF -apt-get clean -EOF - fi fi log "End ${SUB_STAGE_DIR}/${i}-packages" fi @@ -99,16 +89,7 @@ run_stage(){ STAGE_WORK_DIR="${WORK_DIR}/${STAGE}" ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs - if [ "${USE_QCOW2}" = "1" ]; then - if [ ! -f SKIP ]; then - load_qimage - fi - else - # make sure we are not umounting during export-image stage - if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then - unmount "${WORK_DIR}/${STAGE}" - fi - fi + unmount "${WORK_DIR}/${STAGE}" if [ ! -f SKIP_IMAGES ]; then if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then @@ -116,7 +97,7 @@ run_stage(){ fi fi if [ ! -f SKIP ]; then - if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then + if [ "${CLEAN}" = "1" ]; then if [ -d "${ROOTFS_DIR}" ]; then rm -rf "${ROOTFS_DIR}" fi @@ -133,14 +114,7 @@ run_stage(){ done fi - if [ "${USE_QCOW2}" = "1" ]; then - unload_qimage - else - # make sure we are not umounting during export-image stage - if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then - unmount "${WORK_DIR}/${STAGE}" - fi - fi + unmount "${WORK_DIR}/${STAGE}" PREV_STAGE="${STAGE}" PREV_STAGE_DIR="${STAGE_DIR}" @@ -184,10 +158,7 @@ do done term() { - if [ "${USE_QCOW2}" = "1" ]; then - log "Unloading image" - unload_qimage - fi + true; #TODO: Cleanup } trap term EXIT INT TERM @@ -270,18 +241,6 @@ source "${SCRIPT_DIR}/common" # shellcheck source=scripts/dependencies_check source "${SCRIPT_DIR}/dependencies_check" -export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" -export USE_QCOW2="${USE_QCOW2:-0}" -export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G} -source "${SCRIPT_DIR}/qcow2_handling" -if [ "${USE_QCOW2}" = "1" ]; then - NO_PRERUN_QCOW2=1 -else - NO_PRERUN_QCOW2=0 -fi - -export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" - if [ "$SETFCAP" != "1" ]; then export CAPSH_ARG="--drop=cap_setfcap" fi @@ -336,98 +295,22 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_IMAGE" EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs - if [ "${USE_QCOW2}" = "1" ]; then - USE_QCOW2=0 - EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}" - echo "------------------------------------------------------------------------" - echo "Running export stage for ${EXPORT_NAME}" - rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true - rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true - rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true - rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true - EXPORT_STAGE=$(basename "${EXPORT_DIR}") - for s in $STAGE_LIST; do - TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}") - done - FIRST_STAGE=${TMP_LIST%% *} - FIRST_IMAGE="image-${FIRST_STAGE}.qcow2" - - pushd "${WORK_DIR}" > /dev/null - echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}" - cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2" - - ARR=($TMP_LIST) - # rebase stage images to new export base - for CURR_STAGE in "${ARR[@]}"; do - if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then - PREV_IMG="${EXPORT_NAME}" - continue - fi - echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" - qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 - if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then - break - fi - PREV_IMG="image-${CURR_STAGE}" - done - - # commit current export stage into base export image - echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2" - qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2 - - # rebase stage images back to original first stage for easy re-run - for CURR_STAGE in "${ARR[@]}"; do - if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then - PREV_IMG="image-${CURR_STAGE}" - continue - fi - echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" - qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 - if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then - break - fi - PREV_IMG="image-${CURR_STAGE}" - done - popd > /dev/null - - mkdir -p "${WORK_DIR}/export-image/rootfs" - mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/" - echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs" - mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs" - - CLEAN=0 - run_stage - CLEAN=1 - USE_QCOW2=1 - - else - run_stage - fi + run_stage if [ "${USE_QEMU}" != "1" ]; then if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_NOOBS" STAGE_DIR="${BASE_DIR}/export-noobs" - if [ "${USE_QCOW2}" = "1" ]; then - USE_QCOW2=0 - run_stage - USE_QCOW2=1 - else - run_stage - fi + run_stage fi fi done -if [ -x postrun.sh ]; then +if [ -x "${BASE_DIR}/postrun.sh" ]; then log "Begin postrun.sh" cd "${BASE_DIR}" ./postrun.sh log "End postrun.sh" fi -if [ "${USE_QCOW2}" = "1" ]; then - unload_qimage -fi - log "End ${BASE_DIR}" diff --git a/depends b/depends index db881712eb..10fcdd4718 100644 --- a/depends +++ b/depends @@ -17,7 +17,5 @@ file git lsmod:kmod bc -qemu-nbd:qemu-utils -kpartx gpg pigz diff --git a/export-image/04-set-partuuid/00-run.sh b/export-image/04-set-partuuid/00-run.sh index 2694295c3c..99500f36fb 100755 --- a/export-image/04-set-partuuid/00-run.sh +++ b/export-image/04-set-partuuid/00-run.sh @@ -1,17 +1,13 @@ #!/bin/bash -e -if [ "${NO_PRERUN_QCOW2}" = "0" ]; then +IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" - IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" - IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" +BOOT_PARTUUID="${IMGID}-01" +ROOT_PARTUUID="${IMGID}-02" - BOOT_PARTUUID="${IMGID}-01" - ROOT_PARTUUID="${IMGID}-02" - - sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" - sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" - - sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/firmware/cmdline.txt" -fi +sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" +sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" +sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/firmware/cmdline.txt" diff --git a/export-image/05-finalise/01-run.sh b/export-image/05-finalise/01-run.sh index 3b56b20626..8310e870a7 100755 --- a/export-image/05-finalise/01-run.sh +++ b/export-image/05-finalise/01-run.sh @@ -83,24 +83,17 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" dpkg -l --root "$ROOTFS_DIR" } >> "$INFO_FILE" -mkdir -p "${DEPLOY_DIR}" +ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" -rm -f "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.*" -rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +unmount "${ROOTFS_DIR}" +zerofree "${ROOT_DEV}" -mv "$INFO_FILE" "$DEPLOY_DIR/" +unmount_image "${IMG_FILE}" -if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then - ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" - - unmount "${ROOTFS_DIR}" - zerofree "${ROOT_DEV}" +mkdir -p "${DEPLOY_DIR}" - unmount_image "${IMG_FILE}" -else - unload_qimage - make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE" -fi +rm -f "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.*" +rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" case "${DEPLOY_COMPRESSION}" in zip) @@ -121,3 +114,5 @@ none | *) cp "$IMG_FILE" "$DEPLOY_DIR/" ;; esac + +cp "$INFO_FILE" "$DEPLOY_DIR/" diff --git a/export-image/prerun.sh b/export-image/prerun.sh index d042c6ad5e..b95fe3684b 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -1,68 +1,66 @@ #!/bin/bash -e -if [ "${NO_PRERUN_QCOW2}" = "0" ]; then - IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" - unmount_image "${IMG_FILE}" +unmount_image "${IMG_FILE}" - rm -f "${IMG_FILE}" +rm -f "${IMG_FILE}" - rm -rf "${ROOTFS_DIR}" - mkdir -p "${ROOTFS_DIR}" +rm -rf "${ROOTFS_DIR}" +mkdir -p "${ROOTFS_DIR}" - BOOT_SIZE="$((512 * 1024 * 1024))" - ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot/firmware --block-size=1 | cut -f 1) +BOOT_SIZE="$((512 * 1024 * 1024))" +ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot/firmware --block-size=1 | cut -f 1) - # All partition sizes and starts will be aligned to this size - ALIGN="$((4 * 1024 * 1024))" - # Add this much space to the calculated file size. This allows for - # some overhead (since actual space usage is usually rounded up to the - # filesystem block size) and gives some free space on the resulting - # image. - ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" +# All partition sizes and starts will be aligned to this size +ALIGN="$((4 * 1024 * 1024))" +# Add this much space to the calculated file size. This allows for +# some overhead (since actual space usage is usually rounded up to the +# filesystem block size) and gives some free space on the resulting +# image. +ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" - BOOT_PART_START=$((ALIGN)) - BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) - ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) - ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) - IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) +BOOT_PART_START=$((ALIGN)) +BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) +ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) +ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) +IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) - truncate -s "${IMG_SIZE}" "${IMG_FILE}" +truncate -s "${IMG_SIZE}" "${IMG_FILE}" - parted --script "${IMG_FILE}" mklabel msdos - parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" - parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" +parted --script "${IMG_FILE}" mklabel msdos +parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" +parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" - echo "Creating loop device..." - cnt=0 - until ensure_next_loopdev && LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do - if [ $cnt -lt 5 ]; then - cnt=$((cnt + 1)) - echo "Error in losetup. Retrying..." - sleep 5 - else - echo "ERROR: losetup failed; exiting" - exit 1 - fi - done - - ensure_loopdev_partitions "$LOOP_DEV" - BOOT_DEV="${LOOP_DEV}p1" - ROOT_DEV="${LOOP_DEV}p2" - - ROOT_FEATURES="^huge_file" - for FEATURE in 64bit; do - if grep -q "$FEATURE" /etc/mke2fs.conf; then - ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" +echo "Creating loop device..." +cnt=0 +until ensure_next_loopdev && LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do + if [ $cnt -lt 5 ]; then + cnt=$((cnt + 1)) + echo "Error in losetup. Retrying..." + sleep 5 + else + echo "ERROR: losetup failed; exiting" + exit 1 fi - done - mkdosfs -n bootfs -F 32 -s 4 -v "$BOOT_DEV" > /dev/null - mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null +done - mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 - mkdir -p "${ROOTFS_DIR}/boot/firmware" - mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot/firmware" -t vfat +ensure_loopdev_partitions "$LOOP_DEV" +BOOT_DEV="${LOOP_DEV}p1" +ROOT_DEV="${LOOP_DEV}p2" - rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot/firmware "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" - rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/firmware/" "${ROOTFS_DIR}/boot/firmware/" +ROOT_FEATURES="^huge_file" +for FEATURE in 64bit; do +if grep -q "$FEATURE" /etc/mke2fs.conf; then + ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" fi +done +mkdosfs -n bootfs -F 32 -s 4 -v "$BOOT_DEV" > /dev/null +mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null + +mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 +mkdir -p "${ROOTFS_DIR}/boot/firmware" +mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot/firmware" -t vfat + +rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot/firmware "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" +rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/firmware/" "${ROOTFS_DIR}/boot/firmware/" diff --git a/export-noobs/00-release/00-run.sh b/export-noobs/00-release/00-run.sh index bfaea9f812..513cbfd26d 100755 --- a/export-noobs/00-release/00-run.sh +++ b/export-noobs/00-release/00-run.sh @@ -41,8 +41,4 @@ sed "${NOOBS_DIR}/os.json" -i -e "s|KERNEL|$(cat "${STAGE_WORK_DIR}/kernel_versi sed "${NOOBS_DIR}/release_notes.txt" -i -e "s|UNRELEASED|${IMG_DATE}|" -if [ "${USE_QCOW2}" = "1" ]; then - mv "${NOOBS_DIR}" "${DEPLOY_DIR}/" -else - cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" -fi +cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 80cbb8ee68..8e55c9eca7 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -41,8 +41,4 @@ bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs/boot" -cpf - umount "${STAGE_WORK_DIR}/rootfs/boot" bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs" --one-file-system -cpf - . | xz -T0 > "${NOOBS_DIR}/root.tar.xz" -if [ "${USE_QCOW2}" = "1" ]; then - rm "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/apply_noobs_os_config.service" -fi - unmount_image "${IMG_FILE}" diff --git a/imagetool.sh b/imagetool.sh deleted file mode 100755 index 002b50bc34..0000000000 --- a/imagetool.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -if [ "$(id -u)" != "0" ]; then - echo "Please run as root" 1>&2 - exit 1 -fi - -progname=$(basename $0) - -function usage() -{ - cat << HEREDOC - -Usage: - Mount Image : $progname [--mount] [--image-name <path to qcow2 image>] [--mount-point <mount point>] - Umount Image: $progname [--umount] [--mount-point <mount point>] - Cleanup NBD : $progname [--cleanup] - - arguments: - -h, --help show this help message and exit - -c, --cleanup cleanup orphaned device mappings - -m, --mount mount image - -u, --umount umount image - -i, --image-name path to qcow2 image - -p, --mount-point mount point for image - - This tool will use /dev/nbd1 as default for mounting an image. If you want to use another device, execute like this: - NBD_DEV=/dev/nbd2 ./$progname --mount --image-name <your image> --mount-point <your path> - -HEREDOC -} - -MOUNT=0 -UMOUNT=0 -IMAGE="" -MOUNTPOINT="" - -nbd_cleanup() { - DEVS="$(lsblk | grep nbd | grep disk | cut -d" " -f1)" - if [ ! -z "${DEVS}" ]; then - for d in $DEVS; do - if [ ! -z "${d}" ]; then - QDEV="$(ps xa | grep $d | grep -v grep)" - if [ -z "${QDEV}" ]; then - kpartx -d /dev/$d && echo "Unconnected device map removed: /dev/$d" - fi - fi - done - fi -} - -# As long as there is at least one more argument, keep looping -while [[ $# -gt 0 ]]; do - key="$1" - case "$key" in - -h|--help) - usage - exit - ;; - -c|--cleanup) - nbd_cleanup - ;; - -m|--mount) - MOUNT=1 - ;; - -u|--umount) - UMOUNT=1 - ;; - -i|--image-name) - shift - IMAGE="$1" - ;; - -p|--mount-point) - shift - MOUNTPOINT="$1" - ;; - *) - echo "Unknown option '$key'" - usage - exit - ;; - esac - # Shift after checking all the cases to get the next option - shift -done - -if [ "${MOUNT}" = "1" ] && [ "${UMOUNT}" = "1" ]; then - usage - echo "Concurrent mount options not possible." - exit -fi - -if [ "${MOUNT}" = "1" ] && ([ -z "${IMAGE}" ] || [ -z "${MOUNTPOINT}" ]); then - usage - echo "Can not mount image. Image path and/or mount point missing." - exit -fi - -if [ "${UMOUNT}" = "1" ] && [ -z "${MOUNTPOINT}" ]; then - usage - echo "Can not umount. Mount point parameter missing." - exit -fi - -export NBD_DEV="${NBD_DEV:-/dev/nbd1}" -export MAP_BOOT_DEV=/dev/mapper/nbd1p1 -export MAP_ROOT_DEV=/dev/mapper/nbd1p2 -source scripts/qcow2_handling - -if [ "${MOUNT}" = "1" ]; then - mount_qimage "${IMAGE}" "${MOUNTPOINT}" -elif [ "${UMOUNT}" = "1" ]; then - umount_qimage "${MOUNTPOINT}" -fi diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling deleted file mode 100644 index 66708e710e..0000000000 --- a/scripts/qcow2_handling +++ /dev/null @@ -1,256 +0,0 @@ -#!/bin/bash - -# QCOW2 Routines - -export CURRENT_IMAGE -export CURRENT_MOUNTPOINT - -export NBD_DEV -export MAP_BOOT_DEV -export MAP_ROOT_DEV - -# set in build.sh -# should be fairly enough for the beginning -# overwrite here by uncommenting following lines -# BASE_QCOW2_SIZE=12G - -# find and initialize free block device nodes -init_nbd() { - modprobe nbd max_part=16 - if [ -z "${NBD_DEV}" ]; then - for x in /sys/class/block/nbd* ; do - S=`cat $x/size` - if [ "$S" == "0" ] ; then - NBD_DEV=/dev/$(basename $x) - MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1 - MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2 - break - fi - done - fi -} -export -f init_nbd - -# connect image to block device -connect_blkdev() { - init_nbd - qemu-nbd --discard=unmap -c $NBD_DEV "$1" - sync - kpartx -as $NBD_DEV - sync - CURRENT_IMAGE="$1" -} -export -f connect_blkdev - -# disconnect image from block device -disconnect_blkdev() { - kpartx -d $NBD_DEV - qemu-nbd -d $NBD_DEV - NBD_DEV= - MAP_BOOT_DEV= - MAP_ROOT_DEV= - CURRENT_IMAGE= -} -export -f disconnect_blkdev - -# mount qcow2 image: mount_image <image file> <mountpoint> -mount_qimage() { - connect_blkdev "$1" - mount -v -t ext4 $MAP_ROOT_DEV "$2" - mkdir -p "${ROOTFS_DIR}/boot" - mount -v -t vfat $MAP_BOOT_DEV "$2/boot" - CURRENT_MOUNTPOINT="$2" -} -export -f mount_qimage - -# umount qcow2 image: umount_image <current mountpoint> -umount_qimage() { - sync - #umount "$1/boot" - while mount | grep -q "$1"; do - local LOCS - LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r) - for loc in $LOCS; do - echo "$loc" - while mountpoint -q "$loc" && ! umount "$loc"; do - sleep 0.1 - done - done - done - CURRENT_MOUNTPOINT= - disconnect_blkdev -} -export -f umount_qimage - -# create base image / backing image / mount image -load_qimage() { - if [ -z "${CURRENT_MOUNTPOINT}" ]; then - if [ ! -d "${ROOTFS_DIR}" ]; then - mkdir -p "${ROOTFS_DIR}"; - fi - - if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then - rm -f "${WORK_DIR}/image-${STAGE}.qcow2"; - fi - - if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then - pushd ${WORK_DIR} > /dev/null - init_nbd - if [ -z "${PREV_STAGE}" ]; then - echo "Creating base image: image-${STAGE}.qcow2" - # -o preallocation=falloc - qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE - sync - qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 - sync - sfdisk $NBD_DEV << EOF -4MiB,250MiB,c,* -254MiB,,83; -EOF - sync - kpartx -as $NBD_DEV - mkdosfs -n boot -F 32 -s 4 -v $MAP_BOOT_DEV - mkfs.ext4 -L rootfs -O "^huge_file,^64bit" $MAP_ROOT_DEV - sync - else - if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then - exit 1; - fi - echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2" - qemu-img create -f qcow2 \ - -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \ - ${WORK_DIR}/image-${STAGE}.qcow2 - sync - qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 - sync - kpartx -as $NBD_DEV - fi - - mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}" - mkdir -p "${ROOTFS_DIR}/boot" - mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot" - CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2 - CURRENT_MOUNTPOINT=${ROOTFS_DIR} - popd > /dev/null - else - mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}" - fi - echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})" - fi -} -export -f load_qimage - -# umount current image and refresh mount point env var -unload_qimage() { - if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then - fstrim -v "${CURRENT_MOUNTPOINT}" || true - umount_qimage "${CURRENT_MOUNTPOINT}" - fi -} -export -f unload_qimage - -# based on: https://github.com/SirLagz/RaspberryPi-ImgAutoSizer -# helper function for make_bootable_image, do not call directly -function resize_qcow2() { - if [ -z "$CALL_FROM_MBI" ]; then - echo "resize_qcow2: cannot be called directly, use make_bootable_image instead" - return 1 - fi - - # ROOT_MARGIN=$((800*1024*1024)) - ROOT_MARGIN=$((1*1024*1024)) - PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print` - PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '` - PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '` - - e2fsck -y -f $MAP_ROOT_DEV || true - - DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '` - BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }') - BLOCK_SIZE=${BLOCK_SIZE// /} - - let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE - resize2fs -p $MAP_ROOT_DEV $DATA_SIZE - sleep 1 - - let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE - let PART_NEW_END=$PART_START+$PART_NEW_SIZE - ACT1=`parted -s "$NBD_DEV" rm 2` - ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END` - NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '` -} -export -f resize_qcow2 - -# create raw img from qcow2: make_bootable_image <in.qcow2> <out.img> -function make_bootable_image() { - - EXPORT_QCOW2="$1" - EXPORT_IMAGE="$2" - - echo "Connect block device to source qcow2" - connect_blkdev "${EXPORT_QCOW2}" - - echo "Resize fs and partition" - CALL_FROM_MBI=1 - resize_qcow2 - sync - CALL_FROM_MBI= - - echo "Disconnect block device" - disconnect_blkdev - - if [ -z "$NEW_IMG_SIZE" ]; then - echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit." - exit 1 - fi - - echo "Shrinking qcow2 image" - qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE - sync - - echo "Convert qcow2 to raw image" - qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}" - sync - - echo "Get PARTUUIDs from image" - IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")" - - BOOT_PARTUUID="${IMGID}-01" - echo "Boot: $BOOT_PARTUUID" - ROOT_PARTUUID="${IMGID}-02" - echo "Root: $ROOT_PARTUUID" - - echo "Mount image" - MOUNTROOT=${WORK_DIR}/tmpimage - mkdir -p $MOUNTROOT - - MOUNTPT=$MOUNTROOT - PARTITION=2 - mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 - - MOUNTPT=$MOUNTROOT/boot - PARTITION=1 - mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 - - if [ ! -d "${MOUNTROOT}/root" ]; then - echo "Image damaged or not mounted. Exit." - exit 1 - fi - - echo "Setup PARTUUIDs" - if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then - echo "Set UUIDs to make it bootable" - sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" - sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" - sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt" - fi - - echo "Umount image" - sync - umount "${MOUNTROOT}/boot" || exit 1 - umount "${MOUNTROOT}" || exit 1 - - echo "Remove qcow2 export image" - rm -f "${EXPORT_QCOW2}" -} -export -f make_bootable_image diff --git a/stage0/prerun.sh b/stage0/prerun.sh index 0252071470..5f0bd2c78a 100755 --- a/stage0/prerun.sh +++ b/stage0/prerun.sh @@ -5,6 +5,6 @@ if [ "$RELEASE" != "bookworm" ]; then echo " Please check the relevant README.md section." fi -if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then +if [ ! -d "${ROOTFS_DIR}" ]; then bootstrap ${RELEASE} "${ROOTFS_DIR}" http://raspbian.raspberrypi.com/raspbian/ fi From 94cbf476a57bfc7483d1ce08f2ddbe28e4949ffe Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 14 Feb 2024 13:25:42 +0000 Subject: [PATCH 163/222] Create .gitlab-ci.yml file --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..32f2688b52 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - project: serge/pi-gen + ref: ci + file: 'pi-gen.yml' From c029146a132b613f538b1368fd0706cc93fd88b9 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 15 Feb 2024 21:27:39 +0000 Subject: [PATCH 164/222] ensure_loopdev_partitions: let udev settle --- scripts/common | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/common b/scripts/common index cda5551302..6150f2b2f1 100644 --- a/scripts/common +++ b/scripts/common @@ -122,5 +122,7 @@ ensure_loopdev_partitions() { mknod "/dev/$partition" b "${majmin%:*}" "${majmin#*:}" fi done + command -v udevadm >/dev/null 2>&1 || return 0 + udevadm settle 10 } export -f ensure_loopdev_partitions From 82e5642ad16453f47fbbaeb11f8b63d4ccf17d81 Mon Sep 17 00:00:00 2001 From: Christian Kuhtz <christian@kuhtz.com> Date: Mon, 26 Feb 2024 06:12:43 -0800 Subject: [PATCH 165/222] README.md: Add xxd dependency (#750) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3529770836..136e892c52 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ To install the required dependencies for `pi-gen` you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \ -gpg pigz +gpg pigz xxd ``` The file `depends` contains a list of tools needed. The format of this From 1ce1f1c913d06485ef5baf17143c1d326ae40d4d Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 4 Mar 2024 08:54:17 +0000 Subject: [PATCH 166/222] Replace libcamera-apps with rpicam-apps --- stage2/01-sys-tweaks/00-packages-nr | 2 +- stage3/00-install-packages/00-packages | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/01-sys-tweaks/00-packages-nr b/stage2/01-sys-tweaks/00-packages-nr index 6bdb60069e..15c0f6626f 100644 --- a/stage2/01-sys-tweaks/00-packages-nr +++ b/stage2/01-sys-tweaks/00-packages-nr @@ -1,3 +1,3 @@ cifs-utils -libcamera-apps-lite +rpicam-apps-lite mkvtoolnix diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index 9175bcca36..69f8c456aa 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -13,7 +13,7 @@ fonts-liberation2 obconf arandr libcamera-tools -libcamera-apps +rpicam-apps python3-picamera2 python3-pyqt5 python3-opengl From 57c641a64d1ff161cf043b3003fcfe63059eb508 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 7 Mar 2024 23:17:56 +0000 Subject: [PATCH 167/222] Update release notes --- .../00-release/files/release_notes.txt | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index d305e33a06..30afa90437 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,31 @@ UNRELEASED: + * Added setting of headless resolution to Screen Configuration + * Removed setting of headless resolution for Wayland from Raspberry Pi Configuration + * Improved handling of power button on Raspberry Pi 5 + * Popover windows from taskbar replaced with conventional windows + * Shutdown assistant now closes all user processes when logging out + * Wayvnc updated to improve compatibility with various VNC clients + * Wayvnc now controlled by systemd + * Audio icon on taskbar hidden if no audio devices connected + * Alternative mouse cursor shown during drag-and-drop operations + * raspi-config now allows EEPROM to be upgraded + * Speed improvement when opening bluetooth and network menus + * Tweaks to display of some widgets under dark theme + * Improved compatibility with alternative window managers + * Bug fix - prevent multiple file manager confirm dialogs being overlaid + * Bug fix - drag-and-drop in file manager causing incorrect files to move + * Bug fix - memory leaks in volume and bluetooth menus + * Bug fix - GPU load sometimes not correctly reported in plugin and task manager + * Bug fix - crash when closing windows with non-GTK headerbars + * Bug fix - spurious button hover highlights on touchscreens + * Bug fix - windows on other monitors being hidden from taskbar + * Bug fix - corrected power monitoring brownout detection + * Bug fix - wayfire keyboard layout settings sometimes not loading + * Chromium updated to 122.0.6261.89 + * Firefox updated to 123.0 + * Raspberry Pi firmware 6e0ae774407d68659b50cfbeb9f493ed16718866 + * Linux kernel 6.6.20 - 6f16847710cc0502450788b9f12f0a14d3429668 +2023-12-11: * Fix Raspberry Pi Imager's WLAN configuration for lite images 2023-12-05: * Serial port switching in rc_gui and raspi-config modified to support Raspberry Pi 5 From 4ce1f1fa8981c4de70a654899781192f2c64adb4 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 11 Mar 2024 14:49:23 +0000 Subject: [PATCH 168/222] stage3: Remove fbturbo --- stage3/00-install-packages/00-packages-nr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage3/00-install-packages/00-packages-nr b/stage3/00-install-packages/00-packages-nr index 66f3004819..bd310cf545 100644 --- a/stage3/00-install-packages/00-packages-nr +++ b/stage3/00-install-packages/00-packages-nr @@ -1,4 +1,4 @@ -xserver-xorg-video-fbdev xserver-xorg xinit xserver-xorg-video-fbturbo +xserver-xorg-video-fbdev xserver-xorg xinit mousepad eom lxde lxtask menu-xdg From e8d8670b6c2b5d7e2aa28f410c96e50bd68c2662 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 11 Mar 2024 14:52:14 +0000 Subject: [PATCH 169/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 30afa90437..b5afa30bb1 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -21,6 +21,7 @@ UNRELEASED: * Bug fix - windows on other monitors being hidden from taskbar * Bug fix - corrected power monitoring brownout detection * Bug fix - wayfire keyboard layout settings sometimes not loading + * Removed fbturbo xorg video driver as it is no longer useful * Chromium updated to 122.0.6261.89 * Firefox updated to 123.0 * Raspberry Pi firmware 6e0ae774407d68659b50cfbeb9f493ed16718866 From 99e19802c37ec4e1f37cc42bb7f026fb7718226e Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 13 Mar 2024 15:14:05 +0000 Subject: [PATCH 170/222] stage4: remove disable-pwr-button substage Desktop is now started with systemd-inhibit --- stage4/07-disable-pwr-button/00-run.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 stage4/07-disable-pwr-button/00-run.sh diff --git a/stage4/07-disable-pwr-button/00-run.sh b/stage4/07-disable-pwr-button/00-run.sh deleted file mode 100755 index 44646bd122..0000000000 --- a/stage4/07-disable-pwr-button/00-run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -e - -sed -i 's/^.*HandlePowerKey=.*$/HandlePowerKey=ignore/' "${ROOTFS_DIR}/etc/systemd/logind.conf" From d92bef11f4382498e089c0bd7895a937134f4e53 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 13 Mar 2024 15:44:52 +0000 Subject: [PATCH 171/222] stage4: add meson --- stage4/00-install-packages/00-packages | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index 23077524c3..69b1f8fcce 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -19,3 +19,6 @@ rp-prefapps ffmpeg vlc rpi-imager + +# ninja-build needed for vscode pico extension +meson From b829737d2a07b6f971b1c0d97b6962110cc8c46c Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 13 Mar 2024 15:46:00 +0000 Subject: [PATCH 172/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index b5afa30bb1..d3eab8c3b2 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,6 @@ UNRELEASED: + * +2024-03-12: * Added setting of headless resolution to Screen Configuration * Removed setting of headless resolution for Wayland from Raspberry Pi Configuration * Improved handling of power button on Raspberry Pi 5 From 3b5e214f5ec4e23323bdd489b999536c487fcd12 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 13 Mar 2024 18:16:30 +0000 Subject: [PATCH 173/222] export-image: format boot partition with appropriate FAT size --- export-image/prerun.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/export-image/prerun.sh b/export-image/prerun.sh index b95fe3684b..d9778d1db3 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -55,7 +55,14 @@ if grep -q "$FEATURE" /etc/mke2fs.conf; then ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" fi done -mkdosfs -n bootfs -F 32 -s 4 -v "$BOOT_DEV" > /dev/null + +if [ "$BOOT_SIZE" -lt 134742016 ]; then + FAT_SIZE=16 +else + FAT_SIZE=32 +fi + +mkdosfs -n bootfs -F "$FAT_SIZE" -s 4 -v "$BOOT_DEV" > /dev/null mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 From af5a88a20a0327a292ff546f1d852c6a31f7dc76 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 13 Mar 2024 19:53:05 +0000 Subject: [PATCH 174/222] stage4: add openocd --- stage4/00-install-packages/00-packages | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index 69b1f8fcce..89fb59ad9a 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -20,5 +20,5 @@ ffmpeg vlc rpi-imager -# ninja-build needed for vscode pico extension -meson +# ninja-build and openocd needed for vscode pico extension +meson openocd From 11096428148f0f2be3985ef3126ee71f99c7f1c2 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 15 Mar 2024 14:32:31 +0000 Subject: [PATCH 175/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index d3eab8c3b2..63aab16434 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,5 +1,11 @@ UNRELEASED: - * + * Audio streams will now not be interrupted when other audio devices are connected or disconnected + * Keyboard shortcut to install Orca no longer prompts for password, and will now wait for clock synchronisation rather than failing silently + * Orca screen reader updated to version 45 with various additional small bug fixes + * Obsolete fbturbo video driver removed + * Bug fix - saved display resolution settings not reloading under X + * Raspberry Pi firmware 6e0ae774407d68659b50cfbeb9f493ed16718866 + * Linux kernel 6.6.20 - 6f16847710cc0502450788b9f12f0a14d3429668 2024-03-12: * Added setting of headless resolution to Screen Configuration * Removed setting of headless resolution for Wayland from Raspberry Pi Configuration From 7ce2538bca367f87547f50649d4e806d8ebcd087 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 9 Apr 2024 15:27:24 +0100 Subject: [PATCH 176/222] Revert "Added docker-compose.yml for easy apt-cacher-ng startup" This reverts commit fe45b739871c2eba65ec520e00816299abbfbc5b. It's probably a bad idea to reference docker containers from unofficial sources --- .gitignore | 1 - README.md | 6 ------ docker-compose.yml | 10 ---------- 3 files changed, 17 deletions(-) delete mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index a6883c2049..f01ee89098 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,3 @@ SKIP SKIP_IMAGES .pc *-pc -apt-cacher-ng/ diff --git a/README.md b/README.md index 136e892c52..c48e70842a 100644 --- a/README.md +++ b/README.md @@ -77,12 +77,6 @@ The following environment variables are supported: will not be included in the image, making it safe to use an `apt-cacher` or similar package for development. - If you have Docker installed, you can set up a local apt caching proxy to - like speed up subsequent builds like this: - - docker-compose up -d - echo 'APT_PROXY=http://172.17.0.1:3142' >> config - * `BASE_DIR` (Default: location of `build.sh`) **CAUTION**: Currently, changing this value will probably break build.sh diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index f733860857..0000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '2' - -services: - apt-cacher-ng: - restart: unless-stopped - image: sameersbn/apt-cacher-ng:latest - ports: - - "3142:3142" - volumes: - - ./apt-cacher-ng:/var/cache/apt-cacher-ng From c70f96f1464497bd03108385ed1e40fbf39aba94 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 9 Apr 2024 15:51:44 +0100 Subject: [PATCH 177/222] stage5: smartsim has been removed from recommended apps --- stage5/00-install-extras/00-packages | 1 - 1 file changed, 1 deletion(-) diff --git a/stage5/00-install-extras/00-packages b/stage5/00-install-extras/00-packages index ad5dcad6dc..85474ddb2f 100644 --- a/stage5/00-install-extras/00-packages +++ b/stage5/00-install-extras/00-packages @@ -1,6 +1,5 @@ mu-editor scratch nuscratch scratch3 -smartsim wolfram-engine claws-mail realvnc-vnc-viewer From 165e9602376b1ee25d8599389e5ae5643dd9754c Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 18 Apr 2024 11:29:40 +0100 Subject: [PATCH 178/222] Fix keyboard configuration keyboard-configuration scripts ignores debconf settings if it doesn't detect an attached keyboard. Instead, it sets XKBOPTIONS to lv3:ralt_switch, which is not a desired option. It also considers an empty XKBOPTIONS string to be unset. This workaround sets it to a placeholder value which is removed later. dpkg-reconfigure is run so that debconf-get-selections prints the correct values. --- stage2/01-sys-tweaks/00-debconf | 2 ++ stage2/01-sys-tweaks/01-run.sh | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/stage2/01-sys-tweaks/00-debconf b/stage2/01-sys-tweaks/00-debconf index c13e3b1949..884db9351b 100644 --- a/stage2/01-sys-tweaks/00-debconf +++ b/stage2/01-sys-tweaks/00-debconf @@ -24,3 +24,5 @@ keyboard-configuration keyboard-configuration/ctrl_alt_bksp boolean true # Keyboard layout: # Choices: English (UK), English (UK) - English (UK\, Colemak), English (UK) - English (UK\, Dvorak with UK punctuation), English (UK) - English (UK\, Dvorak), English (UK) - English (UK\, Macintosh international), English (UK) - English (UK\, Macintosh), English (UK) - English (UK\, extended WinKeys), English (UK) - English (UK\, international with dead keys), Other keyboard-configuration keyboard-configuration/variant select ${KEYBOARD_LAYOUT} +# for internal use +keyboard-configuration keyboard-configuration/optionscode string PLACEHOLDER diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index 59988b2f89..897463e6df 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -67,3 +67,8 @@ usermod --pass='*' root EOF rm -f "${ROOTFS_DIR}/etc/ssh/"ssh_host_*_key* + +sed -i "s/PLACEHOLDER//" "${ROOTFS_DIR}/etc/default/keyboard" +on_chroot << EOF +DEBIAN_FRONTEND=noninteractive dpkg-reconfigure keyboard-configuration +EOF From 5780b1ff6428965ee4cc3ed1f470481417494b37 Mon Sep 17 00:00:00 2001 From: Kristof Nachtergaele <kristof.nachtergaele@alphatronics.be> Date: Thu, 25 Apr 2024 14:31:24 +0200 Subject: [PATCH 179/222] export stage list --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index f34adfce17..054b3f54ce 100755 --- a/build.sh +++ b/build.sh @@ -283,6 +283,7 @@ mkdir -p "${WORK_DIR}" log "Begin ${BASE_DIR}" STAGE_LIST=${STAGE_LIST:-${BASE_DIR}/stage*} +export STAGE_LIST for STAGE_DIR in $STAGE_LIST; do STAGE_DIR=$(realpath "${STAGE_DIR}") From f1c166a2833950a7c44fe19b01780723635a7aa3 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 29 Apr 2024 14:34:52 +0100 Subject: [PATCH 180/222] export-image: run du with -x to avoid crossing filesystem boundaries Fixes #537 --- export-image/prerun.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export-image/prerun.sh b/export-image/prerun.sh index d9778d1db3..6a5309a0a4 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -10,7 +10,7 @@ rm -rf "${ROOTFS_DIR}" mkdir -p "${ROOTFS_DIR}" BOOT_SIZE="$((512 * 1024 * 1024))" -ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot/firmware --block-size=1 | cut -f 1) +ROOT_SIZE=$(du -x --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot/firmware --block-size=1 | cut -f 1) # All partition sizes and starts will be aligned to this size ALIGN="$((4 * 1024 * 1024))" From e95e00f879e24c0f087763ec471fb9bdf9a8f92b Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 13 May 2024 15:24:21 +0100 Subject: [PATCH 181/222] Set a default IMG_NAME This also adds the 'ARCH' environment variable, which will help reduce the delta between master and arm64 branches. --- README.md | 2 +- build.sh | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c48e70842a..c58482595f 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ environment variables. The following environment variables are supported: - * `IMG_NAME` **required** (Default: unset) + * `IMG_NAME` (Default: raspios-$RELEASE-$ARCH, for example: "raspios-bookworm-armhf") The name of the image to build with the current stage directories. Use this variable to set the root name of your OS, eg `IMG_NAME=Frobulator`. diff --git a/build.sh b/build.sh index 054b3f54ce..3cb10e70ab 100755 --- a/build.sh +++ b/build.sh @@ -167,10 +167,9 @@ export PI_GEN=${PI_GEN:-pi-gen} export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen} export PI_GEN_RELEASE=${PI_GEN_RELEASE:-Raspberry Pi reference} -if [ -z "${IMG_NAME}" ]; then - echo "IMG_NAME not set" 1>&2 - exit 1 -fi +export ARCH=armhf +export RELEASE=${RELEASE:-bookworm} # Don't forget to update stage0/prerun.sh +export IMG_NAME="${IMG_NAME:-raspios-$RELEASE-$ARCH}" export USE_QEMU="${USE_QEMU:-0}" export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}" @@ -198,7 +197,6 @@ export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi} export FIRST_USER_NAME=${FIRST_USER_NAME:-pi} export FIRST_USER_PASS export DISABLE_FIRST_BOOT_USER_RENAME=${DISABLE_FIRST_BOOT_USER_RENAME:-0} -export RELEASE=${RELEASE:-bookworm} # Don't forget to update stage0/prerun.sh export WPA_COUNTRY export ENABLE_SSH="${ENABLE_SSH:-0}" export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}" @@ -215,7 +213,6 @@ export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"} export PUBKEY_SSH_FIRST_USER export CLEAN -export IMG_NAME export APT_PROXY export STAGE From d87f764fcbc2e9bdbfd407d2cbf9b5faf5600df1 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 13 May 2024 15:54:28 +0100 Subject: [PATCH 182/222] build.sh: Warn user of unsupported build environment architecture and fail early if binfmt_misc isn't working as expected --- Dockerfile | 4 ++-- README.md | 2 +- build.sh | 9 +++++++++ depends | 1 + 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 673f7564df..0e6d86df2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,8 @@ RUN apt-get -y update && \ apt-get -y install --no-install-recommends \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ - libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ - binfmt-support ca-certificates fdisk gpg pigz\ + libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc \ + binfmt-support ca-certificates fdisk gpg pigz arch-test \ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ diff --git a/README.md b/README.md index c58482595f..279baaa27a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ To install the required dependencies for `pi-gen` you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \ -gpg pigz xxd +gpg pigz xxd arch-test ``` The file `depends` contains a list of tools needed. The format of this diff --git a/build.sh b/build.sh index 3cb10e70ab..d094eee70f 100755 --- a/build.sh +++ b/build.sh @@ -244,6 +244,15 @@ fi dependencies_check "${BASE_DIR}/depends" +echo "Checking native $ARCH executable support..." +if ! arch-test -n "$ARCH"; then + echo "WARNING: Only a native build environment is supported. Checking emulated support..." + if ! arch-test "$ARCH"; then + echo "No fallback mechanism found. Ensure your OS has binfmt_misc support enabled and configured." + exit 1 + fi +fi + #check username is valid if [[ ! "$FIRST_USER_NAME" =~ ^[a-z][-a-z0-9_]*$ ]]; then echo "Invalid FIRST_USER_NAME: $FIRST_USER_NAME" diff --git a/depends b/depends index 10fcdd4718..acbb5cde9a 100644 --- a/depends +++ b/depends @@ -19,3 +19,4 @@ lsmod:kmod bc gpg pigz +arch-test From 9917c869a7fed6fb210cbd82bda5cb4a28d4119f Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 13 May 2024 16:40:54 +0100 Subject: [PATCH 183/222] README.md: Consistent formatting of 'Default' values --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 279baaa27a..53e1fb91c6 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ environment variables. The following environment variables are supported: - * `IMG_NAME` (Default: raspios-$RELEASE-$ARCH, for example: "raspios-bookworm-armhf") + * `IMG_NAME` (Default: `raspios-$RELEASE-$ARCH`, for example: `raspios-bookworm-armhf`) The name of the image to build with the current stage directories. Use this variable to set the root name of your OS, eg `IMG_NAME=Frobulator`. @@ -63,7 +63,7 @@ The following environment variables are supported: The release name to use in `/etc/issue.txt`. The default should only be used for official Raspberry Pi builds. -* `RELEASE` (Default: bookworm) +* `RELEASE` (Default: `bookworm`) The release version to build images against. Valid values are any supported Debian release. However, since different releases will have different sets of @@ -84,7 +84,7 @@ The following environment variables are supported: Top-level directory for `pi-gen`. Contains stage directories, build scripts, and by default both work and deployment directories. - * `WORK_DIR` (Default: `"$BASE_DIR/work"`) + * `WORK_DIR` (Default: `$BASE_DIR/work`) Directory in which `pi-gen` builds the target system. This value can be changed if you have a suitably large, fast storage location for stages to @@ -94,7 +94,7 @@ The following environment variables are supported: **CAUTION**: If your working directory is on an NTFS partition you probably won't be able to build: make sure this is a proper Linux filesystem. - * `DEPLOY_DIR` (Default: `"$BASE_DIR/deploy"`) + * `DEPLOY_DIR` (Default: `$BASE_DIR/deploy`) Output directory for target system images and NOOBS bundles. @@ -121,20 +121,20 @@ The following environment variables are supported: information on this. Usually 0 is no compression but very fast, up to 9 with the best compression but very slow ). - * `USE_QEMU` (Default: `"0"`) + * `USE_QEMU` (Default: `0`) Setting to '1' enables the QEMU mode - creating an image that can be mounted via QEMU for an emulated environment. These images include "-qemu" in the image file name. - * `LOCALE_DEFAULT` (Default: "en_GB.UTF-8" ) + * `LOCALE_DEFAULT` (Default: 'en_GB.UTF-8' ) Default system locale. - * `TARGET_HOSTNAME` (Default: "raspberrypi" ) + * `TARGET_HOSTNAME` (Default: 'raspberrypi' ) Setting the hostname to the specified value. - * `KEYBOARD_KEYMAP` (Default: "gb" ) + * `KEYBOARD_KEYMAP` (Default: 'gb' ) Default keyboard keymap. @@ -142,7 +142,7 @@ The following environment variables are supported: keyboard-configuration` and look at the `keyboard-configuration/xkb-keymap` value. - * `KEYBOARD_LAYOUT` (Default: "English (UK)" ) + * `KEYBOARD_LAYOUT` (Default: 'English (UK)' ) Default keyboard layout. @@ -150,7 +150,7 @@ The following environment variables are supported: keyboard-configuration` and look at the `keyboard-configuration/variant` value. - * `TIMEZONE_DEFAULT` (Default: "Europe/London" ) + * `TIMEZONE_DEFAULT` (Default: 'Europe/London' ) Default keyboard layout. From d790fed832188d41c1fb5d8506d833d053e4a81c Mon Sep 17 00:00:00 2001 From: Pev <pev@high-resolution.co.uk> Date: Wed, 15 May 2024 15:45:43 +0100 Subject: [PATCH 184/222] Compare mount output using base of dirname (#771) * Compare mount output using base of dirname If you're building from a bind-mounted directory, the build will fail as it will find two entries to unmount, but a single unmount will remove them both causing an error. Adding a space means that the mountpoint will only match with a single mount entry ; the expected path, rather than the pre bind-mount. * Switch to awk instead of mount, grep, cut pipes Retry unmount 5 times and give up, letting the user know that they need to resolve the issue manually --------- Co-authored-by: David Peverley <pev@analogue-micro.com> Co-authored-by: Serge Schneider <serge@raspberrypi.com> --- export-image/05-finalise/01-run.sh | 2 +- scripts/common | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/export-image/05-finalise/01-run.sh b/export-image/05-finalise/01-run.sh index 8310e870a7..771aa7a235 100755 --- a/export-image/05-finalise/01-run.sh +++ b/export-image/05-finalise/01-run.sh @@ -83,7 +83,7 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" dpkg -l --root "$ROOTFS_DIR" } >> "$INFO_FILE" -ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" +ROOT_DEV="$(awk "\$2 == \"${ROOTFS_DIR}\" {print \$1}" /etc/mtab)" unmount "${ROOTFS_DIR}" zerofree "${ROOT_DEV}" diff --git a/scripts/common b/scripts/common index 6150f2b2f1..c305ee2e1a 100644 --- a/scripts/common +++ b/scripts/common @@ -44,12 +44,15 @@ unmount(){ DIR=$1 fi - while mount | grep -q "$DIR"; do - local LOCS - LOCS=$(mount | grep "$DIR" | cut -f 3 -d ' ' | sort -r) - for loc in $LOCS; do - umount "$loc" - done + for i in {1..6}; do + if awk "\$2 ~ /^${DIR//\//\\/}/ {print \$2}" /etc/mtab | sort -r | xargs -r umount; then + break + elif [ "$i" -eq 6 ]; then + log "Failed to unmount ${DIR}. Do not try to delete this directory while it contains mountpoints!" + return 1 + fi + log "Retrying ($i/5)..." + sleep 1 done } export -f unmount From fb48183f0d49df3d069f0086263d829efa899c70 Mon Sep 17 00:00:00 2001 From: Tom Dewey <146750643+tdewey-rpi@users.noreply.github.com> Date: Wed, 15 May 2024 15:50:40 +0100 Subject: [PATCH 185/222] Expand trap cleanup function (#773) * build.sh: Unmount intermediates on trap Iterate through image files which might be in use and detach them Avoid silent failures - let the user know whether the build failed * common: update unmount_image Use udevadm settle instead of sleep if possible Use losetup's -j option to find the loop device associated with a give image file * build.sh: update clean-up trap term --------- Co-authored-by: Serge Schneider <serge@raspberrypi.com> --- build.sh | 12 +++++++++++- scripts/common | 9 ++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index d094eee70f..a96f4e4637 100755 --- a/build.sh +++ b/build.sh @@ -158,7 +158,17 @@ do done term() { - true; #TODO: Cleanup + if [ "$?" -ne 0 ]; then + log "Build failed" + else + log "Build finished" + fi + unmount "${STAGE_WORK_DIR}" + if [ "$STAGE" = "export-image" ]; then + for img in "${STAGE_WORK_DIR}/"*.img; do + unmount_image "$img" + done + fi } trap term EXIT INT TERM diff --git a/scripts/common b/scripts/common index c305ee2e1a..702576fd76 100644 --- a/scripts/common +++ b/scripts/common @@ -58,9 +58,12 @@ unmount(){ export -f unmount unmount_image(){ - sync - sleep 1 - LOOP_DEVICE=$(losetup --list | grep "$1" | cut -f1 -d' ') + if command -v udevadm >/dev/null 2>&1; then + udevadm settle 10 + else + sleep 1 + fi + LOOP_DEVICE=$(losetup -n -O NAME -j "$1") if [ -n "$LOOP_DEVICE" ]; then for part in "$LOOP_DEVICE"p*; do if DIR=$(findmnt -n -o target -S "$part"); then From 75fe47c7571c533ad52c43e03b440b0116a157ea Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 27 May 2024 13:48:11 +0100 Subject: [PATCH 186/222] Increase default swap size to 200 --- stage2/01-sys-tweaks/00-patches/02-swap.diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/01-sys-tweaks/00-patches/02-swap.diff b/stage2/01-sys-tweaks/00-patches/02-swap.diff index 745a344e49..f542ec2411 100644 --- a/stage2/01-sys-tweaks/00-patches/02-swap.diff +++ b/stage2/01-sys-tweaks/00-patches/02-swap.diff @@ -7,7 +7,7 @@ Index: jessie-stage2/rootfs/etc/dphys-swapfile # set size to absolute value, leaving empty (default) then uses computed value # you most likely don't want this, unless you have an special disk situation -#CONF_SWAPSIZE= -+CONF_SWAPSIZE=100 ++CONF_SWAPSIZE=200 # set size to computed value, this times RAM size, dynamically adapts, # guarantees that there is enough swap without wasting disk space on excess From 07ab46485360dea437c3696022436c19ff36bf5a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 10 Jun 2024 13:53:24 +0100 Subject: [PATCH 187/222] stage4: Pre-install labwc --- stage4/00-install-packages/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index 89fb59ad9a..3c526b8129 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -19,6 +19,7 @@ rp-prefapps ffmpeg vlc rpi-imager +labwc # ninja-build and openocd needed for vscode pico extension meson openocd From 9b17be9621b5ba5f946e7435e75c66a31390d573 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 10 Jun 2024 15:46:05 +0100 Subject: [PATCH 188/222] Update release notes --- .../00-release/files/release_notes.txt | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 63aab16434..3512df2e85 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,43 @@ UNRELEASED: + * pipanel - allow customisation of more than 2 desktops + * pipanel - add customisation for labwc + * gui-pkinst - add whitelist to restrict installation to specified packages only + * pixflat-theme - add theme settings for labwc + * pishutdown - revert to original use of pkill to close desktop + * piclone - fix for potential buffer overflow vulnerability (that would never have actually happened…) + * lp-connection-editor - fix dialog icons on taskbar + * rp-prefapps - add Raspberry Pi Connect; remove SmartSim + * piwiz - add page to enable / disable Raspberry Pi Connect + * wf-panel-pi - constrain main menu to fit on small screens + * wf-panel-pi - fix dialog icons on taskbar + * wf-panel-pi - fix keyboard handling and icon highlighting for taskbar buttons + * raspberrypi-ui-mods - add configuration for labwc + * raspberrypi-ui-mods - add support for new touchscreens + * raspberrypi-ui-mods - systemd-inhibit used to override hardware power key on Pi 5 + * rc-gui - add configuration of alternate keyboard layout + * rc-gui - add switching for Raspberry Pi Connect + * arandr - add brightness control for DSI displays + * arandr - more reliable method to detect virtual displays + * raspi-config - add setting of keyboard options + * raspi-config - add setting of PCIe speed + * raspi-config - add switching for Raspberry Pi Connect + * wayvnc - better handling for virtual displays + * wayvnc - improved encryption support + * GTK-3 - add keyboard shortcuts in combo boxes + * pcmanfm - allow customisation of more than 2 desktops + * pcmanfm - fix bug causing crash and inconsistent behaviour on certain drag and drop operations + * raspberrypi-sys-mods - add udev rule to allow backlight change + * raspberrypi-sys-mods - increase swapfile size + * raspberrypi-sys-mods - remove symlinks from paths in initramfs scripts + * wayfire - fix for crash when opening multiple Xwayland windows + * wayfire - fix for touchscreen bug when touching areas without windows + * labwc compositor installed as an alternative to wayfire; can be enabled in raspi-config + * various small bug fixes and tweaks + * Chromium updated to 125.0.6422.133 + * Firefox updated to 126.0 + * Raspberry Pi firmware 3590de0c181d433af368a95f15bc480bdaff8b47 + * Linux kernel 6.6.31 - c1432b4bae5b6582f4d32ba381459f33c34d1424 +2024-03-15: * Audio streams will now not be interrupted when other audio devices are connected or disconnected * Keyboard shortcut to install Orca no longer prompts for password, and will now wait for clock synchronisation rather than failing silently * Orca screen reader updated to version 45 with various additional small bug fixes From ca5eb625d8892212838c4b3b972d9e7e03bfcb29 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 24 Jun 2024 10:43:37 +0100 Subject: [PATCH 189/222] Enable DWC2 host mode support for CM5 by default --- stage1/00-boot-files/files/config.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stage1/00-boot-files/files/config.txt b/stage1/00-boot-files/files/config.txt index 621637afda..2a64594068 100644 --- a/stage1/00-boot-files/files/config.txt +++ b/stage1/00-boot-files/files/config.txt @@ -42,5 +42,7 @@ arm_boost=1 # (e.g. for USB device mode) or if USB support is not required. otg_mode=1 -[all] +[cm5] +dtoverlay=dwc2,dr_mode=host +[all] From 0b115f302a8f1e5bd3523614d7f45b9d447434c7 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 19 Jun 2024 12:24:07 +0100 Subject: [PATCH 190/222] stage4: Pre-install rpi-connect --- stage4/00-install-packages/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index 3c526b8129..ebd1af4d1c 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -18,6 +18,7 @@ piwiz rp-prefapps ffmpeg vlc +rpi-connect rpi-imager labwc From dca93d24c7dfb943d84601d2bc4a089f9b66b87f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Le=20Ray?= <sebastien-github@orniz.org> Date: Tue, 9 Jul 2024 11:08:38 +0200 Subject: [PATCH 191/222] README: add specific stage build for Docker (#777) --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 53e1fb91c6..71be938e20 100644 --- a/README.md +++ b/README.md @@ -392,7 +392,8 @@ follows: * Run build.sh to build all stages * Add SKIP files to the earlier successfully built stages * Modify the last stage - * Rebuild just the last stage using ```sudo CLEAN=1 ./build.sh``` + * Rebuild just the last stage using ```sudo CLEAN=1 ./build.sh``` (or, for docker builds + ```PRESERVE_CONTAINER=1 CONTINUE=1 CLEAN=1 ./build-docker.sh```) * Once you're happy with the image you can remove the SKIP_IMAGES files and export your image to test From 21e8e54d22e2db7382aa3788c98489580f80b28a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 23 Jul 2024 13:39:53 +0100 Subject: [PATCH 192/222] export-image: generate sbom file if syft is available --- export-image/05-finalise/01-run.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/export-image/05-finalise/01-run.sh b/export-image/05-finalise/01-run.sh index 771aa7a235..89c60b90ac 100755 --- a/export-image/05-finalise/01-run.sh +++ b/export-image/05-finalise/01-run.sh @@ -2,6 +2,7 @@ IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" INFO_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.info" +SBOM_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.sbom" sed -i 's/^update_initramfs=.*/update_initramfs=all/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" @@ -61,10 +62,8 @@ if ! [ -L "${ROOTFS_DIR}/boot/issue.txt" ]; then ln -s firmware/issue.txt "${ROOTFS_DIR}/boot/issue.txt" fi - cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" - { if [ -f "$ROOTFS_DIR/usr/share/doc/raspberrypi-kernel/changelog.Debian.gz" ]; then firmware=$(zgrep "firmware as of" \ @@ -83,6 +82,14 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" dpkg -l --root "$ROOTFS_DIR" } >> "$INFO_FILE" +if hash syft 2>/dev/null; then + syft scan dir:"${ROOTFS_DIR}" \ + --base-path="${ROOTFS_DIR}" \ + --source-name="${IMG_NAME}${IMG_SUFFIX}" \ + --source-version="${IMG_DATE}" \ + -o spdx-json="${SBOM_FILE}" +fi + ROOT_DEV="$(awk "\$2 == \"${ROOTFS_DIR}\" {print \$1}" /etc/mtab)" unmount "${ROOTFS_DIR}" @@ -115,4 +122,7 @@ none | *) ;; esac +if [ -f "${SBOM_FILE}" ]; then + xz -c "${SBOM_FILE}" > "$DEPLOY_DIR/image_$(basename "${SBOM_FILE}").xz" +fi cp "$INFO_FILE" "$DEPLOY_DIR/" From 7fe9ae553c26acf359e2073a09d795ee84205239 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 27 Aug 2024 13:00:07 +0100 Subject: [PATCH 193/222] stage3: Switch chromium package and remove unnecessary steps --- stage3/00-install-packages/00-debconf | 2 -- stage3/00-install-packages/00-packages | 2 +- stage3/00-install-packages/01-run.sh | 8 ++------ 3 files changed, 3 insertions(+), 9 deletions(-) delete mode 100644 stage3/00-install-packages/00-debconf diff --git a/stage3/00-install-packages/00-debconf b/stage3/00-install-packages/00-debconf deleted file mode 100644 index 7dbd12eb28..0000000000 --- a/stage3/00-install-packages/00-debconf +++ /dev/null @@ -1,2 +0,0 @@ -# Adobe Flash Player. Copyright 1996-2015. Adobe Systems Incorporated. All Rights Reserved. -rpi-chromium-mods rpi-chromium-mods/adobe note diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index 69f8c456aa..002a32cd90 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -5,7 +5,7 @@ git policykit-1 gvfs rfkill -chromium-browser rpi-chromium-mods libwidevinecdm0 +chromium rpi-chromium-mods libwidevinecdm0 firefox rpi-firefox-mods gldriver-test fonts-droid-fallback diff --git a/stage3/00-install-packages/01-run.sh b/stage3/00-install-packages/01-run.sh index 0e8248dc10..7fb0f4bfd2 100755 --- a/stage3/00-install-packages/01-run.sh +++ b/stage3/00-install-packages/01-run.sh @@ -1,9 +1,5 @@ #!/bin/bash -e -on_chroot << EOF -update-alternatives --install /usr/bin/x-www-browser \ - x-www-browser /usr/bin/chromium-browser 86 -update-alternatives --install /usr/bin/gnome-www-browser \ - gnome-www-browser /usr/bin/chromium-browser 86 - apt-mark auto python3-pyqt5 python3-opengl +on_chroot <<- EOF + apt-mark auto python3-pyqt5 python3-opengl EOF From 40a5c1ceed18a669cc59957e8be94d708a75b39a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 11 Sep 2024 13:38:16 +0100 Subject: [PATCH 194/222] stage3: switch to labwc by default --- stage4/06-enable-wayland/00-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage4/06-enable-wayland/00-run.sh b/stage4/06-enable-wayland/00-run.sh index bd8bbc3bcc..d1f240d24f 100755 --- a/stage4/06-enable-wayland/00-run.sh +++ b/stage4/06-enable-wayland/00-run.sh @@ -1,5 +1,5 @@ #!/bin/bash -e on_chroot << EOF - SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_wayland W2 + SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_wayland W3 EOF From a766e71ff141aba2556599db8f860dbf393b160d Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 11 Sep 2024 13:39:38 +0100 Subject: [PATCH 195/222] stage3: switch from arandr to raindrop --- stage3/00-install-packages/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage3/00-install-packages/00-packages b/stage3/00-install-packages/00-packages index 002a32cd90..329b6fe7e3 100644 --- a/stage3/00-install-packages/00-packages +++ b/stage3/00-install-packages/00-packages @@ -11,7 +11,7 @@ gldriver-test fonts-droid-fallback fonts-liberation2 obconf -arandr +raindrop libcamera-tools rpicam-apps python3-picamera2 From 674c30e864b298ab036a6c8bd0b1037cef87aedd Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 18 Sep 2024 13:53:46 +0100 Subject: [PATCH 196/222] Replace python3-rpi.gpio with python3-rpi-lgpio --- stage2/01-sys-tweaks/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/01-sys-tweaks/00-packages b/stage2/01-sys-tweaks/00-packages index 93a69be89f..f06f7d0c4f 100644 --- a/stage2/01-sys-tweaks/00-packages +++ b/stage2/01-sys-tweaks/00-packages @@ -5,7 +5,7 @@ python-is-python3 v4l-utils gpiod python3-libgpiod python3-gpiozero -pigpio python3-pigpio raspi-gpio python3-rpi.gpio +pigpio python3-pigpio raspi-gpio python3-rpi-lgpio python3-spidev python3-smbus2 avahi-daemon From 8f88b83a9a376d0488b59f4c34d9b67bd6d18c6e Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 23 Sep 2024 15:21:08 +0100 Subject: [PATCH 197/222] stage4: install squeekboard --- stage4/00-install-packages/00-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/stage4/00-install-packages/00-packages b/stage4/00-install-packages/00-packages index ebd1af4d1c..511941c6e5 100644 --- a/stage4/00-install-packages/00-packages +++ b/stage4/00-install-packages/00-packages @@ -21,6 +21,7 @@ vlc rpi-connect rpi-imager labwc +squeekboard # ninja-build and openocd needed for vscode pico extension meson openocd From 9e65ea75d85cebae98d3f303d4cad8e50d11decb Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 26 Sep 2024 17:51:36 +0100 Subject: [PATCH 198/222] Update release notes --- .../00-release/files/release_notes.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 3512df2e85..73ee6e93dc 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,22 @@ UNRELEASED: + * labwc compositor now used as the default on all models of Raspberry Pi + * wf-panel-pi now loads plugins dynamically at runtime + * New raindrop screen configuration tool to replace arandr + * squeekboard on-screen keyboard installed; will automatically run on systems with touchscreens, and can be manually enabled in rc-gui for others + * Update installer moved out of panel code into separate gui-updater package + * Missing cursors added to icon theme + * Wizard no longer auto-pairs Bluetooth HID devices by default - create the file "/boot/firmware/btautopair" to enable, or reboot wizard after first run + * Safety info URL added to main menu + * Various memory leaks in pcmanfm fixed + * Bug fix - crash when running Appearance Settings on headless systems + * Bug fix - crash when opening icon chooser in application properties dialog + * Translations updated + * RPi.GPIO replaced with rpi-lgpio + * Chromium updated to 129.0.6668.89 + * Firefox updated to 131.0 + * Raspberry Pi firmware a2e586ba98ce68f7d11b1c717ad8329b95dcb3b6 + * Linux kernel 6.6.51 - 82a50e430ef1d6eb37d78e25aa572c1f6ea56160 +2024-07-04: * pipanel - allow customisation of more than 2 desktops * pipanel - add customisation for labwc * gui-pkinst - add whitelist to restrict installation to specified packages only From f165f8a3a8c8b286a73cd4d3316b583ad38ba466 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 28 Oct 2024 08:11:44 +0000 Subject: [PATCH 199/222] Update release notes --- .../00-release/files/release_notes.txt | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index 73ee6e93dc..c95afa5b5d 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -2,20 +2,30 @@ UNRELEASED: * labwc compositor now used as the default on all models of Raspberry Pi * wf-panel-pi now loads plugins dynamically at runtime * New raindrop screen configuration tool to replace arandr - * squeekboard on-screen keyboard installed; will automatically run on systems with touchscreens, and can be manually enabled in rc-gui for others + * squeekboard on-screen keyboard installed; will automatically run on systems + with touchscreens, and can be manually enabled in rc-gui for others + * Improved support for touchscreens - long press for right-click, and + double-tap for double-click * Update installer moved out of panel code into separate gui-updater package * Missing cursors added to icon theme - * Wizard no longer auto-pairs Bluetooth HID devices by default - create the file "/boot/firmware/btautopair" to enable, or reboot wizard after first run + * Wizard no longer auto-pairs Bluetooth HID devices by default - create the + file "/boot/firmware/btautopair" to enable, or reboot wizard after first run + * Modified handling of Pi 5 power button to reduce CPU load when in shutdown + dialog + * Raspberry Pi Connect now controlled by dedicated panel plugin instead of + system tray icon and option in rc-gui + * Screen sharing in Raspberry Pi Connect now available as soon as a running + WayVNC server is detected * Safety info URL added to main menu + * Chromium package renamed from chromium-browser to chromium * Various memory leaks in pcmanfm fixed * Bug fix - crash when running Appearance Settings on headless systems * Bug fix - crash when opening icon chooser in application properties dialog * Translations updated - * RPi.GPIO replaced with rpi-lgpio - * Chromium updated to 129.0.6668.89 - * Firefox updated to 131.0 + * Chromium updated to 130.0.6723.58 + * Firefox updated to 131.0.3 * Raspberry Pi firmware a2e586ba98ce68f7d11b1c717ad8329b95dcb3b6 - * Linux kernel 6.6.51 - 82a50e430ef1d6eb37d78e25aa572c1f6ea56160 + * Linux kernel 6.6.51 - 5aeecea9f4a45248bcf564dec924965e066a7bfd 2024-07-04: * pipanel - allow customisation of more than 2 desktops * pipanel - add customisation for labwc From 9dfbb1546db161697dcba1038aabebeb076b0942 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 28 Oct 2024 08:26:59 +0000 Subject: [PATCH 200/222] Update release notes --- export-noobs/00-release/files/release_notes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/export-noobs/00-release/files/release_notes.txt b/export-noobs/00-release/files/release_notes.txt index c95afa5b5d..e905541a99 100644 --- a/export-noobs/00-release/files/release_notes.txt +++ b/export-noobs/00-release/files/release_notes.txt @@ -1,4 +1,6 @@ UNRELEASED: + * +2024-10-22: * labwc compositor now used as the default on all models of Raspberry Pi * wf-panel-pi now loads plugins dynamically at runtime * New raindrop screen configuration tool to replace arandr From b8e497cf217402283b712af5be91c1384946429d Mon Sep 17 00:00:00 2001 From: Matthew Lear <matthew.lear@raspberrypi.com> Date: Tue, 29 Oct 2024 10:15:10 +0000 Subject: [PATCH 201/222] Allow export image customisation (#801) * Allow export image customisation Add new variable EXPORT_CONFIG_DIR to set the location of the scripts pigen will run when exporting an image. Setting this is optional. If not specified, the current location is retained. By utilising STAGE_LIST AND EXPORT_CONFIG_DIR, a user can construct custom images out-of-tree without modification to any defaults. (cherry picked from commit e5e6ceeaf46f52f77b759d3d35aef8bbd0a69c8b) * Shellcheck and style fixes Fixed shellcheck warnings and made the changes more consistent with the surrounding code --------- Co-authored-by: Serge Schneider <serge@raspberrypi.com> --- README.md | 4 ++++ build.sh | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 71be938e20..a246b5f424 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,10 @@ The following environment variables are supported: If set, then instead of working through the numeric stages in order, this list will be followed. For example setting to `"stage0 stage1 mystage stage2"` will run the contents of `mystage` before stage2. Note that quotes are needed around the list. An absolute or relative path can be given for stages outside the pi-gen directory. + * `EXPORT_CONFIG_DIR` (Default: `$BASE_DIR/export-image`) + + If set, use this directory path as the location of scripts to run when generating images. An absolute or relative path can be given for a location outside the pi-gen directory. + A simple example for building Raspberry Pi OS: ```bash diff --git a/build.sh b/build.sh index a96f4e4637..4864dd69e1 100755 --- a/build.sh +++ b/build.sh @@ -301,6 +301,13 @@ log "Begin ${BASE_DIR}" STAGE_LIST=${STAGE_LIST:-${BASE_DIR}/stage*} export STAGE_LIST +EXPORT_CONFIG_DIR=$(realpath "${EXPORT_CONFIG_DIR:-"${BASE_DIR}/export-image"}") +if [ ! -d "${EXPORT_CONFIG_DIR}" ]; then + echo "EXPORT_CONFIG_DIR invalid: ${EXPORT_CONFIG_DIR} does not exist" + exit 1 +fi +export EXPORT_CONFIG_DIR + for STAGE_DIR in $STAGE_LIST; do STAGE_DIR=$(realpath "${STAGE_DIR}") run_stage @@ -308,7 +315,7 @@ done CLEAN=1 for EXPORT_DIR in ${EXPORT_DIRS}; do - STAGE_DIR=${BASE_DIR}/export-image + STAGE_DIR=${EXPORT_CONFIG_DIR} # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_IMAGE" EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs From 2aa7133eea9d20721db43e00e3307a707d5eea49 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 29 Oct 2024 15:15:06 +0000 Subject: [PATCH 202/222] Create WORK_DIR earlier The exit trap uses the log function which writes to a log file in WORK_DIR. --- build.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/build.sh b/build.sh index 4864dd69e1..c53e6437c1 100755 --- a/build.sh +++ b/build.sh @@ -123,6 +123,20 @@ run_stage(){ log "End ${STAGE_DIR}" } +term() { + if [ "$?" -ne 0 ]; then + log "Build failed" + else + log "Build finished" + fi + unmount "${STAGE_WORK_DIR}" + if [ "$STAGE" = "export-image" ]; then + for img in "${STAGE_WORK_DIR}/"*.img; do + unmount_image "$img" + done + fi +} + if [ "$(id -u)" != "0" ]; then echo "Please run as root" 1>&2 exit 1 @@ -157,22 +171,6 @@ do esac done -term() { - if [ "$?" -ne 0 ]; then - log "Build failed" - else - log "Build finished" - fi - unmount "${STAGE_WORK_DIR}" - if [ "$STAGE" = "export-image" ]; then - for img in "${STAGE_WORK_DIR}/"*.img; do - unmount_image "$img" - done - fi -} - -trap term EXIT INT TERM - export PI_GEN=${PI_GEN:-pi-gen} export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen} export PI_GEN_RELEASE=${PI_GEN_RELEASE:-Raspberry Pi reference} @@ -252,6 +250,9 @@ if [ "$SETFCAP" != "1" ]; then export CAPSH_ARG="--drop=cap_setfcap" fi +mkdir -p "${WORK_DIR}" +trap term EXIT INT TERM + dependencies_check "${BASE_DIR}/depends" echo "Checking native $ARCH executable support..." @@ -295,7 +296,6 @@ if [[ "${PUBKEY_ONLY_SSH}" = "1" && -z "${PUBKEY_SSH_FIRST_USER}" ]]; then exit 1 fi -mkdir -p "${WORK_DIR}" log "Begin ${BASE_DIR}" STAGE_LIST=${STAGE_LIST:-${BASE_DIR}/stage*} From df6d0e2a25d920aa32f8f90fe7662ba8b98d2e4a Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 29 Oct 2024 15:32:15 +0000 Subject: [PATCH 203/222] Add kernel page size check --- build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.sh b/build.sh index c53e6437c1..c33abf4bcd 100755 --- a/build.sh +++ b/build.sh @@ -260,6 +260,13 @@ if ! arch-test -n "$ARCH"; then echo "WARNING: Only a native build environment is supported. Checking emulated support..." if ! arch-test "$ARCH"; then echo "No fallback mechanism found. Ensure your OS has binfmt_misc support enabled and configured." + PAGESIZE=$(getconf PAGESIZE) + if [ "$ARCH" == "armhf" ] && [ "$PAGESIZE" != "4096" ]; then + echo + echo "Building an $ARCH image requires a kernel with a 4k page size" + echo "Current pagesize: $PAGESIZE" + echo "On Raspberry Pi OS, you can switch to a suitable kernel by adding kernel=kernel8.img to /boot/firmware/config.txt and rebooting" + fi exit 1 fi fi From 1baee51b8cf7cd1e8e52dbe89171efccc0993200 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 30 Oct 2024 10:08:48 +0000 Subject: [PATCH 204/222] Improve PAGESIZE check and recommendation Setting kernel=kernel8.img while auto_initramfs=1 currently still loads initamfs_2712. This can cause issues on systems which require drivers compiled as modules to boot. Until this is fixed in the bootloader, users should specify initramfs options as well. Also, move the PAGESIZE check before the arch-test checks to make the output clearer. --- build.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/build.sh b/build.sh index c33abf4bcd..f35da03261 100755 --- a/build.sh +++ b/build.sh @@ -255,18 +255,24 @@ trap term EXIT INT TERM dependencies_check "${BASE_DIR}/depends" + +PAGESIZE=$(getconf PAGESIZE) +if [ "$ARCH" == "armhf" ] && [ "$PAGESIZE" != "4096" ]; then + echo + echo "ERROR: Building an $ARCH image requires a kernel with a 4k page size (current: $PAGESIZE)" + echo "On Raspberry Pi OS (64-bit), you can switch to a suitable kernel by adding the following to /boot/firmware/config.txt and rebooting:" + echo + echo "kernel=kernel8.img" + echo "initramfs initramfs8 followkernel" + echo + exit 1 +fi + echo "Checking native $ARCH executable support..." if ! arch-test -n "$ARCH"; then echo "WARNING: Only a native build environment is supported. Checking emulated support..." if ! arch-test "$ARCH"; then echo "No fallback mechanism found. Ensure your OS has binfmt_misc support enabled and configured." - PAGESIZE=$(getconf PAGESIZE) - if [ "$ARCH" == "armhf" ] && [ "$PAGESIZE" != "4096" ]; then - echo - echo "Building an $ARCH image requires a kernel with a 4k page size" - echo "Current pagesize: $PAGESIZE" - echo "On Raspberry Pi OS, you can switch to a suitable kernel by adding kernel=kernel8.img to /boot/firmware/config.txt and rebooting" - fi exit 1 fi fi From 536dd583b4eeef13c3a44964aab53cbc80d9b874 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 30 Oct 2024 11:32:47 +0000 Subject: [PATCH 205/222] README.md: Clarify early on that armhf and arm64 images should be build from different branches. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a246b5f424..51cd422845 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ which was in turn derived from the Raspbian project. **Note**: Raspberry Pi OS 32 bit images are based primarily on Raspbian, while Raspberry Pi OS 64 bit images are based primarily on Debian. +**Note**: 32 bit images should be built from the `master` branch. +64 bit images should be built from the `arm64` branch. + ## Dependencies pi-gen runs on Debian-based operating systems released after 2017, and we From 9dbd88221cdc7fa074242eca580d367758fd9023 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 30 Oct 2024 11:33:11 +0000 Subject: [PATCH 206/222] README.md: Fix TIMEZONE_DEFAULT description --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 51cd422845..e40fc26e76 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ The following environment variables are supported: * `TIMEZONE_DEFAULT` (Default: 'Europe/London' ) - Default keyboard layout. + Default time zone. To get the current value from a running system, look in `/etc/timezone`. From b91ab524b0dc1d46344e24a9ade7e39f4eb79c98 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Wed, 30 Oct 2024 12:46:21 +0000 Subject: [PATCH 207/222] README.md: Remove obsolete warning --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index e40fc26e76..73a526ba09 100644 --- a/README.md +++ b/README.md @@ -407,9 +407,6 @@ follows: # Troubleshooting ## `64 Bit Systems` -Please note there is currently an issue when compiling with a 64 Bit OS. See -https://github.com/RPi-Distro/pi-gen/issues/271 - A 64 bit image can be generated from the `arm64` branch in this repository. Just replace the command from [this section](#getting-started-with-building-your-images) by the one below, and follow the rest of the documentation: From e86b901e25b9d2554bbbcaf3500f0a41b273c0c2 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 31 Oct 2024 13:19:16 +0000 Subject: [PATCH 208/222] Rework wlan blocking --- stage2/02-net-tweaks/01-run.sh | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/stage2/02-net-tweaks/01-run.sh b/stage2/02-net-tweaks/01-run.sh index d49bdebfcd..7ec6f7395a 100755 --- a/stage2/02-net-tweaks/01-run.sh +++ b/stage2/02-net-tweaks/01-run.sh @@ -3,18 +3,25 @@ install -v -d "${ROOTFS_DIR}/etc/wpa_supplicant" install -v -m 600 files/wpa_supplicant.conf "${ROOTFS_DIR}/etc/wpa_supplicant/" +# Newer versions of raspberrypi-sys-mods set rfkill.default_state=0 to prevent +# radiating on 5GHz bands until the WLAN regulatory domain is set. +# Unfortunately, this also blocks bluetooth, so we whitelist the known +# on-board BT adapters here. + +mkdir -p "${ROOTFS_DIR}/var/lib/systemd/rfkill/" +# 5 miniuart 4 miniuart Zero miniuart other other +for addr in 107d50c000.serial 3f215040.serial 20215040.serial fe215040.serial soc; do + echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-${addr}:bluetooth" +done + if [ -v WPA_COUNTRY ]; then on_chroot <<- EOF SUDO_USER="${FIRST_USER_NAME}" raspi-config nonint do_wifi_country "${WPA_COUNTRY}" EOF -fi - -# Disable wifi on 5GHz models if WPA_COUNTRY is not set -mkdir -p "${ROOTFS_DIR}/var/lib/systemd/rfkill/" -if [ -n "$WPA_COUNTRY" ]; then - echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan" - echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan" -else - echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-3f300000.mmcnr:wlan" - echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-fe300000.mmcnr:wlan" +elif [ -d "${ROOTFS_DIR}/var/lib/NetworkManager" ]; then + # NetworkManager unblocks all WLAN devices by default. Prevent that: + cat > "${ROOTFS_DIR}/var/lib/NetworkManager/NetworkManager.state" <<- EOF + [main] + WirelessEnabled=false + EOF fi From c6d303831b9c6214b5d362e82bebb4070f24b291 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 31 Oct 2024 13:21:58 +0000 Subject: [PATCH 209/222] Remove wpa_supplicant tweaks --- stage2/02-net-tweaks/01-run.sh | 3 --- stage2/02-net-tweaks/files/wpa_supplicant.conf | 2 -- 2 files changed, 5 deletions(-) delete mode 100644 stage2/02-net-tweaks/files/wpa_supplicant.conf diff --git a/stage2/02-net-tweaks/01-run.sh b/stage2/02-net-tweaks/01-run.sh index 7ec6f7395a..d719c7902a 100755 --- a/stage2/02-net-tweaks/01-run.sh +++ b/stage2/02-net-tweaks/01-run.sh @@ -1,8 +1,5 @@ #!/bin/bash -e -install -v -d "${ROOTFS_DIR}/etc/wpa_supplicant" -install -v -m 600 files/wpa_supplicant.conf "${ROOTFS_DIR}/etc/wpa_supplicant/" - # Newer versions of raspberrypi-sys-mods set rfkill.default_state=0 to prevent # radiating on 5GHz bands until the WLAN regulatory domain is set. # Unfortunately, this also blocks bluetooth, so we whitelist the known diff --git a/stage2/02-net-tweaks/files/wpa_supplicant.conf b/stage2/02-net-tweaks/files/wpa_supplicant.conf deleted file mode 100644 index 0fc335ebb9..0000000000 --- a/stage2/02-net-tweaks/files/wpa_supplicant.conf +++ /dev/null @@ -1,2 +0,0 @@ -ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev -update_config=1 From fba95332dfc86c7d478f5b6bc9df0a9901fdf7ef Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 31 Oct 2024 16:20:18 +0000 Subject: [PATCH 210/222] Build initial initramfs with 'most' modules, then change the setting to 'dep' --- export-image/05-finalise/01-run.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/export-image/05-finalise/01-run.sh b/export-image/05-finalise/01-run.sh index 89c60b90ac..c59c23b5ab 100755 --- a/export-image/05-finalise/01-run.sh +++ b/export-image/05-finalise/01-run.sh @@ -4,8 +4,6 @@ IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" INFO_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.info" SBOM_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.sbom" -sed -i 's/^update_initramfs=.*/update_initramfs=all/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" - on_chroot << EOF update-initramfs -k all -c if [ -x /etc/init.d/fake-hwclock ]; then @@ -16,6 +14,11 @@ if hash hardlink 2>/dev/null; then fi EOF +if [ -f "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" ]; then + sed -i 's/^update_initramfs=.*/update_initramfs=yes/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf" + sed -i 's/^MODULES=.*/MODULES=dep/' "${ROOTFS_DIR}/etc/initramfs-tools/initramfs.conf" +fi + if [ -d "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config" ]; then chmod 700 "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/.config" fi From d7c4d3078c293441f225028b4790d8a4811cc06f Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 1 Nov 2024 11:43:15 +0000 Subject: [PATCH 211/222] Bump default swap to 512M --- stage2/01-sys-tweaks/00-patches/02-swap.diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/01-sys-tweaks/00-patches/02-swap.diff b/stage2/01-sys-tweaks/00-patches/02-swap.diff index f542ec2411..52cdda4a22 100644 --- a/stage2/01-sys-tweaks/00-patches/02-swap.diff +++ b/stage2/01-sys-tweaks/00-patches/02-swap.diff @@ -7,7 +7,7 @@ Index: jessie-stage2/rootfs/etc/dphys-swapfile # set size to absolute value, leaving empty (default) then uses computed value # you most likely don't want this, unless you have an special disk situation -#CONF_SWAPSIZE= -+CONF_SWAPSIZE=200 ++CONF_SWAPSIZE=512 # set size to computed value, this times RAM size, dynamically adapts, # guarantees that there is enough swap without wasting disk space on excess From 8fd4886e100b7243b5f1aa29d93e55907382605b Mon Sep 17 00:00:00 2001 From: Matt Liggett <mml@users.noreply.github.com> Date: Mon, 11 Nov 2024 00:02:33 -0800 Subject: [PATCH 212/222] Minor spelling and formatting tweaks. (#807) --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 73a526ba09..605f986c6b 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ can do so with: git clone https://github.com/RPI-Distro/pi-gen.git ``` -`--depth 1` can be added afer `git clone` to create a shallow clone, only containing +`--depth 1` can be added after `git clone` to create a shallow clone, only containing the latest revision of the repository. Do not do this on your development machine. Also, be careful to clone the repository to a base path **NOT** containing spaces. @@ -230,12 +230,12 @@ This is parsed after `config` so can be used to override values set there. The following process is followed to build images: - * Interate through all of the stage directories in alphanumeric order + * Iterate through all of the stage directories in alphanumeric order * Bypass a stage directory if it contains a file called "SKIP" - * Run the script ```prerun.sh``` which is generally just used to copy the build + * Run the script `prerun.sh` which is generally just used to copy the build directory between stages. * In each stage directory iterate through each subdirectory and then run each of the @@ -256,7 +256,7 @@ The following process is followed to build images: separated, per line. - **00-packages-nr** - As 00-packages, except these will be installed using - the ```--no-install-recommends -y``` parameters to apt-get. + the `--no-install-recommends -y` parameters to apt-get. - **00-patches** - A directory containing patch files to be applied, using quilt. If a file named 'EDIT' is present in the directory, the build process will @@ -399,8 +399,8 @@ follows: * Run build.sh to build all stages * Add SKIP files to the earlier successfully built stages * Modify the last stage - * Rebuild just the last stage using ```sudo CLEAN=1 ./build.sh``` (or, for docker builds - ```PRESERVE_CONTAINER=1 CONTINUE=1 CLEAN=1 ./build-docker.sh```) + * Rebuild just the last stage using `sudo CLEAN=1 ./build.sh` (or, for docker builds + `PRESERVE_CONTAINER=1 CONTINUE=1 CLEAN=1 ./build-docker.sh`) * Once you're happy with the image you can remove the SKIP_IMAGES files and export your image to test From 449d43e182e432ea0fee7e446f817d153c603087 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 11 Nov 2024 10:59:20 +0000 Subject: [PATCH 213/222] Exclude ifupdown as it slows down boot --- scripts/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/common b/scripts/common index 702576fd76..932a2ee7b4 100644 --- a/scripts/common +++ b/scripts/common @@ -12,7 +12,7 @@ bootstrap(){ BOOTSTRAP_ARGS+=(--arch armhf) BOOTSTRAP_ARGS+=(--components "main,contrib,non-free") BOOTSTRAP_ARGS+=(--keyring "${STAGE_DIR}/files/raspberrypi.gpg") - BOOTSTRAP_ARGS+=(--exclude=info) + BOOTSTRAP_ARGS+=(--exclude=info,ifupdown) BOOTSTRAP_ARGS+=(--include=ca-certificates) BOOTSTRAP_ARGS+=("$@") printf -v BOOTSTRAP_STR '%q ' "${BOOTSTRAP_ARGS[@]}" From c8d0e0b0bb14e04490cf7c6976d6045a48fb5f23 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Mon, 11 Nov 2024 13:21:13 +0000 Subject: [PATCH 214/222] Remove rc.local raspberrypi-sys-mods adds /etc/issue.d/IP.issue instead --- stage1/01-sys-tweaks/00-run.sh | 2 -- stage1/01-sys-tweaks/files/noclear.conf | 2 -- stage2/01-sys-tweaks/01-run.sh | 2 -- stage2/01-sys-tweaks/files/rc.local | 20 -------------------- 4 files changed, 26 deletions(-) delete mode 100644 stage1/01-sys-tweaks/files/noclear.conf delete mode 100755 stage2/01-sys-tweaks/files/rc.local diff --git a/stage1/01-sys-tweaks/00-run.sh b/stage1/01-sys-tweaks/00-run.sh index 0f761bd53e..e024701cec 100755 --- a/stage1/01-sys-tweaks/00-run.sh +++ b/stage1/01-sys-tweaks/00-run.sh @@ -1,7 +1,5 @@ #!/bin/bash -e -install -d "${ROOTFS_DIR}/etc/systemd/system/getty@tty1.service.d" -install -m 644 files/noclear.conf "${ROOTFS_DIR}/etc/systemd/system/getty@tty1.service.d/noclear.conf" install -v -m 644 files/fstab "${ROOTFS_DIR}/etc/fstab" on_chroot << EOF diff --git a/stage1/01-sys-tweaks/files/noclear.conf b/stage1/01-sys-tweaks/files/noclear.conf deleted file mode 100644 index 52671c758d..0000000000 --- a/stage1/01-sys-tweaks/files/noclear.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -TTYVTDisallocate=no diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index 897463e6df..be49d32b0e 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -6,8 +6,6 @@ install -m 644 files/50raspi "${ROOTFS_DIR}/etc/apt/apt.conf.d/" install -m 644 files/console-setup "${ROOTFS_DIR}/etc/default/" -install -m 755 files/rc.local "${ROOTFS_DIR}/etc/" - if [ -n "${PUBKEY_SSH_FIRST_USER}" ]; then install -v -m 0700 -o 1000 -g 1000 -d "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh echo "${PUBKEY_SSH_FIRST_USER}" >"${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys diff --git a/stage2/01-sys-tweaks/files/rc.local b/stage2/01-sys-tweaks/files/rc.local deleted file mode 100755 index 7d39ed7cf0..0000000000 --- a/stage2/01-sys-tweaks/files/rc.local +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -e -# -# rc.local -# -# This script is executed at the end of each multiuser runlevel. -# Make sure that the script will "exit 0" on success or any other -# value on error. -# -# In order to enable or disable this script just change the execution -# bits. -# -# By default this script does nothing. - -# Print the IP address -_IP=$(hostname -I) || true -if [ "$_IP" ]; then - printf "My IP address is %s\n" "$_IP" -fi - -exit 0 From b15527e15ba4d9e65126a454bfec2040b909dc1c Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 19 Nov 2024 12:32:43 +0000 Subject: [PATCH 215/222] Fix bluetooth rfkill unblock - 0 is unblocked --- stage2/02-net-tweaks/01-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/02-net-tweaks/01-run.sh b/stage2/02-net-tweaks/01-run.sh index d719c7902a..1f91095071 100755 --- a/stage2/02-net-tweaks/01-run.sh +++ b/stage2/02-net-tweaks/01-run.sh @@ -8,7 +8,7 @@ mkdir -p "${ROOTFS_DIR}/var/lib/systemd/rfkill/" # 5 miniuart 4 miniuart Zero miniuart other other for addr in 107d50c000.serial 3f215040.serial 20215040.serial fe215040.serial soc; do - echo 1 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-${addr}:bluetooth" + echo 0 > "${ROOTFS_DIR}/var/lib/systemd/rfkill/platform-${addr}:bluetooth" done if [ -v WPA_COUNTRY ]; then From 13c185cb7946efd667f6a516d2a493f24a9542a2 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Tue, 19 Nov 2024 11:49:05 +0000 Subject: [PATCH 216/222] Switch default journald storage to volatile --- stage2/01-sys-tweaks/01-run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index be49d32b0e..2db14cce0b 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -70,3 +70,5 @@ sed -i "s/PLACEHOLDER//" "${ROOTFS_DIR}/etc/default/keyboard" on_chroot << EOF DEBIAN_FRONTEND=noninteractive dpkg-reconfigure keyboard-configuration EOF + +sed -i 's/^#?Storage=.*/Storage=volatile/' "${ROOTFS_DIR}/etc/systemd/journald.conf" From f2d397f3ba2112bc5a8b7051c8c31dac04b62013 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 21 Nov 2024 15:57:13 +0000 Subject: [PATCH 217/222] stage2: set publish-workstation to yes in avahi-daemon.conf --- stage2/01-sys-tweaks/01-run.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index 2db14cce0b..961daed748 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -72,3 +72,7 @@ DEBIAN_FRONTEND=noninteractive dpkg-reconfigure keyboard-configuration EOF sed -i 's/^#?Storage=.*/Storage=volatile/' "${ROOTFS_DIR}/etc/systemd/journald.conf" + +if [ -e "${ROOTFS_DIR}/etc/avahi/avahi-daemon.conf" ]; then + sed -i 's/^#?publish-workstation=.*/publish-workstation=yes/' "${ROOTFS_DIR}/etc/avahi/avahi-daemon.conf" +fi From 8e30e7943412e949ca349b76192490c918c64e5f Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Fri, 22 Nov 2024 14:38:27 +0000 Subject: [PATCH 218/222] stage5: install code-the-classics-2 --- stage5/00-install-extras/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage5/00-install-extras/00-packages b/stage5/00-install-extras/00-packages index 85474ddb2f..c871e09c73 100644 --- a/stage5/00-install-extras/00-packages +++ b/stage5/00-install-extras/00-packages @@ -3,5 +3,5 @@ scratch nuscratch scratch3 wolfram-engine claws-mail realvnc-vnc-viewer -code-the-classics +code-the-classics code-the-classics-2 kicad From ca08900c66c5eb4dc3444ce63e91ee081107ffa2 Mon Sep 17 00:00:00 2001 From: marcone <48169102+marcone@users.noreply.github.com> Date: Thu, 28 Nov 2024 03:05:29 -0800 Subject: [PATCH 219/222] README.md: Fix typo (#813) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 605f986c6b..105f561758 100644 --- a/README.md +++ b/README.md @@ -423,7 +423,7 @@ work from a Raspberry Pi with a 64-bit capable processor (i.e. Raspberry Pi Zero ## `binfmt_misc` -Linux is able execute binaries from other architectures, meaning that it should be +Linux is able to execute binaries from other architectures, meaning that it should be possible to make use of `pi-gen` on an x86_64 system, even though it will be running ARM binaries. This requires support from the [`binfmt_misc`](https://en.wikipedia.org/wiki/Binfmt_misc) kernel module. From fc280356b177100ae09fa3a46df8ac2349809358 Mon Sep 17 00:00:00 2001 From: Serge Schneider <serge@raspberrypi.com> Date: Thu, 2 Jan 2025 09:27:21 +0000 Subject: [PATCH 220/222] Escape the ? expression in sed Fixes #817, fixes #820 --- stage2/01-sys-tweaks/01-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/01-sys-tweaks/01-run.sh b/stage2/01-sys-tweaks/01-run.sh index 961daed748..aca8074083 100755 --- a/stage2/01-sys-tweaks/01-run.sh +++ b/stage2/01-sys-tweaks/01-run.sh @@ -71,8 +71,8 @@ on_chroot << EOF DEBIAN_FRONTEND=noninteractive dpkg-reconfigure keyboard-configuration EOF -sed -i 's/^#?Storage=.*/Storage=volatile/' "${ROOTFS_DIR}/etc/systemd/journald.conf" +sed -i 's/^#\?Storage=.*/Storage=volatile/' "${ROOTFS_DIR}/etc/systemd/journald.conf" if [ -e "${ROOTFS_DIR}/etc/avahi/avahi-daemon.conf" ]; then - sed -i 's/^#?publish-workstation=.*/publish-workstation=yes/' "${ROOTFS_DIR}/etc/avahi/avahi-daemon.conf" + sed -i 's/^#\?publish-workstation=.*/publish-workstation=yes/' "${ROOTFS_DIR}/etc/avahi/avahi-daemon.conf" fi From fb5180d3de101a545c755f2f34ddef0042a6b549 Mon Sep 17 00:00:00 2001 From: Stefan Becker <stefanb2@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:26:42 +0200 Subject: [PATCH 221/222] Dockerfile: fix ENV warning (#791) Newer versions of docker generate the following warning - LegacyKeyValueFormat: "ENV key=value" should be used instead of legacy "ENV key value" format (line 4) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0e6d86df2f..cf03b0678e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG BASE_IMAGE=debian:bullseye FROM ${BASE_IMAGE} -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get -y update && \ apt-get -y install --no-install-recommends \ From e071d0de36561a13039bf4d561f5204a7ecd8e34 Mon Sep 17 00:00:00 2001 From: Stefan Becker <chemobejk@gmail.com> Date: Wed, 18 Oct 2023 12:22:27 +0300 Subject: [PATCH 222/222] Switch docker build to Debian Bookworm --- build-docker.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index 41e341f59e..0591c342c4 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -85,10 +85,10 @@ BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')" # Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead case "$(uname -m)" in x86_64|aarch64) - BASE_IMAGE=i386/debian:bullseye + BASE_IMAGE=i386/debian:bookworm ;; *) - BASE_IMAGE=debian:bullseye + BASE_IMAGE=debian:bookworm ;; esac ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" @@ -151,7 +151,7 @@ time ${DOCKER} run \ pi-gen \ bash -e -o pipefail -c " dpkg-reconfigure qemu-user-static && - # binfmt_misc is sometimes not mounted with debian bullseye image + # binfmt_misc is sometimes not mounted with debian bookworm image (mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) && cd /pi-gen; ./build.sh ${BUILD_OPTS} && rsync -av work/*/build.log deploy/