summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2018-10-05 10:06:15 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2018-10-05 10:06:28 +0000
commit654f31d3e765fdbb005304fa8857df0a9475f3ea (patch)
treef48563fd36298219a836a101b7a47a9265d5a96d
parentReleasing progress-linux version 0.1.0-2~dschinn1. (diff)
downloadmmdebstrap-654f31d3e765fdbb005304fa8857df0a9475f3ea.zip
mmdebstrap-654f31d3e765fdbb005304fa8857df0a9475f3ea.tar.xz
Merging upstream version 0.2.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
-rw-r--r--CHANGELOG.md11
-rwxr-xr-xmake_mirror.sh7
-rwxr-xr-xmmdebstrap184
-rwxr-xr-xtest.sh54
4 files changed, 176 insertions, 80 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..e3efe70
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,11 @@
+0.2.0 (2018-10-03)
+------------------
+
+ - if no MIRROR was specified but there was data on standard input, then use
+ that data as the sources.list instead of falling back to the default mirror
+ - lots of bug fixes
+
+0.1.0 (2018-09-24)
+------------------
+
+ - initial release
diff --git a/make_mirror.sh b/make_mirror.sh
index 9b4a95a..168396d 100755
--- a/make_mirror.sh
+++ b/make_mirror.sh
@@ -3,6 +3,7 @@
set -eu
mirrordir="./mirror"
+cachedir="./cache"
mirror="http://deb.debian.org/debian"
nativearch=$(dpkg --print-architecture)
@@ -18,6 +19,12 @@ if [ -e "$mirrordir/dists/unstable/Release" ]; then
fi
for dist in stable testing unstable; do
+ for variant in minbase buildd -; do
+ rm -f "$cachedir/debian-$dist-$variant.tar"
+ done
+done
+
+for dist in stable testing unstable; do
rootdir=$(mktemp --directory)
for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt /var/lib/apt/lists/partial /var/lib/dpkg; do
diff --git a/mmdebstrap b/mmdebstrap
index 05c105c..0b3b456 100755
--- a/mmdebstrap
+++ b/mmdebstrap
@@ -95,7 +95,7 @@ sub test_unshare() {
# otherwise we will get "Modification of a read-only value attempted"
my $unshare_flags = CLONE_NEWUSER;
# we spawn a new per process because if unshare succeeds, we would
- # otherwise have unshared the sbuild process itself which we don't want
+ # otherwise have unshared the mmdebstrap process itself which we don't want
my $pid = fork() // die "fork() failed: $!";
if ($pid == 0) {
my $ret = syscall &SYS_unshare, $unshare_flags;
@@ -367,7 +367,7 @@ sub havemknod($) {
if (-e "$root/test-dev-null") {
die "/test-dev-null already exists";
}
- while ($havemknod == 0) {
+ TEST: {
# we fork so that we can read STDERR
my $pid = open my $fh, '-|' // die "failed to fork(): $!";
if ($pid == 0) {
@@ -379,10 +379,10 @@ sub havemknod($) {
chomp (my $content = do { local $/; <$fh> });
close $fh;
{
- last unless $? == 0 and $content eq '';
- last unless -c "$root/test-dev-null";
- last unless open my $fh, '>', "$root/test-dev-null";
- last unless print $fh 'test';
+ last TEST unless $? == 0 and $content eq '';
+ last TEST unless -c "$root/test-dev-null";
+ last TEST unless open my $fh, '>', "$root/test-dev-null";
+ last TEST unless print $fh 'test';
}
$havemknod = 1;
}
@@ -667,38 +667,10 @@ sub setup {
chmod 0644, "$options->{root}/etc/fstab" or die "cannot chmod fstab: $!";
}
+ # write /etc/apt/sources.list
{
- my $archopt = '';
- if (scalar @{$options->{foreignarchs}} > 0) {
- $archopt = " [arch=$options->{nativearch}]";
- }
open my $fh, '>', "$options->{root}/etc/apt/sources.list" or die "cannot open /etc/apt/sources.list: $!";
- if (scalar(@{$options->{mirrors}}) > 0) {
- if((grep /^-$/, @{$options->{mirrors}}) > 1 ) {
- die "can only read from stdin once";
- }
- for my $arg (@{$options->{mirrors}}) {
- if ($arg eq '-') {
- # read from stdin
- print STDERR "I: Reading sources.list from standard input...\n";
- copy *STDIN, $fh or die "cannot copy stdin: $!";
- } elsif ($arg =~ /:\/\//) {
- print $fh "deb$archopt $arg $options->{suite} $options->{components}\n";
- } elsif ($arg =~ /^deb(-src)? /) {
- print $fh "$arg\n";
- } elsif (-f $arg) {
- copy($arg, $fh) or die "cannot copy $arg: $!";
- } else {
- die "invalid mirror: $arg";
- }
- }
- } else {
- print $fh "deb$archopt http://deb.debian.org/debian $options->{suite} $options->{components}\n";
- if (any { $_ eq $options->{suite} } ('stable', 'oldstable', 'stretch') ) {
- print $fh "deb$archopt http://deb.debian.org/debian $options->{suite}-updates $options->{components}\n";
- print $fh "deb$archopt http://security.debian.org/debian-security $options->{suite}/updates $options->{components}\n";
- }
- }
+ print $fh $options->{sourceslist};
close $fh;
}
@@ -718,6 +690,8 @@ sub setup {
chomp (my $indextargets = do { local $/; <$fh> });
close $fh;
if ($indextargets eq '') {
+ print STDERR "content of /etc/apt/sources.list:\n";
+ copy("$options->{root}/etc/apt/sources.list", *STDERR);
die "apt-get update didn't download anything";
}
}
@@ -935,7 +909,7 @@ sub setup {
if (!-e $qemubin) {
die "cannot find $qemubin";
}
- copy $qemubin, "$options->{root}/$qemubin";
+ copy $qemubin, "$options->{root}/$qemubin" or die "cannot copy $qemubin: $!";
}
}
@@ -949,6 +923,7 @@ sub setup {
die "type 1 not implemented";
} elsif ($type == 2) { # symlink
# nothing to do
+ next;
} elsif ($type == 3) { # character special
0 == system('mknod', "$options->{root}/$fname", 'c', $devmajor, $devminor) or die "mknod failed: $?";
} elsif ($type == 4) { # block special
@@ -961,7 +936,7 @@ sub setup {
} else {
die "unsupported type: $type";
}
- chmod $mode, "$options->{root}/$fname";
+ chmod $mode, "$options->{root}/$fname" or die "cannot chmod $fname: $!";
}
}
@@ -992,6 +967,16 @@ sub setup {
if (%pkgs_to_install) {
# some packages have to be installed from the outside before anything
# can be installed from the inside.
+ #
+ # we do not need to install any *-archive-keyring packages inside the
+ # chroot prior to installing the packages, because the keyring is only
+ # used when doing "apt-get update" and that was already done at the
+ # beginning using key material from the outside. Since the apt cache
+ # is already filled and we are not calling "apt-get update" again, the
+ # keyring can be installed later during installation. But: if it's not
+ # installed during installation, then we might end up with a fully
+ # installed system without keyrings that are valid for its
+ # sources.list.
my %pkgs_to_install_from_outside;
# install apt if necessary
@@ -1068,17 +1053,23 @@ sub setup {
} elsif ($type == 5) { # directory
if (!$options->{havemknod}) {
make_path "$options->{root}/$fname";
+ chmod $mode, "$options->{root}/$fname" or die "cannot chmod $fname: $!";
}
0 == system('mount', '-o', 'bind', "/$fname", "$options->{root}/$fname") or die "mount failed: $?";
} else {
die "unsupported type: $type";
}
- chmod $mode, "$options->{root}/$fname";
}
# We can only mount /proc and /sys after extracting the essential
- # set because if we mount it before, then base-files not be able
+ # set because if we mount it before, then base-files will not be able
# to extract those
- 0 == system('mount', '-o', 'rbind', '/sys', "$options->{root}/sys") or die "mount failed: $?";
+ if ($options->{mode} eq 'unshare') {
+ # without the network namespace unshared, we cannot mount a new
+ # sysfs. Since we need network, we just bind-mount.
+ 0 == system('mount', '-o', 'rbind', '/sys', "$options->{root}/sys") or die "mount failed: $?";
+ } else {
+ 0 == system('mount', '-t', 'sysfs', '-o', 'nosuid,nodev,noexec', 'sys', "$options->{root}/sys") or die "mount failed: $?";
+ }
0 == system('mount', '-t', 'proc', 'proc', "$options->{root}/proc") or die "mount failed: $?";
# prevent daemons from starting
@@ -1100,8 +1091,8 @@ sub setup {
}
# allow network access from within
- copy("/etc/resolv.conf", "$options->{root}/etc/resolv.conf");
- copy("/etc/hostname", "$options->{root}/etc/hostname");
+ copy("/etc/resolv.conf", "$options->{root}/etc/resolv.conf") or die "cannot copy /etc/resolv.conf: $!";
+ copy("/etc/hostname", "$options->{root}/etc/hostname") or die "cannot copy /etc/hostname: $!";
print STDERR "I: installing remaining packages inside the chroot...\n";
run_apt_progress @chrootcmd, 'apt-get', '--yes', 'install', keys %pkgs_to_install;
@@ -1135,7 +1126,7 @@ sub setup {
}
# no need to umount if the mount namespace was unshared
if ($options->{mode} ne 'unshare') {
- 0 == system('umount', '--no-mtab', '--recursive', '--lazy', "$options->{root}/sys") or die "umount failed: $?";
+ 0 == system('umount', '--no-mtab', "$options->{root}/sys") or die "umount failed: $?";
0 == system('umount', '--no-mtab', "$options->{root}/proc") or die "umount failed: $?";
}
}
@@ -1172,6 +1163,7 @@ sub setup {
unlink "$options->{root}/var/lib/apt/extended_states";
unlink "$options->{root}/var/lib/apt/lists/lock";
unlink "$options->{root}/var/lib/dpkg/lock-frontend";
+ unlink "$options->{root}/var/lib/dpkg/lock";
}
sub main() {
@@ -1244,6 +1236,7 @@ sub main() {
$options->{foreignarchs} = \@foreignarchs;
{
+ # FIXME: autogenerate this list
my $deb2qemu = {
alpha => 'alpha',
amd64 => 'x86_64',
@@ -1271,7 +1264,10 @@ sub main() {
if ($hostarch ne $nativearch) {
my $pid = open my $fh, '-|' // die "failed to fork(): $!";
if ($pid == 0) {
- { exec 'arch-test', '-n', $nativearch; }
+ {
+ no warnings; # don't print a warning if the following fails
+ exec 'arch-test', '-n', $nativearch;
+ }
# if exec didn't work (for example because the arch-test program is
# missing) prepare for the worst and assume that the architecture
# cannot be executed
@@ -1294,19 +1290,77 @@ sub main() {
}
}
- if (scalar @ARGV > 0) {
- $options->{suite} = shift @ARGV;
+ {
+ my $suite;
if (scalar @ARGV > 0) {
- $options->{target} = shift @ARGV;
+ $suite = shift @ARGV;
+ if (scalar @ARGV > 0) {
+ $options->{target} = shift @ARGV;
+ } else {
+ $options->{target} = '-';
+ }
} else {
+ print STDERR "I: No SUITE specified, expecting sources.list on standard input\n";
$options->{target} = '-';
}
- $options->{mirrors} = [@ARGV];
- } else {
- print STDERR "I: No SUITE specified, expecting sources.list on standard input\n";
- $options->{suite} = 'UNDEFINED';
- $options->{target} = '-';
- $options->{mirrors} = ['-'];
+
+ my $sourceslist = '';
+ my $stdindata = '';
+ # make sure that we only attempt to read from STDIN if it's *not*
+ # connected to the terminal (because we don't expect the user to type
+ # the sources.list file
+ if (! -t STDIN) {
+ print STDERR "I: Reading sources.list from standard input...\n";
+ $stdindata = do { local $/; <STDIN> };
+ }
+ if (! defined $suite) {
+ # If no suite was specified, then the whole sources.list has to
+ # come from standard input
+ $sourceslist .= $stdindata;
+ } else {
+ my $archopt = '';
+ if (scalar @{$options->{foreignarchs}} > 0) {
+ $archopt = " [arch=$options->{nativearch}]";
+ }
+ if (scalar @ARGV > 0) {
+ for my $arg (@ARGV) {
+ if ($arg eq '-') {
+ $sourceslist .= $stdindata;
+ } elsif ($arg =~ /^deb(-src)? /) {
+ $sourceslist .= "$arg\n";
+ } elsif ($arg =~ /:\/\//) {
+ $sourceslist .= "deb$archopt $arg $suite $options->{components}\n";
+ } elsif (-f $arg) {
+ open my $fh, '<', $arg or die "cannot open $arg: $!";
+ while (my $line = <$fh>) {
+ $sourceslist .= $line;
+ }
+ close $fh;
+ } else {
+ die "invalid mirror: $arg";
+ }
+ }
+ # if there was no explicit '-' mirror listed and something was
+ # read on standard input, then just append it to the end
+ if (none { $_ eq '-' } @ARGV) {
+ # if nothing was read on standard input then nothing will
+ # be appended
+ $sourceslist .= $stdindata;
+ }
+ } elsif ($stdindata ne '') {
+ $sourceslist .= $stdindata;
+ } else {
+ $sourceslist .= "deb$archopt http://deb.debian.org/debian $suite $options->{components}\n";
+ if (any { $_ eq $suite } ('stable', 'oldstable', 'stretch') ) {
+ $sourceslist .= "deb$archopt http://deb.debian.org/debian $suite-updates $options->{components}\n";
+ $sourceslist .= "deb$archopt http://security.debian.org/debian-security $suite/updates $options->{components}\n";
+ }
+ }
+ }
+ if ($sourceslist eq '') {
+ die "empty apt sources.list";
+ }
+ $options->{sourceslist} = $sourceslist;
}
if ($options->{target} ne '-') {
@@ -1578,11 +1632,9 @@ sub main() {
close $wfh;
if ($options->{maketar}) {
if ($options->{target} ne '-') {
- open(my $fh, '>', $options->{target});
- copy($rfh, $fh);
- close($fh);
+ copy($rfh, $options->{target}) or die "cannot copy to $options->{target}: $!";
} else {
- copy($rfh, *STDOUT);
+ copy($rfh, *STDOUT) or die "cannot copy to standard output: $!";
}
}
close($rfh);
@@ -1644,7 +1696,9 @@ section B<DEBOOTSTRAP>). In contrast to debootstrap it uses apt to resolve
dependencies and is thus able to use more than one mirror and resolve more
complex dependencies.
-If no I<MIRROR> option is provided, http://deb.debian.org/debian is used. If
+If no I<MIRROR> option is provided, http://deb.debian.org/debian is used,
+except if data was given on standard input in which case the lines read from
+there are used as the content of the chroot's sources.list file. If
I<SUITE> is a stable release name, then mirrors for updates and security are
automatically added. If a I<MIRROR> option starts with "deb " or "deb-src "
then it is used as a one-line-style format entry for apt's sources.list inside
@@ -1653,11 +1707,13 @@ a mirror URI and the apt line inside the chroot is assembled as "deb [arch=A]
B C D" where A is the host's native architecture, B is the I<MIRROR>, C is the
given I<SUITE> and D is the components given via --components (defaults to
"main"). If a I<MIRROR> option happens to be an existing file, then its
-contents are pasted into the chroot's sources.list. This can be used to supply
-a deb822 style sources.list. If I<MIRROR> is C<-> then standard input is
-pasted into the chroot's sources.list. More than one mirror can be specified
-and are appended to the chroot's sources.list in the given order. If any
-mirror contains a https URI, then the packages apt-transport-https and
+contents are pasted into the chroot's sources.list. This can be used to
+supply a deb822 style sources.list. If I<MIRROR> is C<-> then standard input
+is pasted into the chroot's sources.list. If there was data on standard input
+but no C<-> mirror was listed, the lines read from standard input will be
+appended to the end of the chroot's sources.list. More than one mirror can be
+specified and are appended to the chroot's sources.list in the given order. If
+any mirror contains a https URI, then the packages apt-transport-https and
ca-certificates will be installed inside the chroot.
The I<TARGET> argument can either be a directory or a tarball filename. If
diff --git a/test.sh b/test.sh
index a0501e2..e0a254e 100755
--- a/test.sh
+++ b/test.sh
@@ -3,11 +3,15 @@
set -eu
mirrordir="./mirror"
+cachedir="./cache"
# by default, use the mmdebstrap executable in the current directory but allow
# overwriting the location
: "${mmdebstrap:=./mmdebstrap}"
+# by default a cache of debootstrap results is maintained
+: "${docache:=true}"
+
mirror="http://deb.debian.org/debian"
rootdir=$(mktemp --directory)
nativearch=$(dpkg --print-architecture)
@@ -30,17 +34,27 @@ fi
./make_mirror.sh
-trap 'kill $pid' INT QUIT TERM EXIT
-
cd mirror
python3 -m http.server 8000 2>/dev/null & pid=$!
cd -
# wait for the server to start
sleep 1
+
+if ! kill -0 $pid; then
+ echo "failed to start http server"
+ exit 1
+fi
+
+trap 'kill $pid' INT QUIT TERM EXIT
+
echo "running http server with pid $pid"
-export SOURCE_DATE_EPOCH=$(date +%s)
+# choose the timestamp of the unstable Release file, so that we get
+# reproducible results for the same mirror timestamp
+export SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' mirror/dists/unstable/Release)" +%s)
+
+mkdir -p "$cachedir"
for dist in stable testing unstable; do
> timings
@@ -59,20 +73,20 @@ for dist in stable testing unstable; do
stat --format=%s debian-$dist-mm.tar >> sizes
mkdir ./debian-$dist-mm
- cd ./debian-$dist-mm
- sudo tar -xf ../debian-$dist-mm.tar
- cd -
-
- echo running debootstrap --merged-usr --variant=$variant $dist ./debian-$dist-debootstrap "http://localhost:8000/"
- /usr/bin/time --output=timings --append --format=%e sudo debootstrap --merged-usr --variant=$variant $dist ./debian-$dist-debootstrap "http://localhost:8000/"
- sudo tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system -C ./debian-$dist-debootstrap -cf debian-$dist-debootstrap.tar .
- sudo rm -r ./debian-$dist-debootstrap
+ sudo tar -C ./debian-$dist-mm -xf ./debian-$dist-mm.tar
+
+ if [ "$docache" != "true" ] || [ ! -e "$cachedir/debian-$dist-$variant.tar" ]; then
+ echo running debootstrap --merged-usr --variant=$variant $dist ./debian-$dist-debootstrap "http://localhost:8000/"
+ /usr/bin/time --output=timings --append --format=%e sudo debootstrap --merged-usr --variant=$variant $dist ./debian-$dist-debootstrap "http://localhost:8000/"
+ sudo tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system -C ./debian-$dist-debootstrap -c . > "$cachedir/debian-$dist-$variant.tar"
+ sudo rm -r ./debian-$dist-debootstrap
+ else
+ echo cached >> timings
+ fi
- stat --format=%s debian-$dist-debootstrap.tar >> sizes
+ stat --format=%s "$cachedir/debian-$dist-$variant.tar" >> sizes
mkdir ./debian-$dist-debootstrap
- cd ./debian-$dist-debootstrap
- sudo tar -xf ../debian-$dist-debootstrap.tar
- cd -
+ sudo tar -C ./debian-$dist-debootstrap -xf "$cachedir/debian-$dist-$variant.tar"
# diff cannot compare device nodes, so we use tar to do that for us and then
# delete the directory
@@ -133,12 +147,20 @@ for dist in stable testing unstable; do
sudo rm debian-$dist-debootstrap/var/lib/systemd/catalog/database
sudo rm debian-$dist-mm/var/lib/systemd/catalog/database
fi
+ sudo rm debian-$dist-debootstrap/var/lib/dpkg/lock
+ # introduced in dpkg 1.19.1
+ if [ "$dist" = "unstable" ]; then
+ sudo rm debian-$dist-debootstrap/var/lib/dpkg/lock-frontend
+ fi
# check if the file content differs
sudo diff --no-dereference --brief --recursive debian-$dist-debootstrap debian-$dist-mm
sudo rm -rf ./debian-$dist-debootstrap ./debian-$dist-mm \
- ./debian-$dist-debootstrap.tar ./debian-$dist-mm.tar
+ ./debian-$dist-mm.tar
+ if [ "$docache" != "true" ]; then
+ rm "$cachedir/debian-$dist-$variant.tar"
+ fi
done
eval $(awk '{print "var"NR"="$1}' timings)