diff --git a/Containerfile.base b/Containerfile.base index 02448c2..bc72d25 100644 --- a/Containerfile.base +++ b/Containerfile.base @@ -1,7 +1,7 @@ FROM registry.fedoraproject.org/fedora:latest # Install required tools -RUN dnf install -y rsync nginx && \ +RUN dnf install -y nginx && \ dnf clean all # Configure nginx diff --git a/README.md b/README.md index c479d7a..52a2472 100644 --- a/README.md +++ b/README.md @@ -8,21 +8,21 @@ Create & serve the mirror. ```sh # Pre-requisites -sudo dnf install -y podman buildah skopeo curl lorax +sudo dnf install -y podman buildah skopeo curl lorax rsync # Create a local mirror of CentOS Stream 10 -sudo ./build.sh +./build.sh # Serve the mirror on port 8080 -sudo podman run -d --rm --name mirror-centos-stream-10-$(date -I) -p 8080:8080 localhost/mirrors/centos-stream-10:$(date -I) +podman run -d --rm --name mirror-centos-stream-10-$(date -I) -p 8080:8080 localhost/mirrors/centos-stream-10:$(date -I) # Mirror is alive! curl http://localhost:8080/centos/10-stream/BaseOS/x86_64/iso/SHA256SUM # Archive the mirror for posterity -sudo podman save --output centos-stream-10-$(date -I) --format oci-dir --uncompressed localhost/mirrors/centos-stream-10:$(date -I) -sudo podman tag localhost/mirrors/centos-stream-10:$(date -I) quay.io/nmasse-redhat/centos-stream-10:$(date -I) -sudo buildah push --disable-compression quay.io/nmasse-redhat/centos-stream-10:$(date -I) +podman save --output centos-stream-10-$(date -I) --format oci-dir --uncompressed localhost/mirrors/centos-stream-10:$(date -I) +podman tag localhost/mirrors/centos-stream-10:$(date -I) quay.io/nmasse-redhat/centos-stream-10:$(date -I) +buildah push --disable-compression quay.io/nmasse-redhat/centos-stream-10:$(date -I) # Install a VM from this mirror using Kickstart sudo mkdir -p /var/lib/libvirt/images/test-centos10 diff --git a/build.sh b/build.sh index 0529921..d9a1513 100755 --- a/build.sh +++ b/build.sh @@ -4,39 +4,63 @@ set -Eeuo pipefail # The version of CentOS Stream to mirror. declare CENTOS_VERSION="10" +declare EPEL_VERSION="${EPEL_VERSION:-${CENTOS_VERSION}}" + +# Sync process configuration +declare IMAGE_BASE="localhost/mirrors/centos-stream-${CENTOS_VERSION}" +declare IMAGE_TAG="$(date -I)" +declare RSYNC_MIRROR="${RSYNC_MIRROR:-rsync://mirror.in2p3.fr}" +declare CENTOS_PATH="${CENTOS_PATH:-/pub/linux/centos-stream}" +declare EPEL_PATH="${EPEL_PATH:-/pub/epel}" +declare RSYNC_OPTS="-azH --progress --delete --exclude-from=${BASH_SOURCE[0]%/*}/rsync-excludes.txt" + +# Pre-requisites +if ! rsync -V &>/dev/null; then + echo "rsync is required on the host to synchronize the repositories. Please install it and try again." >&2 + exit 1 +fi ## ## First stage: build the base image with the necessary tools to synchronize the repositories. ## -if ! podman image inspect "localhost/mirrors/centos-stream-${CENTOS_VERSION}:base" &>/dev/null; then +if ! podman image inspect "${IMAGE_BASE}:base" &>/dev/null; then # Compute podman command line arguments declare -a PODMAN_ARGS=() PODMAN_ARGS+=( --file Containerfile.base ) - PODMAN_ARGS+=( -t "localhost/mirrors/centos-stream-${CENTOS_VERSION}:base" ) + PODMAN_ARGS+=( -t "${IMAGE_BASE}:base" ) podman build "${PODMAN_ARGS[@]}" . fi ## -## Second stage: build the image with the synchronized repositories. +## Second stage: build the image with the synchronized repositories using buildah. ## -if ! podman image inspect "localhost/mirrors/centos-stream-${CENTOS_VERSION}:latest" &>/dev/null; then - podman tag "localhost/mirrors/centos-stream-${CENTOS_VERSION}:base" "localhost/mirrors/centos-stream-${CENTOS_VERSION}:latest" +if ! podman image inspect "${IMAGE_BASE}:latest" &>/dev/null; then + podman tag "${IMAGE_BASE}:base" "${IMAGE_BASE}:latest" fi -# Tag the resulting image with the current date -declare TS="$(date -I)" +# TODO: Maybe let the user specify the Buildah container name to be able have different sessions running in parallel? +# Currently, the container name is deterministic to be able to resume a build if it fails or is interrupted. +# In that case, the synchronization will be resumed from the last successful step instead of starting from scratch. +# But, it is assumed that only one build per version of CentOS Stream is running at the same time to avoid conflicts on the container name. +BUILDAH_CONTAINER_NAME="buildah-sync-centos-${CENTOS_VERSION}" +if buildah inspect "$BUILDAH_CONTAINER_NAME" &>/dev/null; then + echo "Resuming build with existing Buildah container: ${BUILDAH_CONTAINER_NAME}" +else + # Create Buildah container from the base image. + echo "Creating Buildah container from ${IMAGE_BASE}:latest..." + BUILDAH_CONTAINER_NAME=$(buildah from --name="${BUILDAH_CONTAINER_NAME}" "${IMAGE_BASE}:latest") +fi -# Compute podman command line arguments -declare -a PODMAN_ARGS=() -PODMAN_ARGS+=( --build-arg CENTOS_VERSION="${CENTOS_VERSION}" --build-arg EPEL_VERSION="${CENTOS_VERSION}" ) -PODMAN_ARGS+=( --file Containerfile.sync ) -PODMAN_ARGS+=( --from "localhost/mirrors/centos-stream-${CENTOS_VERSION}:latest" ) -PODMAN_ARGS+=( -t "localhost/mirrors/centos-stream-${CENTOS_VERSION}:${TS}" ) -podman build "${PODMAN_ARGS[@]}" . +# Expect unprivileged buildah, so it is mandatory to run the sync script from an dedicated user namespace. +echo "Starting synchronization in a modified user namespace..." +export CENTOS_VERSION EPEL_VERSION RSYNC_MIRROR CENTOS_PATH EPEL_PATH RSYNC_OPTS +buildah unshare --mount=BUILDAH_ROOT="$BUILDAH_CONTAINER_NAME" "${BASH_SOURCE[0]%/*}/sync.sh" -# Tag the image with "latest" as well. -podman tag "localhost/mirrors/centos-stream-${CENTOS_VERSION}:${TS}" "localhost/mirrors/centos-stream-${CENTOS_VERSION}:latest" +# Finalize the image +echo "Creating final image ${IMAGE_BASE} with tag ${IMAGE_TAG} + latest..." +buildah umount "$BUILDAH_CONTAINER_NAME" +buildah commit --quiet "$BUILDAH_CONTAINER_NAME" "${IMAGE_BASE}:${IMAGE_TAG}" +buildah tag "${IMAGE_BASE}:${IMAGE_TAG}" "${IMAGE_BASE}:latest" +buildah rm "$BUILDAH_CONTAINER_NAME" -# Here you can add the "podman push" command to send the mirror to your registry. -# Do not forget to disable layer compression otherwise the push & pull operations -# will be very slow! +echo "Build complete." diff --git a/sync.sh b/sync.sh new file mode 100755 index 0000000..e46488b --- /dev/null +++ b/sync.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -Eeuo pipefail + +# Create directory structure +echo "Creating directory structure..." +mkdir -p "${BUILDAH_ROOT}/var/www/centos/${CENTOS_VERSION}-stream" +mkdir -p "${BUILDAH_ROOT}/var/www/epel/${EPEL_VERSION}" + +# Start synchronization using rsync with the specified options +echo "Starting synchronization from ${RSYNC_MIRROR}..." +rsync ${RSYNC_OPTS} "${RSYNC_MIRROR}${CENTOS_PATH}/${CENTOS_VERSION}-stream/" "${BUILDAH_ROOT}/var/www/centos/${CENTOS_VERSION}-stream/" +rsync ${RSYNC_OPTS} "${RSYNC_MIRROR}${EPEL_PATH}/${EPEL_VERSION}/" "${BUILDAH_ROOT}/var/www/epel/${EPEL_VERSION}/" + +echo "Synchronization complete."