From ae7a4eb06e1cceeb2255f7b0dd6a58159c298294 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20Mass=C3=A9?= <nicolas.masse@itix.fr>
Date: Mon, 7 Jul 2025 17:56:56 +0200
Subject: [PATCH] wip

---
 base-image/jetpack-python/Containerfile       |  10 ++
 base-image/jetpack-python/build.sh            |  38 +++++
 base-image/jetpack/build.sh                   |  17 +-
 bootc/Containerfile                           |  52 +++---
 bootc/Containerfile.vanilla                   |  55 +++++++
 bootc/post/etc/git/.gitignore                 |   1 +
 bootc/post/etc/git/git-credentials.sample     |   1 +
 bootc/post/etc/gitconfig                      |   2 +
 .../system-connections/webcam.nmconnection    |  17 ++
 .../containers/systemd/app-edge-ai.container  |   7 +-
 .../root/etc/systemd/system/git-repo.service  |  14 ++
 tekton/README.md                              |  49 +++++-
 tekton/common/kustomization.yaml              |   3 +-
 tekton/common/task-buildah-bootc.yaml         |  90 ----------
 tekton/common/task-buildah.yaml               | 154 ++++++++++++++++++
 tekton/common/task-rclone.yaml                |  33 ++++
 tekton/pipeline.yaml                          |  80 +++++++--
 tekton/pipelinerun.yaml                       |  39 +++--
 18 files changed, 515 insertions(+), 147 deletions(-)
 create mode 100644 base-image/jetpack-python/Containerfile
 create mode 100755 base-image/jetpack-python/build.sh
 create mode 100644 bootc/Containerfile.vanilla
 create mode 100644 bootc/post/etc/git/.gitignore
 create mode 100644 bootc/post/etc/git/git-credentials.sample
 create mode 100644 bootc/post/etc/gitconfig
 create mode 100644 bootc/root/etc/NetworkManager/system-connections/webcam.nmconnection
 create mode 100644 bootc/root/etc/systemd/system/git-repo.service
 delete mode 100644 tekton/common/task-buildah-bootc.yaml
 create mode 100644 tekton/common/task-buildah.yaml
 create mode 100644 tekton/common/task-rclone.yaml

diff --git a/base-image/jetpack-python/Containerfile b/base-image/jetpack-python/Containerfile
new file mode 100644
index 0000000..50e6bfc
--- /dev/null
+++ b/base-image/jetpack-python/Containerfile
@@ -0,0 +1,10 @@
+FROM artifactory.services.studio.airbushelicopters.com/poc-edge-ai/jetpack-base-image:r36.4.0
+
+RUN <<EOF
+set -Eeu
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install -y python3-pip
+rm -rf /var/lib/apt/lists/*
+apt-get clean
+EOF
diff --git a/base-image/jetpack-python/build.sh b/base-image/jetpack-python/build.sh
new file mode 100755
index 0000000..641b39e
--- /dev/null
+++ b/base-image/jetpack-python/build.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+TARGET_IMAGE="quay.io/nmasse-redhat/jetpack-multiarch-python:r36.4.0"
+SOURCE_IMAGE="quay.io/nmasse-redhat/jetpack-multiarch:r36.4.0"
+SOURCE_REF=jetpack
+TARGET_REF=jetpack-python
+
+# Login to registries
+export REGISTRY_AUTH_FILE="$PWD/auth.json"
+if [ ! -f "$REGISTRY_AUTH_FILE" ]; then
+  echo "Logging in quay.io registry"
+  podman login quay.io
+  echo "Done"
+  read -p "Press enter to continue "
+fi
+
+podman rmi -i "$SOURCE_IMAGE"
+podman pull --platform linux/amd64 "$SOURCE_IMAGE"
+podman tag "$SOURCE_IMAGE" "localhost/$SOURCE_REF-x86_64"
+podman rmi -i "$SOURCE_IMAGE"
+podman pull --platform linux/arm64/v8 "$SOURCE_IMAGE"
+podman tag "$SOURCE_IMAGE" "localhost/$SOURCE_REF-aarch64"
+podman rmi -i "$SOURCE_IMAGE"
+
+buildah build --platform linux/amd64 -t localhost/$TARGET_REF-x86_64 --from "localhost/$SOURCE_REF-x86_64" .
+buildah build --platform linux/arm64/v8 -t localhost/$TARGET_REF-aarch64 --from "localhost/$SOURCE_REF-aarch64" .
+
+if podman manifest exists localhost/$TARGET_REF; then
+  podman manifest rm localhost/$TARGET_REF
+fi
+podman manifest create localhost/$TARGET_REF
+podman manifest add localhost/$TARGET_REF localhost/$TARGET_REF-x86_64
+podman manifest add localhost/$TARGET_REF localhost/$TARGET_REF-aarch64
+echo "pushing to $TARGET_IMAGE..."
+read -p "Press enter to continue "
+podman manifest push --all --format v2s2 localhost/$TARGET_REF "docker://$TARGET_IMAGE"
diff --git a/base-image/jetpack/build.sh b/base-image/jetpack/build.sh
index 3a4e74a..359652a 100755
--- a/base-image/jetpack/build.sh
+++ b/base-image/jetpack/build.sh
@@ -5,6 +5,7 @@ set -Eeuo pipefail
 ARM64_BASE_IMAGE="nvcr.io/nvidia/l4t-jetpack:r36.4.0"
 AMD64_BASE_IMAGE="nvcr.io/nvidia/base/ubuntu:22.04_20240212"
 TARGET_IMAGE="quay.io/nmasse-redhat/jetpack-multiarch:r36.4.0"
+NAME=jetpack
 
 # Login to registries
 export REGISTRY_AUTH_FILE="$PWD/auth.json"
@@ -19,16 +20,16 @@ fi
 
 # Fetch the ARM64 image from Nvidia
 podman pull --platform linux/arm64/v8 "$ARM64_BASE_IMAGE"
-podman tag "$ARM64_BASE_IMAGE" localhost/base-image-aarch64
+podman tag "$ARM64_BASE_IMAGE" localhost/$NAME-aarch64
 
 # Package a similar version for x86 without all the CUDA libraries
 podman pull --platform linux/amd64 "$AMD64_BASE_IMAGE"
-buildah build --platform linux/amd64 --from "$AMD64_BASE_IMAGE" -t localhost/base-image-x86_64 .
+buildah build --platform linux/amd64 --from "$AMD64_BASE_IMAGE" -t localhost/$NAME-x86_64 .
 
-if podman manifest exists localhost/base-image; then
-  podman manifest rm localhost/base-image
+if podman manifest exists localhost/$NAME; then
+  podman manifest rm localhost/$NAME
 fi
-podman manifest create localhost/base-image
-podman manifest add localhost/base-image localhost/base-image-x86_64
-podman manifest add localhost/base-image localhost/base-image-aarch64
-podman manifest push --all --format v2s2 localhost/base-image "docker://$TARGET_IMAGE"
+podman manifest create localhost/$NAME
+podman manifest add localhost/$NAME localhost/$NAME-x86_64
+podman manifest add localhost/$NAME localhost/$NAME-aarch64
+podman manifest push --all --format v2s2 localhost/$NAME "docker://$TARGET_IMAGE"
diff --git a/bootc/Containerfile b/bootc/Containerfile
index 9af1779..437144d 100644
--- a/bootc/Containerfile
+++ b/bootc/Containerfile
@@ -1,31 +1,37 @@
-FROM registry.redhat.io/rhel9/rhel-bootc:9.4
+FROM quay.io/redhat-et/rhel-bootc-tegra:base
 
 ARG ADMIN_USERNAME=demo \
     ADMIN_PASSWORD=redhat \
-    NVIDIA_KERNEL_VERSION=5.14.0-427.22.1.el9_4
+    ENABLE_DNF_CACHE=1 \
+    LOCAL_RPM_REPO=0
 
-RUN set -Eeuo pipefail ; \
-    if ! grep -qxF 'keepcache=1' /etc/dnf/dnf.conf; then \
-      sed -i.bak '/^\[main\]$/a keepcache=1' /etc/dnf/dnf.conf ; \
-    fi ; \
-    echo "Replacing current kernel with a version compatible with the kernel modules shipped by Nvidia" ; \
-    mkdir -p /tmp/rpms ; \
-    dnf download -y --destdir /tmp/rpms kernel{,-core,-modules,-modules-core}-$NVIDIA_KERNEL_VERSION ; \
-    rpm-ostree override replace /tmp/rpms/*.rpm ; \
-    rm -rf /tmp/rpms ; \
-    dnf config-manager --enable codeready-builder-for-rhel-9-$(arch)-rpms ; \
-    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm ; \
-    dnf install -y mkpasswd NetworkManager-wifi podman skopeo git mosquitto ; \
-    if [[ "$(arch)" == "aarch64" ]]; then \
-      echo "Installing the Nvidia stuff..." ; \
-      curl -sSfL -o /etc/yum.repos.d/nvidia-l4t.repo https://repo.download.nvidia.com/jetson/rhel-9.4/r36.3.1/nvidia-l4t.repo ; \
-      curl -sSfL -o /etc/yum.repos.d/nvidia-container-toolkit.repo https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo ; \
-      dnf config-manager --enable nvidia-container-toolkit-experimental ; \
-      dnf install -y nvidia-jetpack-kmod nvidia-jetpack-all nvidia-container-toolkit-base ; \
-    fi ; \
-    useradd -m -G wheel -p "$(echo -n "$ADMIN_PASSWORD" | mkpasswd -m bcrypt --stdin)" "$ADMIN_USERNAME"
+RUN <<EOF
+set -Eeuo pipefail
+
+if [[ "$ENABLE_DNF_CACHE" == "1" ]] && ! grep -qxF 'keepcache=1' /etc/dnf/dnf.conf; then
+  echo "Disabling dnf cache..."
+  sed -i.bak '/^\[main\]$/a keepcache=1' /etc/dnf/dnf.conf
+fi
+
+if [[ "$LOCAL_RPM_REPO" == "1" ]]; then
+  echo "Disabling Subscription Manager because we have no internet connection and no satelite..."
+  echo -e "[main]\nenabled=0" > /etc/dnf/plugins/subscription-manager.conf
+fi
+
+if [[ "$LOCAL_RPM_REPO" != "1" ]]; then
+  #dnf config-manager --enable codeready-builder-for-rhel-9-$(arch)-rpms
+  dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+fi
+
+dnf install -y mkpasswd NetworkManager-wifi podman skopeo git
+
+if [ -n "$ADMIN_USERNAME" ]; then
+  useradd -m -G wheel -p "$(echo -n "$ADMIN_PASSWORD" | mkpasswd -m bcrypt --stdin)" "$ADMIN_USERNAME"
+fi
+EOF
 
 ADD --chown=root:root root /
 
 RUN set -Eeuo pipefail ; \
-    systemctl enable nvidia-ctk-init.service
+    systemctl enable nvidia-ctk-init.service ; \
+    systemctl enable git-repo.service
diff --git a/bootc/Containerfile.vanilla b/bootc/Containerfile.vanilla
new file mode 100644
index 0000000..408d4b9
--- /dev/null
+++ b/bootc/Containerfile.vanilla
@@ -0,0 +1,55 @@
+FROM registry.redhat.io/rhel9/rhel-bootc:9.4
+
+ARG ADMIN_USERNAME=demo \
+    ADMIN_PASSWORD=redhat \
+    NVIDIA_KERNEL_VERSION=5.14.0-427.22.1.el9_4 \
+    ENABLE_DNF_CACHE=1 \
+    LOCAL_RPM_REPO=0
+
+RUN <<EOF
+set -Eeuo pipefail
+
+if [[ "$ENABLE_DNF_CACHE" == "1" ]] && ! grep -qxF 'keepcache=1' /etc/dnf/dnf.conf; then
+  echo "Disabling dnf cache..."
+  sed -i.bak '/^\[main\]$/a keepcache=1' /etc/dnf/dnf.conf
+fi
+
+if [[ "$LOCAL_RPM_REPO" == "1" ]]; then
+  echo "Disabling Subscription Manager because we have no internet connection and no satelite..."
+  echo -e "[main]\nenabled=0" > /etc/dnf/plugins/subscription-manager.conf
+fi
+
+if [ -n "$NVIDIA_KERNEL_VERSION" ]; then
+  echo "Replacing current kernel with a version compatible with the kernel modules shipped by Nvidia"
+  mkdir -p /tmp/rpms
+  dnf download -y --destdir /tmp/rpms kernel{,-core,-modules,-modules-core}-$NVIDIA_KERNEL_VERSION
+  rpm-ostree override replace /tmp/rpms/*.rpm
+  rm -rf /tmp/rpms
+fi
+
+if [[ "$LOCAL_RPM_REPO" != "1" ]]; then
+  dnf config-manager --enable codeready-builder-for-rhel-9-$(arch)-rpms
+  dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+fi
+
+dnf install -y mkpasswd NetworkManager-wifi podman skopeo git
+if [[ "$(arch)" == "aarch64" ]]; then
+  echo "Installing the Nvidia stuff..." ; \
+  if [[ "$LOCAL_RPM_REPO" != "1" ]]; then
+    curl -sSfL -o /etc/yum.repos.d/nvidia-l4t.repo https://repo.download.nvidia.com/jetson/rhel-9.4/r36.3.1/nvidia-l4t.repo
+    curl -sSfL -o /etc/yum.repos.d/nvidia-container-toolkit.repo https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo
+    dnf config-manager --enable nvidia-container-toolkit-experimental
+  fi
+  dnf install -y nvidia-jetpack-kmod nvidia-jetpack-all nvidia-container-toolkit-base
+fi
+
+if [ -n "$ADMIN_USERNAME" ]; then
+  useradd -m -G wheel -p "$(echo -n "$ADMIN_PASSWORD" | mkpasswd -m bcrypt --stdin)" "$ADMIN_USERNAME"
+fi
+EOF
+
+ADD --chown=root:root root /
+
+RUN set -Eeuo pipefail ; \
+    systemctl enable nvidia-ctk-init.service ; \
+    systemctl enable git-repo.service
diff --git a/bootc/post/etc/git/.gitignore b/bootc/post/etc/git/.gitignore
new file mode 100644
index 0000000..c7e5685
--- /dev/null
+++ b/bootc/post/etc/git/.gitignore
@@ -0,0 +1 @@
+git-credentials
diff --git a/bootc/post/etc/git/git-credentials.sample b/bootc/post/etc/git/git-credentials.sample
new file mode 100644
index 0000000..af473c5
--- /dev/null
+++ b/bootc/post/etc/git/git-credentials.sample
@@ -0,0 +1 @@
+https://user:REDACTED@github.com
diff --git a/bootc/post/etc/gitconfig b/bootc/post/etc/gitconfig
new file mode 100644
index 0000000..0597c37
--- /dev/null
+++ b/bootc/post/etc/gitconfig
@@ -0,0 +1,2 @@
+[credential]
+  helper=store --file /etc/git/git-credentials
diff --git a/bootc/root/etc/NetworkManager/system-connections/webcam.nmconnection b/bootc/root/etc/NetworkManager/system-connections/webcam.nmconnection
new file mode 100644
index 0000000..ccfe0ef
--- /dev/null
+++ b/bootc/root/etc/NetworkManager/system-connections/webcam.nmconnection
@@ -0,0 +1,17 @@
+[connection]
+id=webcam
+uuid=a97f051e-2924-4327-9838-80f85f9bcee8
+type=ethernet
+interface-name=eth0
+
+[ethernet]
+
+[ipv4]
+address1=172.168.2.2/24
+method=manual
+
+[ipv6]
+addr-gen-mode=default
+method=disabled
+
+[proxy]
diff --git a/bootc/root/etc/containers/systemd/app-edge-ai.container b/bootc/root/etc/containers/systemd/app-edge-ai.container
index d174178..35c143f 100644
--- a/bootc/root/etc/containers/systemd/app-edge-ai.container
+++ b/bootc/root/etc/containers/systemd/app-edge-ai.container
@@ -1,9 +1,11 @@
 [Unit]
 Description=AI application deployed at the Edge
-After=local-fs.target
+After=local-fs.target nvidia-ctk-init.service
+Wants=nvidia-ctk-init.service
 
 [Service]
 ExecStartPre=-podman network create --ignore app
+Environment=REGISTRY_AUTH_FILE=/etc/ostree/auth.json
 
 [Container]
 ContainerName=app-edge-ai
@@ -11,8 +13,7 @@ Image=quay.io/nmasse-redhat/app-edge-ai:latest
 Network=app
 
 # Needed for Nvidia GPU Acceleration
-PodmanArgs=--runtime /usr/bin/nvidia-container-runtime
-GroupAdd=keep-groups
+PodmanArgs=--runtime /usr/bin/nvidia-container-runtime --group-add=keep-groups
 SecurityLabelDisable=true
 Environment=NVIDIA_VISIBLE_DEVICES=nvidia.com/gpu=all
 
diff --git a/bootc/root/etc/systemd/system/git-repo.service b/bootc/root/etc/systemd/system/git-repo.service
new file mode 100644
index 0000000..cc4f8a1
--- /dev/null
+++ b/bootc/root/etc/systemd/system/git-repo.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Sync the git repo
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+User=demo
+Environment=GIT_REPO=https://github.com/nmasse-itix/bootc-edge-ai.git
+ExecStart=/bin/sh -c 'if [ -d "$HOME/bootc-edge-ai" ]; then cd "$HOME/bootc-edge-ai" && git pull ; else git clone "$GIT_REPO" -b main "$HOME/bootc-edge-ai" ; fi'
+
+[Install]
+WantedBy=multi-user.target
diff --git a/tekton/README.md b/tekton/README.md
index e567bde..32c4dcb 100644
--- a/tekton/README.md
+++ b/tekton/README.md
@@ -13,6 +13,12 @@ oc apply -k common/
 oc apply -f pipeline.yaml
 ```
 
+## Authentication to Pypi
+
+```sh
+oc create secret generic pypi-mirror '--from-literal=PYPI_MIRROR_URL=https://login:password@artifactory-host/artifactory/api/pypi/pypi-virtual/simple'
+```
+
 ## Authentication to the registries
 
 ```sh
@@ -32,6 +38,41 @@ data:
 EOF
 ```
 
+```sh
+oc create configmap registries-conf --from-file=/etc/containers/registries.conf
+```
+
+## Authentication to GitHub
+
+```sh
+cat > gitconfig <<EOF
+[credential]
+  helper=store
+EOF
+oc create secret generic github-authentication --from-literal=.git-credentials=https://user:password@github.com --from-file=.gitconfig=gitconfig
+```
+
+## Rclone config for AWS S3
+
+**rclone.conf**:
+
+```ini
+[aws]
+type = s3
+provider = AWS
+access_key_id = REDACTED
+secret_access_key = REDACTED
+region = eu-west-3
+```
+
+Note: in **rclone.conf**, set **endpoint** to the hostname of your S3 gateway when on-premise.
+
+Create the secret:
+
+```sh
+oc create secret generic rclone-config --from-file=rclone.conf
+```
+
 ## Initialize data inside the PVC
 
 Create a Pod that uses the two previously created PVC :
@@ -54,6 +95,8 @@ spec:
       mountPath: /caches
     - name: bootc-entitlements
       mountPath: /entitlements
+    - name: bootc-rpms
+      mountPath: /rpms
   volumes:
   - name: bootc-caches
     persistentVolumeClaim:
@@ -61,9 +104,12 @@ spec:
   - name: bootc-entitlements
     persistentVolumeClaim:
       claimName: bootc-entitlements
+  - name: bootc-rpms
+    persistentVolumeClaim:
+      claimName: bootc-rpms
 ```
 
-Then copy all the data to `/caches` and `/entitlements`.
+Then copy all the data to `/caches`, `/rpms` and `/entitlements`.
 
 ```sh
 mkdir -p entitlements
@@ -71,6 +117,7 @@ cp etc-x86_64.tar entitlements/x86_64.tar
 cp etc-aarch64.tar entitlements/aarch64.tar
 oc rsync entitlements rsync:/
 oc rsh rsync mkdir -p /caches/{x86_64,aarch64}/{rpm-ostree,dnf}
+tar -c -C /path/to/rpms | oc rsh rsync tar -x -C /rpms
 ```
 
 You can leave the Pod running or delete it with :
diff --git a/tekton/common/kustomization.yaml b/tekton/common/kustomization.yaml
index 2359538..d5d0386 100644
--- a/tekton/common/kustomization.yaml
+++ b/tekton/common/kustomization.yaml
@@ -1,6 +1,7 @@
 resources:
 - serviceaccount-buildbot.yaml
-- task-buildah-bootc.yaml
+- task-buildah.yaml
 - task-git-clone.yaml
+- task-rclone.yaml
 - daemonset-qemu.yaml
 - storage.yaml
diff --git a/tekton/common/task-buildah-bootc.yaml b/tekton/common/task-buildah-bootc.yaml
deleted file mode 100644
index 0b1613f..0000000
--- a/tekton/common/task-buildah-bootc.yaml
+++ /dev/null
@@ -1,90 +0,0 @@
-apiVersion: tekton.dev/v1beta1
-kind: Task
-metadata:
-  name: buildah-bootc
-spec:
-  params:
-  - name: context-dir
-    type: string
-    default: .
-  - name: containerfile-path
-    type: string
-    default: Containerfile
-  - name: image-name
-    type: string
-  workspaces:
-  - name: source-workspace
-    description: Workspace containing source code
-  - name: caches
-    description: RW storage to cache build artefacts
-    mountPath: /caches
-  - name: entitlements
-    description: RW storage for RHEL entitlements
-    mountPath: /entitlements
-  - description: An optional workspace that allows providing a .docker/config.json file for Buildah to access the container registry. The file should be placed at the root of the Workspace with name config.json or .dockerconfigjson.
-    name: dockerconfig
-    optional: true
-    mountPath: /auth
-  volumes:
-  - name: container-storage
-    emptyDir: {}
-  steps:
-  - name: build
-    image: registry.redhat.io/rhel9/buildah:9.6
-    env:
-    - name: STORAGE_DRIVER
-      value: overlay
-    - name: RHEL_IMAGE
-      value: registry.redhat.io/rhel9/rhel-bootc
-    - name: RHEL_VERSION
-      value: "9.4"
-    - name: TARGET_IMAGE
-      value: "$(params.image-name)"
-    - name: REGISTRY_AUTH_FILE
-      value: /auth/.dockerconfigjson
-    script: |
-      #!/bin/bash
-      set -Eeuo pipefail
-
-      # All architectures to build for
-      declare -a ARCHITECTURES=("x86_64" "aarch64")
-
-      # Build images
-      declare -A PODMAN_ARCH_OPTS=(["aarch64"]="--platform linux/arm64/v8" ["x86_64"]="--platform linux/amd64")
-      for arch in "${ARCHITECTURES[@]}"; do
-        buildah pull ${PODMAN_ARCH_OPTS[$arch]} $RHEL_IMAGE:$RHEL_VERSION
-        buildah tag $RHEL_IMAGE:$RHEL_VERSION $RHEL_IMAGE-$arch:$RHEL_VERSION
-        buildah rmi $RHEL_IMAGE:$RHEL_VERSION
-
-        echo "Building image for $arch..."
-        rm -rf /tmp/entitlements
-        mkdir -p /tmp/entitlements
-        tar -xf /entitlements/$arch.tar -C /tmp/entitlements 
-        buildah bud ${PODMAN_ARCH_OPTS[$arch]} --no-cache --from "$RHEL_IMAGE-$arch:$RHEL_VERSION" \
-                    -v /tmp/entitlements/etc/pki/entitlement/:/etc/pki/entitlement:z -v /tmp/entitlements/etc/rhsm:/etc/rhsm:z \
-                    -v /tmp/entitlements/etc/pki/entitlement/:/run/secrets/etc-pki-entitlement:z -v /tmp/entitlements/etc/rhsm:/run/secrets/rhsm:z \
-                    -v /tmp/entitlements/etc/yum.repos.d:/etc/yum.repos.d:z -v /caches/$arch/dnf:/var/cache/dnf:z \
-                    -v /caches/$arch/rpm-ostree:/var/cache/rpm-ostree:z \
-                    -t localhost/image-$arch \
-                    -f $(workspaces.source-workspace.path)/$(params.containerfile-path) \
-                    $(workspaces.source-workspace.path)/$(params.context-dir)
-      done
-
-      # Push Manifest
-      echo "Pushing to $TARGET_IMAGE..."
-      buildah manifest create localhost/image
-      for arch in "${ARCHITECTURES[@]}"; do
-        buildah manifest add localhost/image localhost/image-$arch
-      done
-      buildah manifest push localhost/image docker://$TARGET_IMAGE
-    securityContext:
-      ## Buildah needs privileges to use the "overlay" Storage Driver.
-      privileged: true
-
-      ## The "vfs" Storage Driver however requires less privileges.
-      #capabilities:
-      #  add:
-      #    - SETFCAP
-    volumeMounts:
-    - name: container-storage
-      mountPath: /var/lib/containers
diff --git a/tekton/common/task-buildah.yaml b/tekton/common/task-buildah.yaml
new file mode 100644
index 0000000..f6df322
--- /dev/null
+++ b/tekton/common/task-buildah.yaml
@@ -0,0 +1,154 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: buildah
+spec:
+  params:
+  - name: context-dir
+    type: string
+    default: .
+  - name: containerfile-path
+    type: string
+    default: Containerfile
+  - name: image-name
+    type: string
+  - name: buildah-image
+    type: string
+    default: registry.redhat.io/rhel9/buildah:latest
+  - name: build-architectures
+    type: array
+    default:
+    - x86_64
+    - aarch64
+  - name: pypi-mirror-url
+    type: string
+    optional: true
+  workspaces:
+  - name: source-workspace
+    description: Workspace containing source code
+  - name: caches
+    description: RW storage to cache build artefacts
+    mountPath: /caches
+    optional: true
+  - name: entitlements
+    description: RW storage for RHEL entitlements
+    mountPath: /entitlements
+    optional: true
+  - description: An optional workspace that allows providing a .docker/config.json file for Buildah to access the container registry. The file should be placed at the root of the Workspace with name config.json or .dockerconfigjson.
+    name: dockerconfig
+    optional: true
+    mountPath: /auth
+  - name: registries-conf
+    optional: true
+    mountPath: /registries
+  volumes:
+  - name: container-storage
+    emptyDir: {}
+  steps:
+  - name: build
+    image: $(params.buildah-image)
+    env:
+    - name: STORAGE_DRIVER
+      value: overlay
+    - name: SCRIPT_DEBUG
+      value: "false"
+    - name: TARGET_IMAGE
+      value: "$(params.image-name)"
+    - name: PYPI_MIRROR_URL
+      value: "$(params.pypi-mirror-url)"
+    args:
+    - "$(params.build-architectures)"
+    script: |
+      #!/bin/bash
+      set -Eeuo pipefail
+
+      # If debug is enabled, print out command that are executed
+      if [[ "${SCRIPT_DEBUG:-false}" == "true" ]]; then
+        set -x
+      fi
+
+      # Print versions of the program we use
+      echo "=== Environment ==="
+      echo "---> Buildah"
+      buildah version
+      echo
+      echo "---> bash"
+      bash --version
+      echo
+      echo "---> OS"
+      cat /etc/redhat-release
+      echo
+
+      # Check what is available and set env variables
+      if [ -f /registries/registries.conf ]; then
+        export CONTAINERS_REGISTRIES_CONF=/registries/registries.conf
+      fi
+      if [ -f /auth/.dockerconfigjson ]; then
+        export REGISTRY_AUTH_FILE=/auth/.dockerconfigjson
+      fi
+
+      # Extract the parent image name
+      FROM="$(sed -r 's/^FROM\s+(.*)\s*/\1/;t;d' "$(workspaces.source-workspace.path)/$(params.containerfile-path)")"
+      echo "Detected $FROM as parent image."
+
+      # Build images
+      declare -A PODMAN_ARCH_OPTS=(["aarch64"]="--platform linux/arm64/v8" ["x86_64"]="--platform linux/amd64")
+      buildah manifest create localhost/image
+      for arch; do
+        declare -a PODMAN_OPTS=( )
+        if [ -n "${PYPI_MIRROR_URL:}" ]; then
+          PODMAN_OPTS+=( "--build-arg" "PYPI_MIRROR_URL=${PYPI_MIRROR_URL}" )
+        fi
+        if [ -f "/entitlements/$arch.tar" ]; then
+          echo "Using RHEL entitlements..."
+          rm -rf /tmp/entitlements
+          mkdir -p /tmp/entitlements
+          tar -xf /entitlements/$arch.tar -C /tmp/entitlements
+          PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/pki/entitlement/:/etc/pki/entitlement:z" )
+          PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/rhsm:/etc/rhsm:z" )
+          PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/pki/entitlement/:/run/secrets/etc-pki-entitlement:z" )
+          PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/rhsm:/run/secrets/rhsm:z" )
+          PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/rhsm:/run/secrets/rhsm:z" )
+          PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/yum.repos.d:/etc/yum.repos.d:z" )
+        fi
+        if [ -d "/caches/$arch/" ]; then
+          echo "Enabling cache..."
+          PODMAN_OPTS+=( "-v" "/caches/$arch/dnf:/var/cache/dnf:z" )
+          PODMAN_OPTS+=( "-v" "/caches/$arch/rpm-ostree:/var/cache/rpm-ostree:z" )
+        fi
+        if [ -d "/rpms/$arch/" ]; then
+          echo "Enabling RPM repositories..."
+          mkdir -p /tmp/rpms
+          cat > /tmp/rpms/local-rpms.repo <<EOF
+      [local-rpms]
+      name=Local RPMs Repository
+      baseurl=file:///opt/local-repo
+      enabled=1
+      gpgcheck=0
+      EOF
+          PODMAN_OPTS+=( "-v" "/tmp/rpms:/etc/yum.repos.d:z" )
+          PODMAN_OPTS+=( "-v" "/rpms/$arch:/opt/local-repo:z" )
+        fi
+        PODMAN_OPTS+=( "-f" "$(workspaces.source-workspace.path)/$(params.containerfile-path)" )
+        PODMAN_OPTS+=( "--no-cache" )
+
+        echo "Building image for $arch..."
+        ( set -x ; buildah bud ${PODMAN_ARCH_OPTS[$arch]} "${PODMAN_OPTS[@]}" "-t" "localhost/image-$arch" $(workspaces.source-workspace.path)/$(params.context-dir) )
+        buildah manifest add localhost/image localhost/image-$arch
+        buildah rmi "$FROM"
+      done
+
+      # Push Manifest
+      echo "Pushing to $TARGET_IMAGE..."
+      buildah manifest push localhost/image docker://$TARGET_IMAGE
+    securityContext:
+      ## Buildah needs privileges to use the "overlay" Storage Driver.
+      privileged: true
+
+      ## The "vfs" Storage Driver however requires less privileges.
+      #capabilities:
+      #  add:
+      #    - SETFCAP
+    volumeMounts:
+    - name: container-storage
+      mountPath: /var/lib/containers
diff --git a/tekton/common/task-rclone.yaml b/tekton/common/task-rclone.yaml
new file mode 100644
index 0000000..0c0ac78
--- /dev/null
+++ b/tekton/common/task-rclone.yaml
@@ -0,0 +1,33 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: rclone
+spec:
+  params:
+  - name: rclone-image
+    type: string
+    default: docker.io/rclone/rclone:latest
+  - name: rclone-src
+    type: array
+    default: []
+  - name: rclone-dest
+    type: string
+    default: "."
+  workspaces:
+  - name: source-workspace
+    description: Workspace containing source code
+  - name: rclone-config
+    optional: true
+    mountPath: /etc/rclone
+  steps:
+  - name: copy
+    image: $(params.rclone-image)
+    args:
+    - "$(params.rclone-src[*])"
+    script: |
+      #!/bin/sh
+      set -eu
+      cd $(workspaces.source-workspace.path)
+      for source; do
+        rclone --config=/etc/rclone/rclone.conf copy --progress "$source" "$(params.rclone-dest)"
+      done
diff --git a/tekton/pipeline.yaml b/tekton/pipeline.yaml
index 22deaf7..f6c90b2 100644
--- a/tekton/pipeline.yaml
+++ b/tekton/pipeline.yaml
@@ -7,8 +7,17 @@ spec:
   workspaces:
   - name: source-workspace
   - name: registry-token
+    optional: true
   - name: caches
+    optional: true
   - name: entitlements
+    optional: true
+  - name: git-auth
+    optional: true
+  - name: registries-conf
+    optional: true
+  - name: rclone-config
+    optional: true
 
   params:
   - name: git-url
@@ -16,14 +25,19 @@ spec:
   - name: git-revision
     type: string
     default: main
-  - name: image-name
+  - name: bootc-image-name
     type: string
-  - name: context-dir
+  - name: app-image-name
     type: string
-    default: "."
-  - name: containerfile-path
+  - name: rclone-model-src
     type: string
-    default: "Containerfile"
+  - name: rclone-model-dest
+    type: string
+  - name: build-architectures
+    type: array
+  - name: pypi-mirror-url
+    type: string
+    optional: true
 
   tasks:
 
@@ -39,18 +53,62 @@ spec:
     workspaces:
     - name: output
       workspace: source-workspace
-  
-  - name: buildah-bootc
+    - name: basic-auth
+      workspace: git-auth
+
+  - name: fetch-model
     runAfter: ["clone-repo"]
     taskRef:
-      name: buildah-bootc
+      kind: Task
+      name: rclone
+    params:
+    - name: rclone-src
+      value:
+      - $(params.rclone-model-src)
+    - name: rclone-dest
+      value: $(params.rclone-model-dest)
+    workspaces:
+    - name: source-workspace
+      workspace: source-workspace
+    - name: rclone-config
+      workspace: rclone-config
+
+  - name: buildah-app
+    runAfter: ["fetch-model"]
+    taskRef:
+      name: buildah
+    params:
+    - name: image-name
+      value: $(params.app-image-name)
+    - name: context-dir
+      value: app
+    - name: containerfile-path
+      value: app/Containerfile
+    - name: build-architectures
+      value:
+      - $(params.build-architectures)
+    - name: pypi-mirror-url
+      value: $(params.pypi-mirror-url)
+    workspaces:
+    - name: source-workspace
+      workspace: source-workspace
+    - name: dockerconfig
+      workspace: registry-token
+
+  - name: buildah-bootc
+    runAfter: ["buildah-app"]
+    taskRef:
+      name: buildah
     params:
     - name: image-name
-      value: $(params.image-name)
+      value: $(params.bootc-image-name)
     - name: context-dir
-      value: $(params.context-dir)
+      value: bootc
     - name: containerfile-path
-      value: $(params.containerfile-path)
+      value: bootc/Containerfile
+    - name: build-architectures
+      value:
+      - $(params.build-architectures)
     workspaces:
     - name: source-workspace
       workspace: source-workspace
diff --git a/tekton/pipelinerun.yaml b/tekton/pipelinerun.yaml
index cec7677..3af1cf6 100644
--- a/tekton/pipelinerun.yaml
+++ b/tekton/pipelinerun.yaml
@@ -10,19 +10,29 @@ spec:
     value: https://github.com/nmasse-itix/bootc-edge-ai.git
   - name: git-revision
     value: main
-  - name: image-name
+  - name: bootc-image-name
     value: quay.io/nmasse-redhat/bootc-edge-ai
-  - name: context-dir
-    value: bootc
-  - name: containerfile-path
-    value: "bootc/Containerfile"
+  - name: app-image-name
+    value: quay.io/nmasse-redhat/app-edge-ai
+  - name: rclone-model-src
+    value: aws:nmasse-bootc-edge-ai/model.onnx
+  - name: rclone-model-dest
+    value: app/model-s3.onnx
+  - name: build-architectures
+    value:
+    - aarch64
+  # - name: pypi-mirror-url
+  #   value: http://...
   workspaces:
-  - name: caches
+  # - name: caches
+  #   persistentVolumeClaim:
+  #     claimName: bootc-caches
+  # - name: entitlements
+  #   persistentVolumeClaim:
+  #     claimName: bootc-entitlements
+  - name: rpms
     persistentVolumeClaim:
-      claimName: bootc-caches
-  - name: entitlements
-    persistentVolumeClaim:
-      claimName: bootc-entitlements
+      claimName: bootc-rpms
   - name: source-workspace
     volumeClaimTemplate:
       spec:
@@ -35,5 +45,14 @@ spec:
   - name: registry-token
     secret:
       secretName: registry-authentication
+  - name: rclone-config
+    secret:
+      secretName: rclone-config
+  #- name: git-token
+  #  secret:
+  #    secretName: github-authentication
+  #- name: registries-conf
+  #  configMap:
+  #    name: registries-conf
   taskRunTemplate:
     serviceAccountName: buildbot