Nicolas Massé 5 months ago
parent
commit
ae7a4eb06e
  1. 10
      base-image/jetpack-python/Containerfile
  2. 38
      base-image/jetpack-python/build.sh
  3. 17
      base-image/jetpack/build.sh
  4. 50
      bootc/Containerfile
  5. 55
      bootc/Containerfile.vanilla
  6. 1
      bootc/post/etc/git/.gitignore
  7. 1
      bootc/post/etc/git/git-credentials.sample
  8. 2
      bootc/post/etc/gitconfig
  9. 17
      bootc/root/etc/NetworkManager/system-connections/webcam.nmconnection
  10. 7
      bootc/root/etc/containers/systemd/app-edge-ai.container
  11. 14
      bootc/root/etc/systemd/system/git-repo.service
  12. 49
      tekton/README.md
  13. 3
      tekton/common/kustomization.yaml
  14. 90
      tekton/common/task-buildah-bootc.yaml
  15. 154
      tekton/common/task-buildah.yaml
  16. 33
      tekton/common/task-rclone.yaml
  17. 78
      tekton/pipeline.yaml
  18. 39
      tekton/pipelinerun.yaml

10
base-image/jetpack-python/Containerfile

@ -0,0 +1,10 @@
FROM artifactory.services.studio.airbushelicopters.com/poc-edge-ai/jetpack-base-image:r36.4.0
RUN <<EOF
set -Eeu
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y python3-pip
rm -rf /var/lib/apt/lists/*
apt-get clean
EOF

38
base-image/jetpack-python/build.sh

@ -0,0 +1,38 @@
#!/bin/bash
set -Eeuo pipefail
TARGET_IMAGE="quay.io/nmasse-redhat/jetpack-multiarch-python:r36.4.0"
SOURCE_IMAGE="quay.io/nmasse-redhat/jetpack-multiarch:r36.4.0"
SOURCE_REF=jetpack
TARGET_REF=jetpack-python
# Login to registries
export REGISTRY_AUTH_FILE="$PWD/auth.json"
if [ ! -f "$REGISTRY_AUTH_FILE" ]; then
echo "Logging in quay.io registry"
podman login quay.io
echo "Done"
read -p "Press enter to continue "
fi
podman rmi -i "$SOURCE_IMAGE"
podman pull --platform linux/amd64 "$SOURCE_IMAGE"
podman tag "$SOURCE_IMAGE" "localhost/$SOURCE_REF-x86_64"
podman rmi -i "$SOURCE_IMAGE"
podman pull --platform linux/arm64/v8 "$SOURCE_IMAGE"
podman tag "$SOURCE_IMAGE" "localhost/$SOURCE_REF-aarch64"
podman rmi -i "$SOURCE_IMAGE"
buildah build --platform linux/amd64 -t localhost/$TARGET_REF-x86_64 --from "localhost/$SOURCE_REF-x86_64" .
buildah build --platform linux/arm64/v8 -t localhost/$TARGET_REF-aarch64 --from "localhost/$SOURCE_REF-aarch64" .
if podman manifest exists localhost/$TARGET_REF; then
podman manifest rm localhost/$TARGET_REF
fi
podman manifest create localhost/$TARGET_REF
podman manifest add localhost/$TARGET_REF localhost/$TARGET_REF-x86_64
podman manifest add localhost/$TARGET_REF localhost/$TARGET_REF-aarch64
echo "pushing to $TARGET_IMAGE..."
read -p "Press enter to continue "
podman manifest push --all --format v2s2 localhost/$TARGET_REF "docker://$TARGET_IMAGE"

17
base-image/jetpack/build.sh

@ -5,6 +5,7 @@ set -Eeuo pipefail
ARM64_BASE_IMAGE="nvcr.io/nvidia/l4t-jetpack:r36.4.0"
AMD64_BASE_IMAGE="nvcr.io/nvidia/base/ubuntu:22.04_20240212"
TARGET_IMAGE="quay.io/nmasse-redhat/jetpack-multiarch:r36.4.0"
NAME=jetpack
# Login to registries
export REGISTRY_AUTH_FILE="$PWD/auth.json"
@ -19,16 +20,16 @@ fi
# Fetch the ARM64 image from Nvidia
podman pull --platform linux/arm64/v8 "$ARM64_BASE_IMAGE"
podman tag "$ARM64_BASE_IMAGE" localhost/base-image-aarch64
podman tag "$ARM64_BASE_IMAGE" localhost/$NAME-aarch64
# Package a similar version for x86 without all the CUDA libraries
podman pull --platform linux/amd64 "$AMD64_BASE_IMAGE"
buildah build --platform linux/amd64 --from "$AMD64_BASE_IMAGE" -t localhost/base-image-x86_64 .
buildah build --platform linux/amd64 --from "$AMD64_BASE_IMAGE" -t localhost/$NAME-x86_64 .
if podman manifest exists localhost/base-image; then
podman manifest rm localhost/base-image
if podman manifest exists localhost/$NAME; then
podman manifest rm localhost/$NAME
fi
podman manifest create localhost/base-image
podman manifest add localhost/base-image localhost/base-image-x86_64
podman manifest add localhost/base-image localhost/base-image-aarch64
podman manifest push --all --format v2s2 localhost/base-image "docker://$TARGET_IMAGE"
podman manifest create localhost/$NAME
podman manifest add localhost/$NAME localhost/$NAME-x86_64
podman manifest add localhost/$NAME localhost/$NAME-aarch64
podman manifest push --all --format v2s2 localhost/$NAME "docker://$TARGET_IMAGE"

50
bootc/Containerfile

@ -1,31 +1,37 @@
FROM registry.redhat.io/rhel9/rhel-bootc:9.4
FROM quay.io/redhat-et/rhel-bootc-tegra:base
ARG ADMIN_USERNAME=demo \
ADMIN_PASSWORD=redhat \
NVIDIA_KERNEL_VERSION=5.14.0-427.22.1.el9_4
ENABLE_DNF_CACHE=1 \
LOCAL_RPM_REPO=0
RUN set -Eeuo pipefail ; \
if ! grep -qxF 'keepcache=1' /etc/dnf/dnf.conf; then \
sed -i.bak '/^\[main\]$/a keepcache=1' /etc/dnf/dnf.conf ; \
fi ; \
echo "Replacing current kernel with a version compatible with the kernel modules shipped by Nvidia" ; \
mkdir -p /tmp/rpms ; \
dnf download -y --destdir /tmp/rpms kernel{,-core,-modules,-modules-core}-$NVIDIA_KERNEL_VERSION ; \
rpm-ostree override replace /tmp/rpms/*.rpm ; \
rm -rf /tmp/rpms ; \
dnf config-manager --enable codeready-builder-for-rhel-9-$(arch)-rpms ; \
dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm ; \
dnf install -y mkpasswd NetworkManager-wifi podman skopeo git mosquitto ; \
if [[ "$(arch)" == "aarch64" ]]; then \
echo "Installing the Nvidia stuff..." ; \
curl -sSfL -o /etc/yum.repos.d/nvidia-l4t.repo https://repo.download.nvidia.com/jetson/rhel-9.4/r36.3.1/nvidia-l4t.repo ; \
curl -sSfL -o /etc/yum.repos.d/nvidia-container-toolkit.repo https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo ; \
dnf config-manager --enable nvidia-container-toolkit-experimental ; \
dnf install -y nvidia-jetpack-kmod nvidia-jetpack-all nvidia-container-toolkit-base ; \
fi ; \
RUN <<EOF
set -Eeuo pipefail
if [[ "$ENABLE_DNF_CACHE" == "1" ]] && ! grep -qxF 'keepcache=1' /etc/dnf/dnf.conf; then
echo "Disabling dnf cache..."
sed -i.bak '/^\[main\]$/a keepcache=1' /etc/dnf/dnf.conf
fi
if [[ "$LOCAL_RPM_REPO" == "1" ]]; then
echo "Disabling Subscription Manager because we have no internet connection and no satelite..."
echo -e "[main]\nenabled=0" > /etc/dnf/plugins/subscription-manager.conf
fi
if [[ "$LOCAL_RPM_REPO" != "1" ]]; then
#dnf config-manager --enable codeready-builder-for-rhel-9-$(arch)-rpms
dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
fi
dnf install -y mkpasswd NetworkManager-wifi podman skopeo git
if [ -n "$ADMIN_USERNAME" ]; then
useradd -m -G wheel -p "$(echo -n "$ADMIN_PASSWORD" | mkpasswd -m bcrypt --stdin)" "$ADMIN_USERNAME"
fi
EOF
ADD --chown=root:root root /
RUN set -Eeuo pipefail ; \
systemctl enable nvidia-ctk-init.service
systemctl enable nvidia-ctk-init.service ; \
systemctl enable git-repo.service

55
bootc/Containerfile.vanilla

@ -0,0 +1,55 @@
FROM registry.redhat.io/rhel9/rhel-bootc:9.4
ARG ADMIN_USERNAME=demo \
ADMIN_PASSWORD=redhat \
NVIDIA_KERNEL_VERSION=5.14.0-427.22.1.el9_4 \
ENABLE_DNF_CACHE=1 \
LOCAL_RPM_REPO=0
RUN <<EOF
set -Eeuo pipefail
if [[ "$ENABLE_DNF_CACHE" == "1" ]] && ! grep -qxF 'keepcache=1' /etc/dnf/dnf.conf; then
echo "Disabling dnf cache..."
sed -i.bak '/^\[main\]$/a keepcache=1' /etc/dnf/dnf.conf
fi
if [[ "$LOCAL_RPM_REPO" == "1" ]]; then
echo "Disabling Subscription Manager because we have no internet connection and no satelite..."
echo -e "[main]\nenabled=0" > /etc/dnf/plugins/subscription-manager.conf
fi
if [ -n "$NVIDIA_KERNEL_VERSION" ]; then
echo "Replacing current kernel with a version compatible with the kernel modules shipped by Nvidia"
mkdir -p /tmp/rpms
dnf download -y --destdir /tmp/rpms kernel{,-core,-modules,-modules-core}-$NVIDIA_KERNEL_VERSION
rpm-ostree override replace /tmp/rpms/*.rpm
rm -rf /tmp/rpms
fi
if [[ "$LOCAL_RPM_REPO" != "1" ]]; then
dnf config-manager --enable codeready-builder-for-rhel-9-$(arch)-rpms
dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
fi
dnf install -y mkpasswd NetworkManager-wifi podman skopeo git
if [[ "$(arch)" == "aarch64" ]]; then
echo "Installing the Nvidia stuff..." ; \
if [[ "$LOCAL_RPM_REPO" != "1" ]]; then
curl -sSfL -o /etc/yum.repos.d/nvidia-l4t.repo https://repo.download.nvidia.com/jetson/rhel-9.4/r36.3.1/nvidia-l4t.repo
curl -sSfL -o /etc/yum.repos.d/nvidia-container-toolkit.repo https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo
dnf config-manager --enable nvidia-container-toolkit-experimental
fi
dnf install -y nvidia-jetpack-kmod nvidia-jetpack-all nvidia-container-toolkit-base
fi
if [ -n "$ADMIN_USERNAME" ]; then
useradd -m -G wheel -p "$(echo -n "$ADMIN_PASSWORD" | mkpasswd -m bcrypt --stdin)" "$ADMIN_USERNAME"
fi
EOF
ADD --chown=root:root root /
RUN set -Eeuo pipefail ; \
systemctl enable nvidia-ctk-init.service ; \
systemctl enable git-repo.service

1
bootc/post/etc/git/.gitignore

@ -0,0 +1 @@
git-credentials

1
bootc/post/etc/git/git-credentials.sample

@ -0,0 +1 @@
https://user:REDACTED@github.com

2
bootc/post/etc/gitconfig

@ -0,0 +1,2 @@
[credential]
helper=store --file /etc/git/git-credentials

17
bootc/root/etc/NetworkManager/system-connections/webcam.nmconnection

@ -0,0 +1,17 @@
[connection]
id=webcam
uuid=a97f051e-2924-4327-9838-80f85f9bcee8
type=ethernet
interface-name=eth0
[ethernet]
[ipv4]
address1=172.168.2.2/24
method=manual
[ipv6]
addr-gen-mode=default
method=disabled
[proxy]

7
bootc/root/etc/containers/systemd/app-edge-ai.container

@ -1,9 +1,11 @@
[Unit]
Description=AI application deployed at the Edge
After=local-fs.target
After=local-fs.target nvidia-ctk-init.service
Wants=nvidia-ctk-init.service
[Service]
ExecStartPre=-podman network create --ignore app
Environment=REGISTRY_AUTH_FILE=/etc/ostree/auth.json
[Container]
ContainerName=app-edge-ai
@ -11,8 +13,7 @@ Image=quay.io/nmasse-redhat/app-edge-ai:latest
Network=app
# Needed for Nvidia GPU Acceleration
PodmanArgs=--runtime /usr/bin/nvidia-container-runtime
GroupAdd=keep-groups
PodmanArgs=--runtime /usr/bin/nvidia-container-runtime --group-add=keep-groups
SecurityLabelDisable=true
Environment=NVIDIA_VISIBLE_DEVICES=nvidia.com/gpu=all

14
bootc/root/etc/systemd/system/git-repo.service

@ -0,0 +1,14 @@
[Unit]
Description=Sync the git repo
Wants=network-online.target
After=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
User=demo
Environment=GIT_REPO=https://github.com/nmasse-itix/bootc-edge-ai.git
ExecStart=/bin/sh -c 'if [ -d "$HOME/bootc-edge-ai" ]; then cd "$HOME/bootc-edge-ai" && git pull ; else git clone "$GIT_REPO" -b main "$HOME/bootc-edge-ai" ; fi'
[Install]
WantedBy=multi-user.target

49
tekton/README.md

@ -13,6 +13,12 @@ oc apply -k common/
oc apply -f pipeline.yaml
```
## Authentication to Pypi
```sh
oc create secret generic pypi-mirror '--from-literal=PYPI_MIRROR_URL=https://login:password@artifactory-host/artifactory/api/pypi/pypi-virtual/simple'
```
## Authentication to the registries
```sh
@ -32,6 +38,41 @@ data:
EOF
```
```sh
oc create configmap registries-conf --from-file=/etc/containers/registries.conf
```
## Authentication to GitHub
```sh
cat > gitconfig <<EOF
[credential]
helper=store
EOF
oc create secret generic github-authentication --from-literal=.git-credentials=https://user:password@github.com --from-file=.gitconfig=gitconfig
```
## Rclone config for AWS S3
**rclone.conf**:
```ini
[aws]
type = s3
provider = AWS
access_key_id = REDACTED
secret_access_key = REDACTED
region = eu-west-3
```
Note: in **rclone.conf**, set **endpoint** to the hostname of your S3 gateway when on-premise.
Create the secret:
```sh
oc create secret generic rclone-config --from-file=rclone.conf
```
## Initialize data inside the PVC
Create a Pod that uses the two previously created PVC :
@ -54,6 +95,8 @@ spec:
mountPath: /caches
- name: bootc-entitlements
mountPath: /entitlements
- name: bootc-rpms
mountPath: /rpms
volumes:
- name: bootc-caches
persistentVolumeClaim:
@ -61,9 +104,12 @@ spec:
- name: bootc-entitlements
persistentVolumeClaim:
claimName: bootc-entitlements
- name: bootc-rpms
persistentVolumeClaim:
claimName: bootc-rpms
```
Then copy all the data to `/caches` and `/entitlements`.
Then copy all the data to `/caches`, `/rpms` and `/entitlements`.
```sh
mkdir -p entitlements
@ -71,6 +117,7 @@ cp etc-x86_64.tar entitlements/x86_64.tar
cp etc-aarch64.tar entitlements/aarch64.tar
oc rsync entitlements rsync:/
oc rsh rsync mkdir -p /caches/{x86_64,aarch64}/{rpm-ostree,dnf}
tar -c -C /path/to/rpms | oc rsh rsync tar -x -C /rpms
```
You can leave the Pod running or delete it with :

3
tekton/common/kustomization.yaml

@ -1,6 +1,7 @@
resources:
- serviceaccount-buildbot.yaml
- task-buildah-bootc.yaml
- task-buildah.yaml
- task-git-clone.yaml
- task-rclone.yaml
- daemonset-qemu.yaml
- storage.yaml

90
tekton/common/task-buildah-bootc.yaml

@ -1,90 +0,0 @@
apiVersion: tekton.dev/v1beta1
kind: Task
metadata:
name: buildah-bootc
spec:
params:
- name: context-dir
type: string
default: .
- name: containerfile-path
type: string
default: Containerfile
- name: image-name
type: string
workspaces:
- name: source-workspace
description: Workspace containing source code
- name: caches
description: RW storage to cache build artefacts
mountPath: /caches
- name: entitlements
description: RW storage for RHEL entitlements
mountPath: /entitlements
- description: An optional workspace that allows providing a .docker/config.json file for Buildah to access the container registry. The file should be placed at the root of the Workspace with name config.json or .dockerconfigjson.
name: dockerconfig
optional: true
mountPath: /auth
volumes:
- name: container-storage
emptyDir: {}
steps:
- name: build
image: registry.redhat.io/rhel9/buildah:9.6
env:
- name: STORAGE_DRIVER
value: overlay
- name: RHEL_IMAGE
value: registry.redhat.io/rhel9/rhel-bootc
- name: RHEL_VERSION
value: "9.4"
- name: TARGET_IMAGE
value: "$(params.image-name)"
- name: REGISTRY_AUTH_FILE
value: /auth/.dockerconfigjson
script: |
#!/bin/bash
set -Eeuo pipefail
# All architectures to build for
declare -a ARCHITECTURES=("x86_64" "aarch64")
# Build images
declare -A PODMAN_ARCH_OPTS=(["aarch64"]="--platform linux/arm64/v8" ["x86_64"]="--platform linux/amd64")
for arch in "${ARCHITECTURES[@]}"; do
buildah pull ${PODMAN_ARCH_OPTS[$arch]} $RHEL_IMAGE:$RHEL_VERSION
buildah tag $RHEL_IMAGE:$RHEL_VERSION $RHEL_IMAGE-$arch:$RHEL_VERSION
buildah rmi $RHEL_IMAGE:$RHEL_VERSION
echo "Building image for $arch..."
rm -rf /tmp/entitlements
mkdir -p /tmp/entitlements
tar -xf /entitlements/$arch.tar -C /tmp/entitlements
buildah bud ${PODMAN_ARCH_OPTS[$arch]} --no-cache --from "$RHEL_IMAGE-$arch:$RHEL_VERSION" \
-v /tmp/entitlements/etc/pki/entitlement/:/etc/pki/entitlement:z -v /tmp/entitlements/etc/rhsm:/etc/rhsm:z \
-v /tmp/entitlements/etc/pki/entitlement/:/run/secrets/etc-pki-entitlement:z -v /tmp/entitlements/etc/rhsm:/run/secrets/rhsm:z \
-v /tmp/entitlements/etc/yum.repos.d:/etc/yum.repos.d:z -v /caches/$arch/dnf:/var/cache/dnf:z \
-v /caches/$arch/rpm-ostree:/var/cache/rpm-ostree:z \
-t localhost/image-$arch \
-f $(workspaces.source-workspace.path)/$(params.containerfile-path) \
$(workspaces.source-workspace.path)/$(params.context-dir)
done
# Push Manifest
echo "Pushing to $TARGET_IMAGE..."
buildah manifest create localhost/image
for arch in "${ARCHITECTURES[@]}"; do
buildah manifest add localhost/image localhost/image-$arch
done
buildah manifest push localhost/image docker://$TARGET_IMAGE
securityContext:
## Buildah needs privileges to use the "overlay" Storage Driver.
privileged: true
## The "vfs" Storage Driver however requires less privileges.
#capabilities:
# add:
# - SETFCAP
volumeMounts:
- name: container-storage
mountPath: /var/lib/containers

154
tekton/common/task-buildah.yaml

@ -0,0 +1,154 @@
apiVersion: tekton.dev/v1beta1
kind: Task
metadata:
name: buildah
spec:
params:
- name: context-dir
type: string
default: .
- name: containerfile-path
type: string
default: Containerfile
- name: image-name
type: string
- name: buildah-image
type: string
default: registry.redhat.io/rhel9/buildah:latest
- name: build-architectures
type: array
default:
- x86_64
- aarch64
- name: pypi-mirror-url
type: string
optional: true
workspaces:
- name: source-workspace
description: Workspace containing source code
- name: caches
description: RW storage to cache build artefacts
mountPath: /caches
optional: true
- name: entitlements
description: RW storage for RHEL entitlements
mountPath: /entitlements
optional: true
- description: An optional workspace that allows providing a .docker/config.json file for Buildah to access the container registry. The file should be placed at the root of the Workspace with name config.json or .dockerconfigjson.
name: dockerconfig
optional: true
mountPath: /auth
- name: registries-conf
optional: true
mountPath: /registries
volumes:
- name: container-storage
emptyDir: {}
steps:
- name: build
image: $(params.buildah-image)
env:
- name: STORAGE_DRIVER
value: overlay
- name: SCRIPT_DEBUG
value: "false"
- name: TARGET_IMAGE
value: "$(params.image-name)"
- name: PYPI_MIRROR_URL
value: "$(params.pypi-mirror-url)"
args:
- "$(params.build-architectures)"
script: |
#!/bin/bash
set -Eeuo pipefail
# If debug is enabled, print out command that are executed
if [[ "${SCRIPT_DEBUG:-false}" == "true" ]]; then
set -x
fi
# Print versions of the program we use
echo "=== Environment ==="
echo "---> Buildah"
buildah version
echo
echo "---> bash"
bash --version
echo
echo "---> OS"
cat /etc/redhat-release
echo
# Check what is available and set env variables
if [ -f /registries/registries.conf ]; then
export CONTAINERS_REGISTRIES_CONF=/registries/registries.conf
fi
if [ -f /auth/.dockerconfigjson ]; then
export REGISTRY_AUTH_FILE=/auth/.dockerconfigjson
fi
# Extract the parent image name
FROM="$(sed -r 's/^FROM\s+(.*)\s*/\1/;t;d' "$(workspaces.source-workspace.path)/$(params.containerfile-path)")"
echo "Detected $FROM as parent image."
# Build images
declare -A PODMAN_ARCH_OPTS=(["aarch64"]="--platform linux/arm64/v8" ["x86_64"]="--platform linux/amd64")
buildah manifest create localhost/image
for arch; do
declare -a PODMAN_OPTS=( )
if [ -n "${PYPI_MIRROR_URL:}" ]; then
PODMAN_OPTS+=( "--build-arg" "PYPI_MIRROR_URL=${PYPI_MIRROR_URL}" )
fi
if [ -f "/entitlements/$arch.tar" ]; then
echo "Using RHEL entitlements..."
rm -rf /tmp/entitlements
mkdir -p /tmp/entitlements
tar -xf /entitlements/$arch.tar -C /tmp/entitlements
PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/pki/entitlement/:/etc/pki/entitlement:z" )
PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/rhsm:/etc/rhsm:z" )
PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/pki/entitlement/:/run/secrets/etc-pki-entitlement:z" )
PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/rhsm:/run/secrets/rhsm:z" )
PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/rhsm:/run/secrets/rhsm:z" )
PODMAN_OPTS+=( "-v" "/tmp/entitlements/etc/yum.repos.d:/etc/yum.repos.d:z" )
fi
if [ -d "/caches/$arch/" ]; then
echo "Enabling cache..."
PODMAN_OPTS+=( "-v" "/caches/$arch/dnf:/var/cache/dnf:z" )
PODMAN_OPTS+=( "-v" "/caches/$arch/rpm-ostree:/var/cache/rpm-ostree:z" )
fi
if [ -d "/rpms/$arch/" ]; then
echo "Enabling RPM repositories..."
mkdir -p /tmp/rpms
cat > /tmp/rpms/local-rpms.repo <<EOF
[local-rpms]
name=Local RPMs Repository
baseurl=file:///opt/local-repo
enabled=1
gpgcheck=0
EOF
PODMAN_OPTS+=( "-v" "/tmp/rpms:/etc/yum.repos.d:z" )
PODMAN_OPTS+=( "-v" "/rpms/$arch:/opt/local-repo:z" )
fi
PODMAN_OPTS+=( "-f" "$(workspaces.source-workspace.path)/$(params.containerfile-path)" )
PODMAN_OPTS+=( "--no-cache" )
echo "Building image for $arch..."
( set -x ; buildah bud ${PODMAN_ARCH_OPTS[$arch]} "${PODMAN_OPTS[@]}" "-t" "localhost/image-$arch" $(workspaces.source-workspace.path)/$(params.context-dir) )
buildah manifest add localhost/image localhost/image-$arch
buildah rmi "$FROM"
done
# Push Manifest
echo "Pushing to $TARGET_IMAGE..."
buildah manifest push localhost/image docker://$TARGET_IMAGE
securityContext:
## Buildah needs privileges to use the "overlay" Storage Driver.
privileged: true
## The "vfs" Storage Driver however requires less privileges.
#capabilities:
# add:
# - SETFCAP
volumeMounts:
- name: container-storage
mountPath: /var/lib/containers

33
tekton/common/task-rclone.yaml

@ -0,0 +1,33 @@
apiVersion: tekton.dev/v1beta1
kind: Task
metadata:
name: rclone
spec:
params:
- name: rclone-image
type: string
default: docker.io/rclone/rclone:latest
- name: rclone-src
type: array
default: []
- name: rclone-dest
type: string
default: "."
workspaces:
- name: source-workspace
description: Workspace containing source code
- name: rclone-config
optional: true
mountPath: /etc/rclone
steps:
- name: copy
image: $(params.rclone-image)
args:
- "$(params.rclone-src[*])"
script: |
#!/bin/sh
set -eu
cd $(workspaces.source-workspace.path)
for source; do
rclone --config=/etc/rclone/rclone.conf copy --progress "$source" "$(params.rclone-dest)"
done

78
tekton/pipeline.yaml

@ -7,8 +7,17 @@ spec:
workspaces:
- name: source-workspace
- name: registry-token
optional: true
- name: caches
optional: true
- name: entitlements
optional: true
- name: git-auth
optional: true
- name: registries-conf
optional: true
- name: rclone-config
optional: true
params:
- name: git-url
@ -16,14 +25,19 @@ spec:
- name: git-revision
type: string
default: main
- name: image-name
- name: bootc-image-name
type: string
- name: context-dir
- name: app-image-name
type: string
default: "."
- name: containerfile-path
- name: rclone-model-src
type: string
- name: rclone-model-dest
type: string
default: "Containerfile"
- name: build-architectures
type: array
- name: pypi-mirror-url
type: string
optional: true
tasks:
@ -39,18 +53,62 @@ spec:
workspaces:
- name: output
workspace: source-workspace
- name: basic-auth
workspace: git-auth
- name: buildah-bootc
- name: fetch-model
runAfter: ["clone-repo"]
taskRef:
name: buildah-bootc
kind: Task
name: rclone
params:
- name: rclone-src
value:
- $(params.rclone-model-src)
- name: rclone-dest
value: $(params.rclone-model-dest)
workspaces:
- name: source-workspace
workspace: source-workspace
- name: rclone-config
workspace: rclone-config
- name: buildah-app
runAfter: ["fetch-model"]
taskRef:
name: buildah
params:
- name: image-name
value: $(params.app-image-name)
- name: context-dir
value: app
- name: containerfile-path
value: app/Containerfile
- name: build-architectures
value:
- $(params.build-architectures)
- name: pypi-mirror-url
value: $(params.pypi-mirror-url)
workspaces:
- name: source-workspace
workspace: source-workspace
- name: dockerconfig
workspace: registry-token
- name: buildah-bootc
runAfter: ["buildah-app"]
taskRef:
name: buildah
params:
- name: image-name
value: $(params.image-name)
value: $(params.bootc-image-name)
- name: context-dir
value: $(params.context-dir)
value: bootc
- name: containerfile-path
value: $(params.containerfile-path)
value: bootc/Containerfile
- name: build-architectures
value:
- $(params.build-architectures)
workspaces:
- name: source-workspace
workspace: source-workspace

39
tekton/pipelinerun.yaml

@ -10,19 +10,29 @@ spec:
value: https://github.com/nmasse-itix/bootc-edge-ai.git
- name: git-revision
value: main
- name: image-name
- name: bootc-image-name
value: quay.io/nmasse-redhat/bootc-edge-ai
- name: context-dir
value: bootc
- name: containerfile-path
value: "bootc/Containerfile"
- name: app-image-name
value: quay.io/nmasse-redhat/app-edge-ai
- name: rclone-model-src
value: aws:nmasse-bootc-edge-ai/model.onnx
- name: rclone-model-dest
value: app/model-s3.onnx
- name: build-architectures
value:
- aarch64
# - name: pypi-mirror-url
# value: http://...
workspaces:
- name: caches
# - name: caches
# persistentVolumeClaim:
# claimName: bootc-caches
# - name: entitlements
# persistentVolumeClaim:
# claimName: bootc-entitlements
- name: rpms
persistentVolumeClaim:
claimName: bootc-caches
- name: entitlements
persistentVolumeClaim:
claimName: bootc-entitlements
claimName: bootc-rpms
- name: source-workspace
volumeClaimTemplate:
spec:
@ -35,5 +45,14 @@ spec:
- name: registry-token
secret:
secretName: registry-authentication
- name: rclone-config
secret:
secretName: rclone-config
#- name: git-token
# secret:
# secretName: github-authentication
#- name: registries-conf
# configMap:
# name: registries-conf
taskRunTemplate:
serviceAccountName: buildbot

Loading…
Cancel
Save