Create skeleton scripts for ci gpu build.

This commit is contained in:
Kishore Nallan 2023-07-24 20:27:03 +05:30
parent e623454ec5
commit 1f026d4266
3 changed files with 128 additions and 0 deletions

42
ci_build_v2.sh Normal file
View File

@ -0,0 +1,42 @@
#!/bin/bash
# TYPESENSE_VERSION=nightly TYPESENSE_TARGET=typesense-server|typesense-test bash ci_build.sh
set -ex
PROJECT_DIR=`dirname $0 | while read a; do cd $a && pwd && break; done`
BUILD_DIR=bazel-bin
if [ -z "$TYPESENSE_VERSION" ]; then
TYPESENSE_VERSION="nightly"
fi
ARCH_NAME="amd64"
if [[ "$@" == *"--graviton2"* ]] || [[ "$@" == *"--arm"* ]]; then
ARCH_NAME="arm64"
fi
if [[ "$@" == *"--with-cuda"* ]]; then
CUDA_FLAGS="--define use_cuda=on --action_env=CUDA_HOME=/usr/local/cuda --action_env=CUDNN_HOME=/usr/local/cuda"
fi
bazel build --verbose_failures --jobs=6 $CUDA_FLAGS \
--define=TYPESENSE_VERSION=\"$TYPESENSE_VERSION\" //:$TYPESENSE_TARGET
if [[ "$@" == *"--build-deploy-image"* ]]; then
echo "Creating deployment image for Typesense $TYPESENSE_VERSION server ..."
docker build --platform linux/${ARCH_NAME} --file $PROJECT_DIR/docker/deployment.Dockerfile \
--tag typesense/typesense:$TYPESENSE_VERSION $PROJECT_DIR/$BUILD_DIR
fi
if [[ "$@" == *"--package-binary"* ]]; then
OS_FAMILY=linux
RELEASE_NAME=typesense-server-$TYPESENSE_VERSION-$OS_FAMILY-$ARCH_NAME
printf `md5sum $PROJECT_DIR/$BUILD_DIR/typesense-server | cut -b-32` > $PROJECT_DIR/$BUILD_DIR/typesense-server.md5.txt
tar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz -C $PROJECT_DIR/$BUILD_DIR typesense-server typesense-server.md5.txt
echo "Built binary successfully: $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz"
GPU_DEPS_NAME=typesense-gpu-deps-$TYPESENSE_VERSION-$OS_FAMILY-$ARCH_NAME
tar -cvzf $PROJECT_DIR/$BUILD_DIR/$GPU_DEPS_NAME.tar.gz -C $PROJECT_DIR/$BUILD_DIR libonnxruntime_providers_cuda.so libonnxruntime_providers_shared.so
echo "Built binary successfully: $PROJECT_DIR/$BUILD_DIR/$GPU_DEPS_NAME.tar.gz"
fi

View File

@ -0,0 +1,79 @@
#!/bin/bash
# TSV is passed as an environment variable to the script
if [ -z "$TSV" ]
then
echo '$TSV is not provided. Quitting.'
exit 1
fi
if [ -z "$ARCH" ]
then
echo '$ARCH is not provided. Quitting.'
exit 1
fi
RPM_ARCH=$ARCH
if [ "$ARCH" == "amd64" ]; then
RPM_ARCH="x86_64"
fi
set -ex
CURR_DIR=`dirname $0 | while read a; do cd $a && pwd && break; done`
rm -rf /tmp/typesense-gpu-deb-build && mkdir /tmp/typesense-gpu-deb-build
cp -r $CURR_DIR/typesense-gpu-deps /tmp/typesense-gpu-deb-build
rm -rf /tmp/typesense-gpu-deps-$TSV && mkdir /tmp/typesense-gpu-deps-$TSV
tar -xzf $CURR_DIR/../bazel-bin/typesense-gpu-deps-$TSV-linux-${ARCH}.tar.gz -C /tmp/typesense-gpu-deps-$TSV
rm -rf /tmp/typesense-gpu-deps-$TSV /tmp/typesense-gpu-deps-$TSV.tar.gz
sed -i "s/\$VERSION/$TSV/g" `find /tmp/typesense-gpu-deb-build -maxdepth 10 -type f`
sed -i "s/\$ARCH/$ARCH/g" `find /tmp/typesense-gpu-deb-build -maxdepth 10 -type f`
dpkg-deb -Zgzip -z6 \
-b /tmp/typesense-gpu-deb-build/typesense-gpu-deps "/tmp/typesense-gpu-deb-build/typesense-gpu-deps-${TSV}-${ARCH}.deb"
# Generate RPM
rm -rf /tmp/typesense-gpu-rpm-build && mkdir /tmp/typesense-gpu-rpm-build
cp "/tmp/typesense-gpu-deb-build/typesense-gpu-deps-${TSV}-${ARCH}.deb" /tmp/typesense-gpu-rpm-build
cd /tmp/typesense-gpu-rpm-build && alien --scripts -k -r -g -v /tmp/typesense-gpu-rpm-build/typesense-gpu-deps-${TSV}-${ARCH}.deb
sed -i 's#%dir "/"##' `find /tmp/typesense-gpu-rpm-build/*/*.spec -maxdepth 10 -type f`
sed -i 's#%dir "/usr/bin/"##' `find /tmp/typesense-gpu-rpm-build/*/*.spec -maxdepth 10 -type f`
sed -i 's/%config/%config(noreplace)/g' `find /tmp/typesense-gpu-rpm-build/*/*.spec -maxdepth 10 -type f`
SPEC_FILE="/tmp/typesense-gpu-rpm-build/typesense-gpu-deps-${TSV}/typesense-gpu-deps-${TSV}-1.spec"
SPEC_FILE_COPY="/tmp/typesense-gpu-rpm-build/typesense-gpu-deps-${TSV}/typesense-gpu-deps-${TSV}-copy.spec"
cp $SPEC_FILE $SPEC_FILE_COPY
PRE_LINE=`grep -n "%pre" $SPEC_FILE_COPY | cut -f1 -d:`
START_LINE=`expr $PRE_LINE - 1`
head -$START_LINE $SPEC_FILE_COPY > $SPEC_FILE
echo "%prep" >> $SPEC_FILE
echo "cat >/tmp/find_requires.sh <<EOF
#!/bin/sh
%{__find_requires} | grep -v GLIBC_PRIVATE
exit 0
EOF" >> $SPEC_FILE
echo "chmod +x /tmp/find_requires.sh" >> $SPEC_FILE
echo "%define _use_internal_dependency_generator 0" >> $SPEC_FILE
echo "%define __find_requires /tmp/find_requires.sh" >> $SPEC_FILE
tail -n+$START_LINE $SPEC_FILE_COPY >> $SPEC_FILE
rm $SPEC_FILE_COPY
cd /tmp/typesense-gpu-rpm-build/typesense-gpu-deps-${TSV} && \
rpmbuild --target=${RPM_ARCH} --buildroot /tmp/typesense-gpu-rpm-build/typesense-gpu-deps-${TSV} -bb \
$SPEC_FILE
cp "/tmp/typesense-gpu-rpm-build/typesense-gpu-deps-${TSV}-${ARCH}.deb" $CURR_DIR/../bazel-bin
cp "/tmp/typesense-gpu-rpm-build/typesense-gpu-deps-${TSV}-1.${RPM_ARCH}.rpm" $CURR_DIR/../bazel-bin

View File

@ -0,0 +1,7 @@
Package: typesense-gpu
Version: $VERSION
Maintainer: Typesense Inc. <contact@typesense.org>
Description: Typesense GPU Dependencies
Homepage: https://typesense.org
Architecture: $ARCH
Depends: curl