From bc3f4f5724fa514a220e9051fe0dbc9959f6c958 Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Fri, 16 Jun 2023 16:55:50 -0500 Subject: [PATCH 01/13] Round 1 --- .circleci/config.yml | 32 ------------------------- .github/tests.yml | 56 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 32 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/tests.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index ce8cb697..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: typesense/typesense-development:03-JAN-2023-1 - environment: - - PROJECT_DIR: /typesense - - TYPESENSE_VERSION: $CIRCLE_BRANCH-$CIRCLE_SHA1 - working_directory: /typesense - steps: - - checkout - - run: - name: generate cache key source - command: git log --pretty=format:'%H' -n 1 -- cmake CMakeLists.txt build.sh > last-changed-git-sha-for-dependency-listing - - restore_cache: - keys: - - external-Linux-cache-{{ .Branch }}-{{ checksum "last-changed-git-sha-for-dependency-listing" }} - - external-Linux-cache-{{ .Branch }} - - external-Linux-cache - - run: - name: build - command: $PROJECT_DIR/build.sh --clean - - save_cache: - key: external-Linux-cache-{{ .Branch }}-{{ checksum "last-changed-git-sha-for-dependency-listing" }} - paths: - - /typesense/external-Linux - - store_artifacts: - path: /typesense/build-Linux - destination: build - - run: - name: test - command: $PROJECT_DIR/build-Linux/typesense-test diff --git a/.github/tests.yml b/.github/tests.yml new file mode 100644 index 00000000..a9fb0084 --- /dev/null +++ b/.github/tests.yml @@ -0,0 +1,56 @@ +name: tests + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Bazel + uses: bazelbuild/setup-bazelisk@v2 + + - name: Mount bazel cache + uses: actions/cache@v3 + with: + path: "~/.cache/bazel" + key: bazel + + - name: Build + run: bazel build //:typesense-server + + - name: Save build artifacts + uses: actions/upload-artifact@v3 + with: + name: build-artifacts + path: bazel-bin + + test: + runs-on: ubuntu-latest + + needs: build + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Bazel + uses: bazelbuild/setup-bazelisk@v2 + + - name: Mount bazel cache + uses: actions/cache@v3 + with: + path: "~/.cache/bazel" + key: bazel + + - name: Download build artifacts + uses: actions/download-artifact@v3 + with: + name: build-artifacts + path: bazel-bin + + - name: Run tests + run: bazel test //:typesense-test From 3017e3d779d03400324c52779b14f8bbb231c4e4 Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Fri, 16 Jun 2023 16:59:02 -0500 Subject: [PATCH 02/13] Fix path --- .github/{ => workflows}/tests.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/tests.yml (100%) diff --git a/.github/tests.yml b/.github/workflows/tests.yml similarity index 100% rename from .github/tests.yml rename to .github/workflows/tests.yml From b8b708b90601c0ee1a0bcb26d0671da74a94ce20 Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Fri, 16 Jun 2023 17:12:08 -0500 Subject: [PATCH 03/13] Install dependencies --- .github/workflows/tests.yml | 72 +++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a9fb0084..294b4484 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,13 +3,49 @@ name: tests on: [push, pull_request] jobs: - build: + build_and_test: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v3 + - name: Install dependencies + run: | + add-apt-repository -y ppa:ubuntu-toolchain-r/test + apt-get update + + apt install -y g++-10 make git zlib1g-dev m4 + + # Define the compiler + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 + + update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 + update-alternatives --set cc /usr/bin/gcc + + update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 + update-alternatives --set c++ /usr/bin/g++ + + # Install CUDA + + wget https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run + sh cuda_11.8.0_520.61.05_linux.run + rm cuda_11.8.0_520.61.05_linux.run + + # Install cudnn + tar -xf cudnn-linux-x86_64-8.9.2.26_cuda11-archive.tar.xz + mv cudnn-linux-x86_64-8.9.2.26_cuda11-archive cudnn + cp cudnn/include/cudnn*.h /usr/local/cuda/include + cp cudnn/lib/libcudnn* /usr/local/cuda/lib64 + chmod a+r /usr/local/cuda/include/cudnn*.h /usr/local/cuda/lib64/libcudnn* + rm -rf cudnn* + + # Copy these to ~/.profile + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/cuda/lib64" + export CUDA_HOME=/usr/local/cuda + export PATH="/usr/local/cuda/bin:$PATH" + - name: Set up Bazel uses: bazelbuild/setup-bazelisk@v2 @@ -22,35 +58,11 @@ jobs: - name: Build run: bazel build //:typesense-server + - name: Run tests + run: bazel test //:typesense-test + - name: Save build artifacts uses: actions/upload-artifact@v3 with: - name: build-artifacts - path: bazel-bin - - test: - runs-on: ubuntu-latest - - needs: build - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Bazel - uses: bazelbuild/setup-bazelisk@v2 - - - name: Mount bazel cache - uses: actions/cache@v3 - with: - path: "~/.cache/bazel" - key: bazel - - - name: Download build artifacts - uses: actions/download-artifact@v3 - with: - name: build-artifacts - path: bazel-bin - - - name: Run tests - run: bazel test //:typesense-test + name: typesense-server + path: bazel-bin/typesense-server From 0fba094c2a81c496a8ac7d73eae0fae3c5f898bc Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Fri, 16 Jun 2023 17:17:18 -0500 Subject: [PATCH 04/13] Use sudo to install --- .github/workflows/tests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 294b4484..7f633bf7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,20 +12,20 @@ jobs: - name: Install dependencies run: | - add-apt-repository -y ppa:ubuntu-toolchain-r/test - apt-get update + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update - apt install -y g++-10 make git zlib1g-dev m4 + sudo apt install -y g++-10 make git zlib1g-dev m4 # Define the compiler - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 - update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 - update-alternatives --set cc /usr/bin/gcc + sudo update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 + sudo update-alternatives --set cc /usr/bin/gcc - update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 - update-alternatives --set c++ /usr/bin/g++ + sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 + sudo update-alternatives --set c++ /usr/bin/g++ # Install CUDA @@ -42,9 +42,9 @@ jobs: rm -rf cudnn* # Copy these to ~/.profile - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/cuda/lib64" - export CUDA_HOME=/usr/local/cuda - export PATH="/usr/local/cuda/bin:$PATH" + echo "LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH:/usr/local/cuda/lib64)" >> $GITHUB_ENV + echo "CUDA_HOME=/usr/local/cuda" >> $GITHUB_ENV + echo "PATH=$(echo /usr/local/cuda/bin:$PATH) >> $GITHUB_ENV - name: Set up Bazel uses: bazelbuild/setup-bazelisk@v2 From df144f5414b6aca29cc29a8f65e0a1299c27ac20 Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Fri, 16 Jun 2023 17:21:30 -0500 Subject: [PATCH 05/13] Use bash instead of sh --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7f633bf7..5ce1215f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,7 +30,7 @@ jobs: # Install CUDA wget https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run - sh cuda_11.8.0_520.61.05_linux.run + bash cuda_11.8.0_520.61.05_linux.run rm cuda_11.8.0_520.61.05_linux.run # Install cudnn From 7cac0f4f542dc201e05cfb2f5ae1f991ebf1e987 Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Fri, 16 Jun 2023 17:24:57 -0500 Subject: [PATCH 06/13] Cancel multiple runs --- .github/workflows/tests.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5ce1215f..9af767db 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,6 +2,12 @@ name: tests on: [push, pull_request] +# Cancel previous running if a new push is made +# Source: https://stackoverflow.com/a/72408109/123545 +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build_and_test: runs-on: ubuntu-latest @@ -15,7 +21,7 @@ jobs: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get update - sudo apt install -y g++-10 make git zlib1g-dev m4 + sudo apt-get install -y g++-10 make git zlib1g-dev m4 # Define the compiler sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 @@ -29,7 +35,7 @@ jobs: # Install CUDA - wget https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run + wget --no-verbose https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run bash cuda_11.8.0_520.61.05_linux.run rm cuda_11.8.0_520.61.05_linux.run From c95c75ac12f4a5db580a3832f416198367dcbf73 Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Fri, 16 Jun 2023 17:30:14 -0500 Subject: [PATCH 07/13] Use github action from marketplace to install cuda --- .github/workflows/tests.yml | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9af767db..a5cc070b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,25 +32,12 @@ jobs: sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 sudo update-alternatives --set c++ /usr/bin/g++ - - # Install CUDA - - wget --no-verbose https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run - bash cuda_11.8.0_520.61.05_linux.run - rm cuda_11.8.0_520.61.05_linux.run - - # Install cudnn - tar -xf cudnn-linux-x86_64-8.9.2.26_cuda11-archive.tar.xz - mv cudnn-linux-x86_64-8.9.2.26_cuda11-archive cudnn - cp cudnn/include/cudnn*.h /usr/local/cuda/include - cp cudnn/lib/libcudnn* /usr/local/cuda/lib64 - chmod a+r /usr/local/cuda/include/cudnn*.h /usr/local/cuda/lib64/libcudnn* - rm -rf cudnn* - - # Copy these to ~/.profile - echo "LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH:/usr/local/cuda/lib64)" >> $GITHUB_ENV - echo "CUDA_HOME=/usr/local/cuda" >> $GITHUB_ENV - echo "PATH=$(echo /usr/local/cuda/bin:$PATH) >> $GITHUB_ENV + + - name: Install cuda + uses: Jimver/cuda-toolkit@v0.2.10 + id: cuda-toolkit + with: + cuda: '11.8.0' - name: Set up Bazel uses: bazelbuild/setup-bazelisk@v2 From b7a299bbe739837028953781581a408b41f78c12 Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Fri, 16 Jun 2023 17:40:15 -0500 Subject: [PATCH 08/13] Disable CUDA for now --- .github/workflows/tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a5cc070b..ddccec3b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,12 +33,6 @@ jobs: sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 sudo update-alternatives --set c++ /usr/bin/g++ - - name: Install cuda - uses: Jimver/cuda-toolkit@v0.2.10 - id: cuda-toolkit - with: - cuda: '11.8.0' - - name: Set up Bazel uses: bazelbuild/setup-bazelisk@v2 From 27b29355dc0aa4c54ddb110c89f91378a55a7dbd Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Sat, 17 Jun 2023 12:12:54 -0500 Subject: [PATCH 09/13] Try running on Ubuntu 20.04 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ddccec3b..875d7508 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ concurrency: jobs: build_and_test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout code From c3659cae8a57c02e4253a5e06f72a4ca1d39828b Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Sat, 17 Jun 2023 18:51:00 -0500 Subject: [PATCH 10/13] Disable saving artifact for now --- .github/workflows/tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 875d7508..972780a0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,8 +48,8 @@ jobs: - name: Run tests run: bazel test //:typesense-test - - name: Save build artifacts - uses: actions/upload-artifact@v3 - with: - name: typesense-server - path: bazel-bin/typesense-server +# - name: Save build artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: typesense-server +# path: bazel-bin/typesense-server From 199934da4815a466caf83cec3bad632e3f34d211 Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Sun, 18 Jun 2023 00:42:09 -0500 Subject: [PATCH 11/13] Renable saving build artifact --- .github/workflows/tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 972780a0..875d7508 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,8 +48,8 @@ jobs: - name: Run tests run: bazel test //:typesense-test -# - name: Save build artifacts -# uses: actions/upload-artifact@v3 -# with: -# name: typesense-server -# path: bazel-bin/typesense-server + - name: Save build artifacts + uses: actions/upload-artifact@v3 + with: + name: typesense-server + path: bazel-bin/typesense-server From 0cb69b73b7c1472b03b23ba7eeb6c58daf46ccfe Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Sun, 18 Jun 2023 00:44:51 -0500 Subject: [PATCH 12/13] ls --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 875d7508..a8c07266 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,6 +48,9 @@ jobs: - name: Run tests run: bazel test //:typesense-test + - name: List dir + run: ls -al /home/runner/work/typesense/typesense/ + - name: Save build artifacts uses: actions/upload-artifact@v3 with: From e7c9bee06ae1a3f10b48deaab263d4e4c551f1c7 Mon Sep 17 00:00:00 2001 From: Jason Bosco Date: Sun, 18 Jun 2023 00:51:36 -0500 Subject: [PATCH 13/13] Read symlink and then upload --- .github/workflows/tests.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a8c07266..640fc88a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,11 +48,10 @@ jobs: - name: Run tests run: bazel test //:typesense-test - - name: List dir - run: ls -al /home/runner/work/typesense/typesense/ - + # Source: https://github.com/actions/upload-artifact/issues/92#issuecomment-1080347032 + - run: echo "BAZEL_BIN_FULL_PATH=$(readlink -f bazel-bin)" >> $GITHUB_ENV - name: Save build artifacts uses: actions/upload-artifact@v3 with: name: typesense-server - path: bazel-bin/typesense-server + path: ${{ env.BAZEL_BIN_FULL_PATH }}/typesense-server