Added GitHub Actions

prabhuomkar · prabhuomkar · commit 243f35228d09 · 2025-03-31T18:57:50.000+05:30
diff --git a/.github/workflows/pyworker.yaml b/.github/workflows/pyworker.yaml
@@ -54,7 +54,7 @@ jobs:
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: ./worker
+        working-directory: ./pyworker
     steps:
       - name: Git Checkout
         uses: actions/checkout@v4
diff --git a/.github/workflows/worker.yaml b/.github/workflows/worker.yaml
@@ -0,0 +1,87 @@
+name: Smriti Worker CI
+on:
+  push:
+    branches:    
+      - master
+    paths:
+      - worker/**
+      - .github/workflows/worker.yaml
+  pull_request:
+    branches:
+      - master
+    paths:
+      - worker/**
+      - .github/workflows/worker.yaml
+  release:
+    types: [published]
+jobs:
+  ci:
+    name: Integration Check
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./worker
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v4
+      - name: Setup cmake
+        uses: jwlawson/actions-setup-cmake@v2
+        with:
+          cmake-version: '3.30.x'
+      - name: CMake Version
+        run: cmake --version
+      - name: Install Dependencies
+        run: |
+          sudo apt install libspdlog-dev
+      - name: Install Lint
+        run: |
+          pip install cpplint
+          sudo apt install -y clang-tidy clang-format
+          cpplint --version
+          clang-tidy --version
+          clang-format --version
+      - name: Run Lint
+        run: make lint
+      - name: Run Build
+        run: make build
+      - name: Run Test & Cover
+        run: |
+          make test-install
+          make cover
+      - name: Publish Coverage
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: worker
+          files: worker/coverage.xml
+  publish:
+    if: ${{ (github.event_name == 'release' && github.event.action == 'published') || github.ref == 'refs/heads/master' }}
+    needs: ci
+    name: Publish Docker Image
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./worker
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v4
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Docker Build and Push
+        uses: docker/build-push-action@v6
+        with:
+          context: worker
+          file: ./worker/Dockerfile
+          push: true
+          platforms: linux/amd64,linux/arm64/v8
+          build-args: |
+            GITSHA=${{ github.sha }}
+            VERSION=${ github.ref_name }}
+          tags: smritihq/worker:${{ github.ref_name }}
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,9 @@
+[submodule "worker/third_party/spdlog"]
+	path = worker/third_party/spdlog
+	url = https://github.com/gabime/spdlog.git
+[submodule "worker/third_party/googletest"]
+	path = worker/third_party/googletest
+	url = https://github.com/google/googletest.git
+[submodule "worker/third_party/benchmark"]
+	path = worker/third_party/benchmark
+	url = https://github.com/google/benchmark.git
diff --git a/Makefile b/Makefile
@@ -1,4 +1,5 @@
-setup: setup-api setup-worker setup-docs setup-tests
+setup: setup-api setup-pyworker setup-docs setup-tests
+
 setup-api:
 	@cd api; \
 		echo "[setup-api]: Verifying go modules..."; \
@@ -7,32 +8,49 @@ setup-api:
 		make lint; \
 		echo "[setup-api]: Running unit tests..."; \
 		make test
+
 setup-worker:
 	@cd worker; \
-		echo "[setup-worker]: Installing requirements"; \
+		echo "[setup-worker]: Installing dependencies"; \
 		make install; \
 		make test-install; \
 		echo "[setup-worker]: Running linter..."; \
 		make lint; \
 		echo "[setup-worker]: Running unit tests..."; \
 		make test
+
+setup-pyworker:
+	@cd pyworker; \
+		echo "[setup-pyworker]: Installing requirements"; \
+		make install; \
+		make test-install; \
+		echo "[setup-pyworker]: Running linter..."; \
+		make lint; \
+		echo "[setup-pyworker]: Running unit tests..."; \
+		make test
+
 setup-docs:
 	@cd docs; \
 		echo "[setup-docs]: Installing dependencies"; \
 		npm install
+
 setup-tests:
 	@cd tests; \
 		echo "[setup-tests]: Installing dependencies"; \
 		make setup
+
 setup-models:
 	@echo "Setting up models..."; \
 		python3 scripts/setup_models.py
+
 start: setup-models
 	@echo "Starting smriti services..."; \
 		docker compose up -d
+
 stop:
 	@echo "Stopping smriti services..."; \
 		docker compose down
+
 gen-test-data: start
 	@echo "Generating test data"; \
 		python3 scripts/generate_test_data.py
diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 Smarter Home for all your Photos and Videos
 
 [![API](https://github.com/prabhuomkar/smriti/actions/workflows/api.yaml/badge.svg)](https://github.com/prabhuomkar/smriti/actions/workflows/api.yaml)
-[![Worker](https://github.com/prabhuomkar/smriti/actions/workflows/worker.yaml/badge.svg)](https://github.com/prabhuomkar/smriti/actions/workflows/worker.yaml)
+[![Worker](https://github.com/prabhuomkar/smriti/actions/workflows/pyworker.yaml/badge.svg)](https://github.com/prabhuomkar/smriti/actions/workflows/worker.yaml)
 [![Coverage](https://codecov.io/gh/prabhuomkar/smriti/branch/master/graph/badge.svg?token=D32LxO5fIj)](https://codecov.io/gh/prabhuomkar/smriti) 
 [![License](https://img.shields.io/github/license/prabhuomkar/smriti)](LICENSE) 
 [![Twitter](https://img.shields.io/twitter/follow/smritihq?style=social)](https://twitter.com/smritihq)
diff --git a/api/Makefile b/api/Makefile
@@ -1,21 +1,28 @@
+.PHONY: build
 build: 
 	@go build -o api --ldflags="-X 'api/internal/models.DefaultVersion=$(VERSION)' -X 'api/internal/models.DefaultGitSHA=$(shell git rev-parse HEAD)'"
 
+.PHONY: run
 run: build
 	@./api
 
+.PHONY: test
 test: 
 	@go test ./...
 
+.PHONY: cover
 cover: 
 	@go test $$(go list ./... | grep -v api/pkg/services) -race -coverprofile=coverage.txt -covermode=atomic ./...
 	@go tool cover -func coverage.txt | grep total
 
+.PHONY: cover-html
 cover-html: cover
 	@go tool cover -html coverage.txt -o coverage.html
 
+.PHONY: lint
 lint:
 	@LOG_LEVEL=error golangci-lint run ./...
 
+.PHONY: proto
 proto:
 	@protoc --proto_path=../protos ../protos/*.proto --go_out=pkg/services --go-grpc_out=pkg/services --experimental_allow_proto3_optional
diff --git a/docs/docs/dev-guide/environment.md b/docs/docs/dev-guide/environment.md
@@ -5,6 +5,7 @@ The following guide will help you in setting up the development environment to w
 Make sure, before you start any development, following things are installed and available on your system of choice: 
 - [Git](https://git-scm.com/)
 - [Docker](https://www.docker.com/)
+- [Protocol Buffers](https://protobuf.dev/installation/)
 - [Common Sense](https://en.wikipedia.org/wiki/Common_sense)
 
 ## Getting Codebase
@@ -24,11 +25,20 @@ make setup-api
 ```
 
 ### Worker
+- Install [CMake 3.30](https://cmake.org/download/) or above
+- Install [cpplint 2.0.0](https://pypi.org/project/cpplint/) or above
+- Install [clang-tidy 19.1.7](https://clang.llvm.org/extra/clang-tidy/) or above
+- Install [clang-format 19.1.7](https://clang.llvm.org/docs/ClangFormat.html) or above
+```
+make setup-worker
+```
+
+### PyWorker
 - Install [Python 3.12](https://www.python.org/downloads/) or above
 - Install [pylint 3.3.4](https://pypi.org/project/pylint/) or above
 - Run the following command to finish your Worker setup
 ```
-make setup-worker
+make setup-pyworker
 ```
 
 ### Docs
diff --git a/infra/deployments/docker-swarm/docker-compose.yaml b/infra/deployments/docker-swarm/docker-compose.yaml
@@ -20,7 +20,7 @@ services:
     volumes:
       - ./storage:/storage:rw
   worker:
-    image: smritihq/worker:master
+    image: smritihq/pyworker:master
     ports:
       - '15002:15002'
     environment:
diff --git a/pyworker/README.md b/pyworker/README.md
@@ -1,3 +1,3 @@
-# Smriti Worker
+# Smriti PyWorker
 
-Setup Guide is available [here](https://smriti.omkar.xyz/docs/dev-guide/environment#worker).
+Setup Guide is available [here](https://smriti.omkar.xyz/docs/dev-guide/environment#pyworker).
diff --git a/worker/CMakeLists.txt b/worker/CMakeLists.txt
@@ -13,7 +13,7 @@ execute_process(
 if(NOT GIT_SHA)
     set(GIT_SHA "-")
 endif()
-find_package(spdlog REQUIRED)
+add_subdirectory(third_party/spdlog)
 add_executable(worker
     src/main.cc
     src/metadata.cc
diff --git a/worker/Dockerfile b/worker/Dockerfile
@@ -0,0 +1,34 @@
+FROM ubuntu:24.10 AS builder
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    cmake \
+    git \
+    wget \
+    autoconf \
+    automake \
+    libtool \
+    pkg-config \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /opt
+RUN git clone --recurse-submodules -b v1.71.0 https://github.com/grpc/grpc.git && \
+    cd grpc && \
+    mkdir -p cmake/build && cd cmake/build && \
+    cmake -DgRPC_INSTALL=ON \
+          -DgRPC_BUILD_TESTS=OFF \
+          -DCMAKE_INSTALL_PREFIX=/usr/local \
+          ../.. && \
+    make -j$(nproc) && \
+    make install
+WORKDIR /app
+COPY . /app
+RUN mkdir -p build && cd build && \
+    cmake -DCMAKE_PREFIX_PATH=/usr/local .. && \
+    make -j$(nproc)
+
+FROM gcr.io/distroless/cc
+WORKDIR /app
+COPY --from=builder /app/build/worker .
+EXPOSE 5002
+EXPOSE 15002
+CMD ["/app/worker"]
diff --git a/worker/Makefile b/worker/Makefile
@@ -0,0 +1,60 @@
+GRPC_VERSION ?= v1.71.0
+MY_INSTALL_DIR ?= ${HOME}/.local
+
+.PHONY: install
+install:
+	@if [ ! -d "grpc" ]; then \
+		git clone --recurse-submodules -b ${GRPC_VERSION} --depth 1 --shallow-submodules https://github.com/grpc/grpc; \
+	else \
+		echo "grpc already downloaded"; \
+		cd grpc && git checkout ${GRPC_VERSION}; \
+		git submodule update --init --recursive; \
+	fi
+	@cd grpc && \
+		mkdir -p cmake/build && \
+		cd cmake/build && \
+		cmake -DgRPC_INSTALL=ON \
+			-DgRPC_BUILD_TESTS=OFF \
+			-DCMAKE_CXX_STANDARD=17 \
+			-DCMAKE_INSTALL_PREFIX=${MY_INSTALL_DIR} \
+      		../.. && \
+		make -j 4 && \
+		make install && \
+		cd ../..
+	@rm -rf grpc
+
+.PHONY: build
+build:
+	@mkdir -p build
+	@cmake -DCMAKE_PREFIX_PATH=${MY_INSTALL_DIR} . -B build
+	@cd build && make -j 4
+
+.PHONY: run
+run:
+	@./build/worker
+
+.PHONY: test-install
+test-install:
+	@git submodule update --init --recursive
+
+.PHONY: test
+test: 
+	@./build/worker_tests
+	@./build/worker_benchmarks
+
+.PHONY: cover
+cover: 
+	@echo "TODO(omkar)"
+
+.PHONY: cover-html
+cover-html: cover
+	@echo "TODO(omkar)"
+
+.PHONY: lint
+lint:
+	@cpplint --recursive include/ src/ tests/
+	@clang-tidy src/*.cc -- -std=c++17 -I./include -I/usr/include -I/usr/local/include/ -I/opt/homebrew/include/
+
+.PHONY: proto
+proto:
+	@protoc -I../ --cpp_out=./generated ../protos/*.proto
diff --git a/worker/README.md b/worker/README.md
@@ -0,0 +1,3 @@
+# Smriti Worker
+
+Setup Guide is available [here](https://smriti.omkar.xyz/docs/dev-guide/environment#worker).
diff --git a/worker/third_party/benchmark b/worker/third_party/benchmark
@@ -0,0 +1 @@
+Subproject commit 0da57b85cf23e48d0e515f58c65a25425dbde012
diff --git a/worker/third_party/googletest b/worker/third_party/googletest
@@ -0,0 +1 @@
+Subproject commit 52204f78f94d7512df1f0f3bea1d47437a2c3a58
diff --git a/worker/third_party/spdlog b/worker/third_party/spdlog
@@ -0,0 +1 @@
+Subproject commit 48bcf39a661a13be22666ac64db8a7f886f2637e
